From 05651b9f50d367b6b7b92d628d258112985390f0 Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Tue, 25 Jul 2023 10:56:24 -0400 Subject: [PATCH] mainnet core models (#34) * mainnet core models * docs ref to base * workflow cron comment * weird error * exclude goerli tests * overview --------- Co-authored-by: drethereum --- .../workflows/dbt_run_full_observability.yml | 47 +++ .github/workflows/dbt_run_incremental.yml | 6 +- .../workflows/dbt_run_streamline_realtime.yml | 4 +- .../{dbt_test.yml => dbt_test_daily.yml} | 6 +- ..._run_30_mins.yml => dbt_test_intraday.yml} | 21 +- dbt_project.yml | 6 +- macros/streamline/models.sql | 6 +- macros/tests/missing_txs.sql | 103 +++++ .../L1_fields/base_max_block.md | 5 + .../L1_fields/base_min_block.md | 5 + .../doc_descriptions/L1_fields/batch_size.md | 5 + .../L1_fields/l1_batch_root.md | 5 + .../L1_fields/l1_block_number.md | 5 + .../L1_fields/l1_block_timestamp.md | 5 + .../L1_fields/l1_fee_scalar.md | 5 + .../L1_fields/l1_gas_price.md | 5 + .../doc_descriptions/L1_fields/l1_gas_used.md | 5 + .../L1_fields/l1_state_root_batch_index.md | 5 + .../L1_fields/l1_state_root_tx_hash.md | 5 + .../L1_fields/l1_submission_batch_index.md | 5 + .../L1_fields/l1_submission_tx_hash.md | 5 + .../L1_fields/prev_total_elements.md | 5 + .../blocks/base_block_header_json.md | 5 + .../blocks/base_blockchain.md | 5 + .../blocks/base_blocks_hash.md | 5 + .../blocks/base_blocks_nonce.md | 5 + .../blocks/base_blocks_table_doc.md | 5 + .../blocks/base_difficulty.md | 5 + .../blocks/base_extra_data.md | 5 + .../doc_descriptions/blocks/base_gas_limit.md | 5 + .../doc_descriptions/blocks/base_gas_used.md | 5 + models/doc_descriptions/blocks/base_miner.md | 5 + .../doc_descriptions/blocks/base_network.md | 5 + .../blocks/base_parent_hash.md | 5 + .../blocks/base_receipts_root.md | 5 + .../blocks/base_sha3_uncles.md | 5 + models/doc_descriptions/blocks/base_size.md | 5 + .../blocks/base_total_difficulty.md | 5 + .../doc_descriptions/blocks/base_tx_count.md | 5 + .../blocks/base_uncle_blocks.md | 5 + .../event_logs/base_event_index.md | 5 + .../event_logs/base_event_inputs.md | 5 + .../event_logs/base_event_name.md | 5 + .../event_logs/base_event_removed.md | 5 + .../event_logs/base_event_sig.md | 5 + .../event_logs/base_log_id_events.md | 5 + .../event_logs/base_logs_contract_address.md | 5 + .../event_logs/base_logs_contract_name.md | 5 + .../event_logs/base_logs_data.md | 5 + .../event_logs/base_logs_table_doc.md | 5 + .../event_logs/base_logs_tx_hash.md | 5 + .../event_logs/base_origin_from.md | 5 + .../event_logs/base_origin_to.md | 5 + .../event_logs/base_topics.md | 5 + .../doc_descriptions/general/__overview__.md | 9 +- .../general/base_block_number.md | 5 + .../general/base_block_timestamp.md | 5 + .../doc_descriptions/general/base_decimals.md | 5 + .../general/base_from_address.md | 5 + .../general/base_ingested_at.md | 5 + .../doc_descriptions/general/base_symbol.md | 5 + .../general/base_to_address.md | 5 + .../traces/base_traces_block_no.md | 5 + .../traces/base_traces_blocktime.md | 5 + .../traces/base_traces_call_data.md | 5 + .../traces/base_traces_from.md | 5 + .../traces/base_traces_gas.md | 5 + .../traces/base_traces_gas_used.md | 5 + .../traces/base_traces_identifier.md | 5 + .../traces/base_traces_input.md | 5 + .../traces/base_traces_output.md | 5 + .../traces/base_traces_sub.md | 5 + .../traces/base_traces_table_doc.md | 5 + .../doc_descriptions/traces/base_traces_to.md | 5 + .../traces/base_traces_tx_hash.md | 5 + .../traces/base_traces_type.md | 5 + .../traces/base_traces_value.md | 5 + .../transactions/base_cumulative_gas_used.md | 5 + .../transactions/base_tx_block_hash.md | 5 + .../transactions/base_tx_fee.md | 5 + .../transactions/base_tx_gas_limit.md | 5 + .../transactions/base_tx_gas_price.md | 5 + .../transactions/base_tx_gas_used.md | 5 + .../transactions/base_tx_hash.md | 5 + .../transactions/base_tx_input_data.md | 5 + .../transactions/base_tx_json.md | 5 + .../transactions/base_tx_nonce.md | 5 + .../transactions/base_tx_origin_sig.md | 5 + .../transactions/base_tx_position.md | 5 + .../transactions/base_tx_status.md | 5 + .../transactions/base_tx_table_doc.md | 5 + .../transactions/base_value.md | 5 + .../transfers/base_eth_amount.md | 5 + .../transfers/base_eth_amount_usd.md | 6 + .../base_ez_eth_transfers_table_doc.md | 5 + .../transfers/base_ez_transfer_table_doc.md | 5 + .../transfers/base_log_id_transfers.md | 5 + .../transfers/base_origin_from.md | 5 + .../transfers/base_origin_to.md | 5 + .../transfers/base_transfer_amount.md | 5 + .../transfers/base_transfer_amount_usd.md | 5 + .../base_transfer_contract_address.md | 5 + .../transfers/base_transfer_from_address.md | 5 + .../transfers/base_transfer_has_decimal.md | 5 + .../transfers/base_transfer_has_price.md | 5 + .../transfers/base_transfer_raw_amount.md | 5 + .../transfers/base_transfer_table_doc.md | 5 + .../transfers/base_transfer_to_address.md | 5 + .../transfers/base_transfer_token_price.md | 5 + .../transfers/base_transfer_tx_hash.md | 5 + .../gold/base_goerli/goerli__fact_blocks.yml | 38 +- .../base_goerli/goerli__fact_event_logs.yml | 42 +- .../gold/base_goerli/goerli__fact_traces.yml | 38 +- .../base_goerli/goerli__fact_transactions.yml | 50 +-- models/gold/core/core__fact_blocks.sql | 68 ++++ models/gold/core/core__fact_blocks.yml | 40 ++ models/gold/core/core__fact_event_logs.sql | 22 ++ models/gold/core/core__fact_event_logs.yml | 32 ++ .../gold/core/core__fact_token_transfers.sql | 20 + .../gold/core/core__fact_token_transfers.yml | 28 ++ models/gold/core/core__fact_traces.sql | 27 ++ models/gold/core/core__fact_traces.yml | 46 +++ models/gold/core/core__fact_transactions.sql | 36 ++ models/gold/core/core__fact_transactions.yml | 60 +++ ...ver_observability__blocks_completeness.sql | 166 ++++++++ ...ilver_observability__logs_completeness.sql | 121 ++++++ ...r_observability__receipts_completeness.sql | 119 ++++++ ...ver_observability__traces_completeness.sql | 118 ++++++ ...servability__transactions_completeness.sql | 119 ++++++ models/silver/core/silver__blocks.sql | 68 ++++ .../silver/core/silver__confirmed_blocks.sql | 50 +++ models/silver/core/silver__logs.sql | 183 +++++++++ models/silver/core/silver__receipts.sql | 124 ++++++ models/silver/core/silver__traces.sql | 369 ++++++++++++++++++ models/silver/core/silver__transactions.sql | 363 +++++++++++++++++ models/silver/core/silver__transfers.sql | 61 +++ models/silver/core/silver__transfers.yml | 68 ++++ models/silver/core/silver__tx_count.sql | 23 ++ .../tests/blocks/test_silver__blocks_full.sql | 9 + .../tests/blocks/test_silver__blocks_full.yml | 95 +++++ .../blocks/test_silver__blocks_recent.sql | 27 ++ .../blocks/test_silver__blocks_recent.yml | 27 ++ .../test_silver__confirmed_blocks_full.sql | 9 + .../test_silver__confirmed_blocks_full.yml | 34 ++ .../test_silver__confirmed_blocks_recent.sql | 27 ++ .../test_silver__confirmed_blocks_recent.yml | 34 ++ .../event_logs/test_silver__logs_full.sql | 9 + .../event_logs/test_silver__logs_full.yml | 77 ++++ .../event_logs/test_silver__logs_recent.sql | 27 ++ .../event_logs/test_silver__logs_recent.yml | 33 ++ .../receipts/test_silver__receipts_full.sql | 9 + .../receipts/test_silver__receipts_full.yml | 82 ++++ .../receipts/test_silver__receipts_recent.sql | 27 ++ .../receipts/test_silver__receipts_recent.yml | 28 ++ .../tests/traces/test_silver__traces_full.sql | 9 + .../tests/traces/test_silver__traces_full.yml | 59 +++ .../traces/test_silver__traces_recent.sql | 27 ++ .../traces/test_silver__traces_recent.yml | 35 ++ .../test_silver__transactions_full.sql | 9 + .../test_silver__transactions_full.yml | 119 ++++++ .../test_silver__transactions_recent.sql | 27 ++ .../test_silver__transactions_recent.yml | 22 ++ .../methods/silver_goerli__blocks_method.sql | 0 .../methods/silver_goerli__blocks_method.yml | 0 .../silver_goerli__receipts_method.sql | 0 .../silver_goerli__receipts_method.yml | 0 .../methods/silver_goerli__traces_method.sql | 0 .../methods/silver_goerli__traces_method.yml | 0 .../methods/silver_goerli__tx_method.sql | 0 .../methods/silver_goerli__tx_method.yml | 0 .../base_goerli/silver_goerli__blocks.sql | 0 .../base_goerli/silver_goerli__blocks.yml | 0 .../base_goerli/silver_goerli__logs.sql | 0 .../base_goerli/silver_goerli__logs.yml | 0 .../base_goerli/silver_goerli__traces.sql | 0 .../base_goerli/silver_goerli__traces.yml | 0 .../silver_goerli__transactions.sql | 0 .../silver_goerli__transactions.yml | 0 .../silver__eth_goerli_block_ranges.sql | 0 .../silver__eth_goerli_contract_reads.sql | 0 .../silver__eth_goerli_contracts.sql | 0 .../eth_goerli/silver__eth_goerli_logs.sql | 0 .../eth_goerli/silver__eth_goerli_logs.yml | 0 .../silver/streamline/_max_block_by_date.sql | 27 ++ .../silver/streamline/_max_block_by_hour.sql | 37 ++ ...ine__debug_traceBlockByNumber_realtime.sql | 25 +- ...line__qn_getBlockWithReceipts_realtime.sql | 31 +- .../core/retry/_missing_receipts.sql | 32 ++ .../streamline/core/retry/_missing_traces.sql | 25 ++ .../streamline/core/retry/_missing_txs.sql | 32 ++ .../core/retry/_unconfirmed_blocks.sql | 32 ++ models/sources.yml | 5 + tests/.gitkeep | 0 tests/base/test_silver__confirmed_blocks.sql | 1 + tests/base/test_silver__missing_receipts.sql | 2 + tests/base/test_silver__missing_traces.sql | 2 + .../test_silver__recent_confirmed_blocks.sql | 1 + .../test_silver__recent_missing_receipts.sql | 2 + .../test_silver__recent_missing_traces.sql | 2 + 199 files changed, 4138 insertions(+), 192 deletions(-) create mode 100644 .github/workflows/dbt_run_full_observability.yml rename .github/workflows/{dbt_test.yml => dbt_test_daily.yml} (87%) rename .github/workflows/{dbt_run_30_mins.yml => dbt_test_intraday.yml} (68%) create mode 100644 macros/tests/missing_txs.sql create mode 100644 models/doc_descriptions/L1_fields/base_max_block.md create mode 100644 models/doc_descriptions/L1_fields/base_min_block.md create mode 100644 models/doc_descriptions/L1_fields/batch_size.md create mode 100644 models/doc_descriptions/L1_fields/l1_batch_root.md create mode 100644 models/doc_descriptions/L1_fields/l1_block_number.md create mode 100644 models/doc_descriptions/L1_fields/l1_block_timestamp.md create mode 100644 models/doc_descriptions/L1_fields/l1_fee_scalar.md create mode 100644 models/doc_descriptions/L1_fields/l1_gas_price.md create mode 100644 models/doc_descriptions/L1_fields/l1_gas_used.md create mode 100644 models/doc_descriptions/L1_fields/l1_state_root_batch_index.md create mode 100644 models/doc_descriptions/L1_fields/l1_state_root_tx_hash.md create mode 100644 models/doc_descriptions/L1_fields/l1_submission_batch_index.md create mode 100644 models/doc_descriptions/L1_fields/l1_submission_tx_hash.md create mode 100644 models/doc_descriptions/L1_fields/prev_total_elements.md create mode 100644 models/doc_descriptions/blocks/base_block_header_json.md create mode 100644 models/doc_descriptions/blocks/base_blockchain.md create mode 100644 models/doc_descriptions/blocks/base_blocks_hash.md create mode 100644 models/doc_descriptions/blocks/base_blocks_nonce.md create mode 100644 models/doc_descriptions/blocks/base_blocks_table_doc.md create mode 100644 models/doc_descriptions/blocks/base_difficulty.md create mode 100644 models/doc_descriptions/blocks/base_extra_data.md create mode 100644 models/doc_descriptions/blocks/base_gas_limit.md create mode 100644 models/doc_descriptions/blocks/base_gas_used.md create mode 100644 models/doc_descriptions/blocks/base_miner.md create mode 100644 models/doc_descriptions/blocks/base_network.md create mode 100644 models/doc_descriptions/blocks/base_parent_hash.md create mode 100644 models/doc_descriptions/blocks/base_receipts_root.md create mode 100644 models/doc_descriptions/blocks/base_sha3_uncles.md create mode 100644 models/doc_descriptions/blocks/base_size.md create mode 100644 models/doc_descriptions/blocks/base_total_difficulty.md create mode 100644 models/doc_descriptions/blocks/base_tx_count.md create mode 100644 models/doc_descriptions/blocks/base_uncle_blocks.md create mode 100644 models/doc_descriptions/event_logs/base_event_index.md create mode 100644 models/doc_descriptions/event_logs/base_event_inputs.md create mode 100644 models/doc_descriptions/event_logs/base_event_name.md create mode 100644 models/doc_descriptions/event_logs/base_event_removed.md create mode 100644 models/doc_descriptions/event_logs/base_event_sig.md create mode 100644 models/doc_descriptions/event_logs/base_log_id_events.md create mode 100644 models/doc_descriptions/event_logs/base_logs_contract_address.md create mode 100644 models/doc_descriptions/event_logs/base_logs_contract_name.md create mode 100644 models/doc_descriptions/event_logs/base_logs_data.md create mode 100644 models/doc_descriptions/event_logs/base_logs_table_doc.md create mode 100644 models/doc_descriptions/event_logs/base_logs_tx_hash.md create mode 100644 models/doc_descriptions/event_logs/base_origin_from.md create mode 100644 models/doc_descriptions/event_logs/base_origin_to.md create mode 100644 models/doc_descriptions/event_logs/base_topics.md create mode 100644 models/doc_descriptions/general/base_block_number.md create mode 100644 models/doc_descriptions/general/base_block_timestamp.md create mode 100644 models/doc_descriptions/general/base_decimals.md create mode 100644 models/doc_descriptions/general/base_from_address.md create mode 100644 models/doc_descriptions/general/base_ingested_at.md create mode 100644 models/doc_descriptions/general/base_symbol.md create mode 100644 models/doc_descriptions/general/base_to_address.md create mode 100644 models/doc_descriptions/traces/base_traces_block_no.md create mode 100644 models/doc_descriptions/traces/base_traces_blocktime.md create mode 100644 models/doc_descriptions/traces/base_traces_call_data.md create mode 100644 models/doc_descriptions/traces/base_traces_from.md create mode 100644 models/doc_descriptions/traces/base_traces_gas.md create mode 100644 models/doc_descriptions/traces/base_traces_gas_used.md create mode 100644 models/doc_descriptions/traces/base_traces_identifier.md create mode 100644 models/doc_descriptions/traces/base_traces_input.md create mode 100644 models/doc_descriptions/traces/base_traces_output.md create mode 100644 models/doc_descriptions/traces/base_traces_sub.md create mode 100644 models/doc_descriptions/traces/base_traces_table_doc.md create mode 100644 models/doc_descriptions/traces/base_traces_to.md create mode 100644 models/doc_descriptions/traces/base_traces_tx_hash.md create mode 100644 models/doc_descriptions/traces/base_traces_type.md create mode 100644 models/doc_descriptions/traces/base_traces_value.md create mode 100644 models/doc_descriptions/transactions/base_cumulative_gas_used.md create mode 100644 models/doc_descriptions/transactions/base_tx_block_hash.md create mode 100644 models/doc_descriptions/transactions/base_tx_fee.md create mode 100644 models/doc_descriptions/transactions/base_tx_gas_limit.md create mode 100644 models/doc_descriptions/transactions/base_tx_gas_price.md create mode 100644 models/doc_descriptions/transactions/base_tx_gas_used.md create mode 100644 models/doc_descriptions/transactions/base_tx_hash.md create mode 100644 models/doc_descriptions/transactions/base_tx_input_data.md create mode 100644 models/doc_descriptions/transactions/base_tx_json.md create mode 100644 models/doc_descriptions/transactions/base_tx_nonce.md create mode 100644 models/doc_descriptions/transactions/base_tx_origin_sig.md create mode 100644 models/doc_descriptions/transactions/base_tx_position.md create mode 100644 models/doc_descriptions/transactions/base_tx_status.md create mode 100644 models/doc_descriptions/transactions/base_tx_table_doc.md create mode 100644 models/doc_descriptions/transactions/base_value.md create mode 100644 models/doc_descriptions/transfers/base_eth_amount.md create mode 100644 models/doc_descriptions/transfers/base_eth_amount_usd.md create mode 100644 models/doc_descriptions/transfers/base_ez_eth_transfers_table_doc.md create mode 100644 models/doc_descriptions/transfers/base_ez_transfer_table_doc.md create mode 100644 models/doc_descriptions/transfers/base_log_id_transfers.md create mode 100644 models/doc_descriptions/transfers/base_origin_from.md create mode 100644 models/doc_descriptions/transfers/base_origin_to.md create mode 100644 models/doc_descriptions/transfers/base_transfer_amount.md create mode 100644 models/doc_descriptions/transfers/base_transfer_amount_usd.md create mode 100644 models/doc_descriptions/transfers/base_transfer_contract_address.md create mode 100644 models/doc_descriptions/transfers/base_transfer_from_address.md create mode 100644 models/doc_descriptions/transfers/base_transfer_has_decimal.md create mode 100644 models/doc_descriptions/transfers/base_transfer_has_price.md create mode 100644 models/doc_descriptions/transfers/base_transfer_raw_amount.md create mode 100644 models/doc_descriptions/transfers/base_transfer_table_doc.md create mode 100644 models/doc_descriptions/transfers/base_transfer_to_address.md create mode 100644 models/doc_descriptions/transfers/base_transfer_token_price.md create mode 100644 models/doc_descriptions/transfers/base_transfer_tx_hash.md create mode 100644 models/gold/core/core__fact_blocks.sql create mode 100644 models/gold/core/core__fact_blocks.yml create mode 100644 models/gold/core/core__fact_event_logs.sql create mode 100644 models/gold/core/core__fact_event_logs.yml create mode 100644 models/gold/core/core__fact_token_transfers.sql create mode 100644 models/gold/core/core__fact_token_transfers.yml create mode 100644 models/gold/core/core__fact_traces.sql create mode 100644 models/gold/core/core__fact_traces.yml create mode 100644 models/gold/core/core__fact_transactions.sql create mode 100644 models/gold/core/core__fact_transactions.yml create mode 100644 models/silver/_observability/silver_observability__blocks_completeness.sql create mode 100644 models/silver/_observability/silver_observability__logs_completeness.sql create mode 100644 models/silver/_observability/silver_observability__receipts_completeness.sql create mode 100644 models/silver/_observability/silver_observability__traces_completeness.sql create mode 100644 models/silver/_observability/silver_observability__transactions_completeness.sql create mode 100644 models/silver/core/silver__blocks.sql create mode 100644 models/silver/core/silver__confirmed_blocks.sql create mode 100644 models/silver/core/silver__logs.sql create mode 100644 models/silver/core/silver__receipts.sql create mode 100644 models/silver/core/silver__traces.sql create mode 100644 models/silver/core/silver__transactions.sql create mode 100644 models/silver/core/silver__transfers.sql create mode 100644 models/silver/core/silver__transfers.yml create mode 100644 models/silver/core/silver__tx_count.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_full.yml create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.sql create mode 100644 models/silver/core/tests/blocks/test_silver__blocks_recent.yml create mode 100644 models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.sql create mode 100644 models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.yml create mode 100644 models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql create mode 100644 models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.yml create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_full.sql create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_full.yml create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_recent.sql create mode 100644 models/silver/core/tests/event_logs/test_silver__logs_recent.yml create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_full.sql create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_full.yml create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_recent.sql create mode 100644 models/silver/core/tests/receipts/test_silver__receipts_recent.yml create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_full.yml create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.sql create mode 100644 models/silver/core/tests/traces/test_silver__traces_recent.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_full.yml create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.sql create mode 100644 models/silver/core/tests/transactions/test_silver__transactions_recent.yml rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__blocks_method.sql (100%) rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__blocks_method.yml (100%) rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__receipts_method.sql (100%) rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__receipts_method.yml (100%) rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__traces_method.sql (100%) rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__traces_method.yml (100%) rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__tx_method.sql (100%) rename models/silver/{ => goerli}/base_goerli/methods/silver_goerli__tx_method.yml (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__blocks.sql (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__blocks.yml (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__logs.sql (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__logs.yml (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__traces.sql (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__traces.yml (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__transactions.sql (100%) rename models/silver/{ => goerli}/base_goerli/silver_goerli__transactions.yml (100%) rename models/silver/{ => goerli}/eth_goerli/silver__eth_goerli_block_ranges.sql (100%) rename models/silver/{ => goerli}/eth_goerli/silver__eth_goerli_contract_reads.sql (100%) rename models/silver/{ => goerli}/eth_goerli/silver__eth_goerli_contracts.sql (100%) rename models/silver/{ => goerli}/eth_goerli/silver__eth_goerli_logs.sql (100%) rename models/silver/{ => goerli}/eth_goerli/silver__eth_goerli_logs.yml (100%) create mode 100644 models/silver/streamline/_max_block_by_date.sql create mode 100644 models/silver/streamline/_max_block_by_hour.sql create mode 100644 models/silver/streamline/core/retry/_missing_receipts.sql create mode 100644 models/silver/streamline/core/retry/_missing_traces.sql create mode 100644 models/silver/streamline/core/retry/_missing_txs.sql create mode 100644 models/silver/streamline/core/retry/_unconfirmed_blocks.sql create mode 100644 tests/.gitkeep create mode 100644 tests/base/test_silver__confirmed_blocks.sql create mode 100644 tests/base/test_silver__missing_receipts.sql create mode 100644 tests/base/test_silver__missing_traces.sql create mode 100644 tests/base/test_silver__recent_confirmed_blocks.sql create mode 100644 tests/base/test_silver__recent_missing_receipts.sql create mode 100644 tests/base/test_silver__recent_missing_traces.sql diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml new file mode 100644 index 0000000..ea543e1 --- /dev/null +++ b/.github/workflows/dbt_run_full_observability.yml @@ -0,0 +1,47 @@ +name: dbt_run_full_observability +run-name: dbt_run_full_observability + +on: + workflow_dispatch: + schedule: + # Runs “At 06:00 on day-of-month 1.” (see https://crontab.guru) + - cron: '0 18 1 * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_2xl + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m models/silver/_observability + + + diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 65595be..97657f7 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -4,8 +4,8 @@ run-name: dbt_run_scheduled on: workflow_dispatch: schedule: - # Runs "every 2 hours" (see https://crontab.guru) - - cron: '0 */2 * * *' + # Runs "At minute 40, every hour" (see https://crontab.guru) + - cron: '40 * * * *' env: DBT_PROFILES_DIR: ./ @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --exclude models/silver/eth_goerli models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.sql models/silver/streamline \ No newline at end of file + dbt run --exclude models/silver/goerli models/silver/streamline models/bronze/eth_goerli models/silver/_observability \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index fea58c3..f242366 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -4,8 +4,8 @@ run-name: dbt_run_streamline_realtime on: workflow_dispatch: schedule: - # Runs "at minute 5 and 35, every hour" (see https://crontab.guru) - - cron: '5,35 * * * *' + # Runs "at minute 25 and 55, every hour" (see https://crontab.guru) + - cron: '25,55 * * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test_daily.yml similarity index 87% rename from .github/workflows/dbt_test.yml rename to .github/workflows/dbt_test_daily.yml index 5db5bd4..81e559c 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -1,5 +1,5 @@ -name: dbt_test -run-name: dbt_test +name: dbt_test_daily +run-name: dbt_test_daily on: workflow_dispatch: @@ -41,7 +41,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test + dbt test --exclude tag:full_test tag:recent_test models/silver/goerli diff --git a/.github/workflows/dbt_run_30_mins.yml b/.github/workflows/dbt_test_intraday.yml similarity index 68% rename from .github/workflows/dbt_run_30_mins.yml rename to .github/workflows/dbt_test_intraday.yml index 5726fa7..5329c95 100644 --- a/.github/workflows/dbt_run_30_mins.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -1,12 +1,12 @@ -name: dbt_run_scheduled_30_mins -run-name: dbt_run_scheduled_30_mins +name: dbt_test_intraday +run-name: dbt_test_intraday on: workflow_dispatch: schedule: - # Runs "every 30 mins" (see https://crontab.guru) - - cron: '*/30 * * * *' - + # Runs “At minute 5 past every 4th hour.” (see https://crontab.guru) + - cron: '5 */4 * * *' + env: DBT_PROFILES_DIR: ./ @@ -21,11 +21,11 @@ env: concurrency: group: ${{ github.workflow }} - + jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -37,8 +37,11 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake~=${{ vars.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/eth_goerli models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.sql + dbt test -m tag:recent_test + + + diff --git a/dbt_project.yml b/dbt_project.yml index debeb29..6e94aa1 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -37,6 +37,9 @@ on-run-end: # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models +models: + +copy_grants: true + # In this example config, we tell dbt to build all models in the example/ directory # as tables. These settings can be overridden in the individual model files # using the `{{ config(...) }}` macro. @@ -46,4 +49,5 @@ vars: STREAMLINE_INVOKE_STREAMS: False STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False UPDATE_UDFS_AND_SPS: False - UPDATE_SNOWFLAKE_TAGS: True \ No newline at end of file + UPDATE_SNOWFLAKE_TAGS: True + OBSERV_FULL_TEST: False \ No newline at end of file diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql index b26d9c8..4fe0e64 100644 --- a/macros/streamline/models.sql +++ b/macros/streamline/models.sql @@ -51,7 +51,8 @@ '-32007', '-32008', '-32009', - '-32010' + '-32010', + '-32608' ) ) {% endmacro %} @@ -109,7 +110,8 @@ WHERE '-32007', '-32008', '-32009', - '-32010' + '-32010', + '-32608' ) ) {% endmacro %} diff --git a/macros/tests/missing_txs.sql b/macros/tests/missing_txs.sql new file mode 100644 index 0000000..293fb1b --- /dev/null +++ b/macros/tests/missing_txs.sql @@ -0,0 +1,103 @@ +{% macro missing_txs( + model + ) %} + WITH txs_base AS ( + SELECT + block_number AS base_block_number, + tx_hash AS base_tx_hash + FROM + {{ ref('test_silver__transactions_full') }} + ), + model_name AS ( + SELECT + block_number AS model_block_number, + tx_hash AS model_tx_hash + FROM + {{ model }} + ) +SELECT + base_block_number, + base_tx_hash, + model_block_number, + model_tx_hash +FROM + txs_base + LEFT JOIN model_name + ON base_block_number = model_block_number + AND base_tx_hash = model_tx_hash +WHERE + ( + model_tx_hash IS NULL + OR model_block_number IS NULL + ) +{% endmacro %} + +{% macro recent_missing_txs( + model + ) %} + WITH txs_base AS ( + SELECT + block_number AS base_block_number, + tx_hash AS base_tx_hash + FROM + {{ ref('test_silver__transactions_recent') }} + ), + model_name AS ( + SELECT + block_number AS model_block_number, + tx_hash AS model_tx_hash + FROM + {{ model }} + ) +SELECT + base_block_number, + base_tx_hash, + model_block_number, + model_tx_hash +FROM + txs_base + LEFT JOIN model_name + ON base_block_number = model_block_number + AND base_tx_hash = model_tx_hash +WHERE + model_tx_hash IS NULL + OR model_block_number IS NULL +{% endmacro %} + +{% macro missing_confirmed_txs( + model1, + model2 + ) %} + WITH txs_base AS ( + SELECT + block_number AS base_block_number, + block_hash AS base_block_hash, + tx_hash AS base_tx_hash + FROM + {{ model1 }} + ), + model_name AS ( + SELECT + block_number AS model_block_number, + block_hash AS model_block_hash, + tx_hash AS model_tx_hash + FROM + {{ model2 }} + ) +SELECT + DISTINCT base_block_number AS block_number +FROM + txs_base + LEFT JOIN model_name + ON base_block_number = model_block_number + AND base_tx_hash = model_tx_hash + AND base_block_hash = model_block_hash +WHERE + model_tx_hash IS NULL + AND model_block_number <= ( + SELECT + MAX(base_block_number) + FROM + txs_base + ) +{% endmacro %} diff --git a/models/doc_descriptions/L1_fields/base_max_block.md b/models/doc_descriptions/L1_fields/base_max_block.md new file mode 100644 index 0000000..789d7df --- /dev/null +++ b/models/doc_descriptions/L1_fields/base_max_block.md @@ -0,0 +1,5 @@ +{% docs base_max_block %} + +The max block on Base this batch relates to. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/base_min_block.md b/models/doc_descriptions/L1_fields/base_min_block.md new file mode 100644 index 0000000..4a7a2db --- /dev/null +++ b/models/doc_descriptions/L1_fields/base_min_block.md @@ -0,0 +1,5 @@ +{% docs base_min_block %} + +The min block on Base this batch relates to. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/batch_size.md b/models/doc_descriptions/L1_fields/batch_size.md new file mode 100644 index 0000000..e4f1fb2 --- /dev/null +++ b/models/doc_descriptions/L1_fields/batch_size.md @@ -0,0 +1,5 @@ +{% docs base_batch_size %} + +Total Base Txs included within batch. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_batch_root.md b/models/doc_descriptions/L1_fields/l1_batch_root.md new file mode 100644 index 0000000..e6706e0 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_batch_root.md @@ -0,0 +1,5 @@ +{% docs base_batch_root %} + +Root of batch, either for sumbission or state. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_block_number.md b/models/doc_descriptions/L1_fields/l1_block_number.md new file mode 100644 index 0000000..32037af --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_block_number.md @@ -0,0 +1,5 @@ +{% docs base_l1_block_no %} + +The Ethereum block number that contained the batch. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_block_timestamp.md b/models/doc_descriptions/L1_fields/l1_block_timestamp.md new file mode 100644 index 0000000..76b5a69 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_block_timestamp.md @@ -0,0 +1,5 @@ +{% docs base_l1_block_time %} + +The timestamp of the Ethereum block that contained this batch. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_fee_scalar.md b/models/doc_descriptions/L1_fields/l1_fee_scalar.md new file mode 100644 index 0000000..71e88a1 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_fee_scalar.md @@ -0,0 +1,5 @@ +{% docs base_l1_fee_scalar %} + +This value covers the change in L1 gas price between the time the transaction is submitted and when it is published. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_gas_price.md b/models/doc_descriptions/L1_fields/l1_gas_price.md new file mode 100644 index 0000000..bb66684 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_gas_price.md @@ -0,0 +1,5 @@ +{% docs base_l1_gas_price %} + +The gas price for L1 transactions when the transaction was processed. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_gas_used.md b/models/doc_descriptions/L1_fields/l1_gas_used.md new file mode 100644 index 0000000..8f00a30 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_gas_used.md @@ -0,0 +1,5 @@ +{% docs base_l1_gas_used %} + +The gas used on L1 to publish the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_state_root_batch_index.md b/models/doc_descriptions/L1_fields/l1_state_root_batch_index.md new file mode 100644 index 0000000..f953d22 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_state_root_batch_index.md @@ -0,0 +1,5 @@ +{% docs base_l1_state_batch %} + +The batch index of when this block was included in the Ethereum state root. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_state_root_tx_hash.md b/models/doc_descriptions/L1_fields/l1_state_root_tx_hash.md new file mode 100644 index 0000000..8a58871 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_state_root_tx_hash.md @@ -0,0 +1,5 @@ +{% docs base_l1_state_tx_hash %} + +The L1 tx hash of when this block was included in the Ethereum state root. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_submission_batch_index.md b/models/doc_descriptions/L1_fields/l1_submission_batch_index.md new file mode 100644 index 0000000..93e4697 --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_submission_batch_index.md @@ -0,0 +1,5 @@ +{% docs base_l1_sub_batch %} + +The batch index of when this block was submitted to L1. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/l1_submission_tx_hash.md b/models/doc_descriptions/L1_fields/l1_submission_tx_hash.md new file mode 100644 index 0000000..41fd53e --- /dev/null +++ b/models/doc_descriptions/L1_fields/l1_submission_tx_hash.md @@ -0,0 +1,5 @@ +{% docs base_l1_sub_tx_hash %} + +The L1 tx hash of when this block was submitted to L1. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/L1_fields/prev_total_elements.md b/models/doc_descriptions/L1_fields/prev_total_elements.md new file mode 100644 index 0000000..a48930f --- /dev/null +++ b/models/doc_descriptions/L1_fields/prev_total_elements.md @@ -0,0 +1,5 @@ +{% docs base_prev_total_elements %} + +Confirmed blocks prior to this batch. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_block_header_json.md b/models/doc_descriptions/blocks/base_block_header_json.md new file mode 100644 index 0000000..ab710e5 --- /dev/null +++ b/models/doc_descriptions/blocks/base_block_header_json.md @@ -0,0 +1,5 @@ +{% docs base_block_header_json %} + +This JSON column contains the block header details. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_blockchain.md b/models/doc_descriptions/blocks/base_blockchain.md new file mode 100644 index 0000000..6ee077a --- /dev/null +++ b/models/doc_descriptions/blocks/base_blockchain.md @@ -0,0 +1,5 @@ +{% docs base_blockchain %} + +The blockchain on which transactions are being confirmed. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_blocks_hash.md b/models/doc_descriptions/blocks/base_blocks_hash.md new file mode 100644 index 0000000..214321c --- /dev/null +++ b/models/doc_descriptions/blocks/base_blocks_hash.md @@ -0,0 +1,5 @@ +{% docs base_blocks_hash %} + +The hash of the block header for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_blocks_nonce.md b/models/doc_descriptions/blocks/base_blocks_nonce.md new file mode 100644 index 0000000..f8d4a9b --- /dev/null +++ b/models/doc_descriptions/blocks/base_blocks_nonce.md @@ -0,0 +1,5 @@ +{% docs base_blocks_nonce %} + +Block nonce is a value used during mining to demonstrate proof of work for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_blocks_table_doc.md b/models/doc_descriptions/blocks/base_blocks_table_doc.md new file mode 100644 index 0000000..8a21771 --- /dev/null +++ b/models/doc_descriptions/blocks/base_blocks_table_doc.md @@ -0,0 +1,5 @@ +{% docs base_blocks_table_doc %} + +This table contains block level data for the Base Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information on EVM transactions, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/) + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_difficulty.md b/models/doc_descriptions/blocks/base_difficulty.md new file mode 100644 index 0000000..144e6f5 --- /dev/null +++ b/models/doc_descriptions/blocks/base_difficulty.md @@ -0,0 +1,5 @@ +{% docs base_difficulty %} + +The effort required to mine the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_extra_data.md b/models/doc_descriptions/blocks/base_extra_data.md new file mode 100644 index 0000000..aa507c5 --- /dev/null +++ b/models/doc_descriptions/blocks/base_extra_data.md @@ -0,0 +1,5 @@ +{% docs base_extra_data %} + +Any data included by the validator for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_gas_limit.md b/models/doc_descriptions/blocks/base_gas_limit.md new file mode 100644 index 0000000..4b6d763 --- /dev/null +++ b/models/doc_descriptions/blocks/base_gas_limit.md @@ -0,0 +1,5 @@ +{% docs base_gas_limit %} + +Total gas limit provided by all transactions in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_gas_used.md b/models/doc_descriptions/blocks/base_gas_used.md new file mode 100644 index 0000000..be34b92 --- /dev/null +++ b/models/doc_descriptions/blocks/base_gas_used.md @@ -0,0 +1,5 @@ +{% docs base_gas_used %} + +Total gas used in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_miner.md b/models/doc_descriptions/blocks/base_miner.md new file mode 100644 index 0000000..95e3cc8 --- /dev/null +++ b/models/doc_descriptions/blocks/base_miner.md @@ -0,0 +1,5 @@ +{% docs base_miner %} + +Miner who successfully added a given block to the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_network.md b/models/doc_descriptions/blocks/base_network.md new file mode 100644 index 0000000..ef05423 --- /dev/null +++ b/models/doc_descriptions/blocks/base_network.md @@ -0,0 +1,5 @@ +{% docs base_network %} + +The network on the blockchain used by a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_parent_hash.md b/models/doc_descriptions/blocks/base_parent_hash.md new file mode 100644 index 0000000..beb821f --- /dev/null +++ b/models/doc_descriptions/blocks/base_parent_hash.md @@ -0,0 +1,5 @@ +{% docs base_parent_hash %} + +The hash of the block from which a given block is generated. Also known as the parent block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_receipts_root.md b/models/doc_descriptions/blocks/base_receipts_root.md new file mode 100644 index 0000000..fc69c1a --- /dev/null +++ b/models/doc_descriptions/blocks/base_receipts_root.md @@ -0,0 +1,5 @@ +{% docs base_receipts_root %} + +The root of the state trie. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_sha3_uncles.md b/models/doc_descriptions/blocks/base_sha3_uncles.md new file mode 100644 index 0000000..3a1a39a --- /dev/null +++ b/models/doc_descriptions/blocks/base_sha3_uncles.md @@ -0,0 +1,5 @@ +{% docs base_sha3_uncles %} + +The mechanism which Ethereum Javascript RLP encodes an empty string. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_size.md b/models/doc_descriptions/blocks/base_size.md new file mode 100644 index 0000000..408d974 --- /dev/null +++ b/models/doc_descriptions/blocks/base_size.md @@ -0,0 +1,5 @@ +{% docs base_size %} + +Block size, which is determined by a given block's gas limit. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_total_difficulty.md b/models/doc_descriptions/blocks/base_total_difficulty.md new file mode 100644 index 0000000..097da9e --- /dev/null +++ b/models/doc_descriptions/blocks/base_total_difficulty.md @@ -0,0 +1,5 @@ +{% docs base_total_difficulty %} + +Total difficulty of the chain at a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_tx_count.md b/models/doc_descriptions/blocks/base_tx_count.md new file mode 100644 index 0000000..85722d7 --- /dev/null +++ b/models/doc_descriptions/blocks/base_tx_count.md @@ -0,0 +1,5 @@ +{% docs base_tx_count %} + +Total number of transactions within a block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/base_uncle_blocks.md b/models/doc_descriptions/blocks/base_uncle_blocks.md new file mode 100644 index 0000000..98af016 --- /dev/null +++ b/models/doc_descriptions/blocks/base_uncle_blocks.md @@ -0,0 +1,5 @@ +{% docs base_uncle_blocks %} + +Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_event_index.md b/models/doc_descriptions/event_logs/base_event_index.md new file mode 100644 index 0000000..4ec95f8 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_event_index.md @@ -0,0 +1,5 @@ +{% docs base_event_index %} + +Event number within a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_event_inputs.md b/models/doc_descriptions/event_logs/base_event_inputs.md new file mode 100644 index 0000000..2fd3283 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_event_inputs.md @@ -0,0 +1,5 @@ +{% docs base_event_inputs %} + +The decoded event inputs for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_event_name.md b/models/doc_descriptions/event_logs/base_event_name.md new file mode 100644 index 0000000..83786a6 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_event_name.md @@ -0,0 +1,5 @@ +{% docs base_event_name %} + +The decoded event name for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_event_removed.md b/models/doc_descriptions/event_logs/base_event_removed.md new file mode 100644 index 0000000..bc9bd30 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_event_removed.md @@ -0,0 +1,5 @@ +{% docs base_event_removed %} + +Whether the event has been removed from the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_event_sig.md b/models/doc_descriptions/event_logs/base_event_sig.md new file mode 100644 index 0000000..b6972eb --- /dev/null +++ b/models/doc_descriptions/event_logs/base_event_sig.md @@ -0,0 +1,5 @@ +{% docs base_origin_sig %} + +The function signature of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_log_id_events.md b/models/doc_descriptions/event_logs/base_log_id_events.md new file mode 100644 index 0000000..9c57b48 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_log_id_events.md @@ -0,0 +1,5 @@ +{% docs base_log_id_events %} + +This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_logs_contract_address.md b/models/doc_descriptions/event_logs/base_logs_contract_address.md new file mode 100644 index 0000000..c2643f4 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_logs_contract_address.md @@ -0,0 +1,5 @@ +{% docs base_logs_contract_address %} + +The address interacted with for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_logs_contract_name.md b/models/doc_descriptions/event_logs/base_logs_contract_name.md new file mode 100644 index 0000000..798e1ae --- /dev/null +++ b/models/doc_descriptions/event_logs/base_logs_contract_name.md @@ -0,0 +1,5 @@ +{% docs base_logs_contract_name %} + +The name of the contract or token, where possible. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_logs_data.md b/models/doc_descriptions/event_logs/base_logs_data.md new file mode 100644 index 0000000..8777041 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_logs_data.md @@ -0,0 +1,5 @@ +{% docs base_logs_data %} + +The un-decoded event data. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_logs_table_doc.md b/models/doc_descriptions/event_logs/base_logs_table_doc.md new file mode 100644 index 0000000..c0f1d57 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_logs_table_doc.md @@ -0,0 +1,5 @@ +{% docs base_logs_table_doc %} + +This table contains flattened event logs from transactions on the Base Blockchain. Transactions may have multiple events, which are denoted by the event index for a transaction hash. Therefore, this table is unique on the combination of transaction hash and event index. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_logs_tx_hash.md b/models/doc_descriptions/event_logs/base_logs_tx_hash.md new file mode 100644 index 0000000..3dfe3d6 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_logs_tx_hash.md @@ -0,0 +1,5 @@ +{% docs base_logs_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This field will not be unique in this table, as a given transaction can include multiple events. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_origin_from.md b/models/doc_descriptions/event_logs/base_origin_from.md new file mode 100644 index 0000000..7c0db2b --- /dev/null +++ b/models/doc_descriptions/event_logs/base_origin_from.md @@ -0,0 +1,5 @@ +{% docs base_origin_from %} + +The from address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_origin_to.md b/models/doc_descriptions/event_logs/base_origin_to.md new file mode 100644 index 0000000..086b3fd --- /dev/null +++ b/models/doc_descriptions/event_logs/base_origin_to.md @@ -0,0 +1,5 @@ +{% docs base_origin_to %} + +The to address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/base_topics.md b/models/doc_descriptions/event_logs/base_topics.md new file mode 100644 index 0000000..0878f89 --- /dev/null +++ b/models/doc_descriptions/event_logs/base_topics.md @@ -0,0 +1,5 @@ +{% docs base_topics %} + +The un-decoded event input topics. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/__overview__.md b/models/doc_descriptions/general/__overview__.md index 2becc09..954a37f 100644 --- a/models/doc_descriptions/general/__overview__.md +++ b/models/doc_descriptions/general/__overview__.md @@ -19,10 +19,11 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Tables **Fact Tables:** -- [goerli.fact_blocks](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_blocks) -- [goerli.fact_transactions](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_transactions) -- [goerli.fact_event_logs](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_event_logs) -- [goerli.fact_traces](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_traces) +- [fact_blocks](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_blocks) +- [fact_transactions](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_transactions) +- [fact_event_logs](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_event_logs) +- [fact_traces](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_traces) +- [fact_token_transfers](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_token_transfers) ## **Helpful User-Defined Functions (UDFs)** diff --git a/models/doc_descriptions/general/base_block_number.md b/models/doc_descriptions/general/base_block_number.md new file mode 100644 index 0000000..58a21af --- /dev/null +++ b/models/doc_descriptions/general/base_block_number.md @@ -0,0 +1,5 @@ +{% docs base_block_number %} + +Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. + +{% enddocs %} diff --git a/models/doc_descriptions/general/base_block_timestamp.md b/models/doc_descriptions/general/base_block_timestamp.md new file mode 100644 index 0000000..142ca21 --- /dev/null +++ b/models/doc_descriptions/general/base_block_timestamp.md @@ -0,0 +1,5 @@ +{% docs base_block_timestamp %} + +The date and time at which the block was produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/base_decimals.md b/models/doc_descriptions/general/base_decimals.md new file mode 100644 index 0000000..41abd5d --- /dev/null +++ b/models/doc_descriptions/general/base_decimals.md @@ -0,0 +1,5 @@ +{% docs base_decimals %} + +The number of decimal places this contract needs adjusted where token values exist. For example, use the decimal field to correctly transform raw amounts in ```fact_token_transfers```. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/base_from_address.md b/models/doc_descriptions/general/base_from_address.md new file mode 100644 index 0000000..bd291d6 --- /dev/null +++ b/models/doc_descriptions/general/base_from_address.md @@ -0,0 +1,5 @@ +{% docs base_from_address %} + +The sending address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/base_ingested_at.md b/models/doc_descriptions/general/base_ingested_at.md new file mode 100644 index 0000000..c1c0671 --- /dev/null +++ b/models/doc_descriptions/general/base_ingested_at.md @@ -0,0 +1,5 @@ +{% docs base_ingested_at %} + +Internal column. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/base_symbol.md b/models/doc_descriptions/general/base_symbol.md new file mode 100644 index 0000000..4731365 --- /dev/null +++ b/models/doc_descriptions/general/base_symbol.md @@ -0,0 +1,5 @@ +{% docs base_symbol %} + +The symbol belonging to the address of the token + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/base_to_address.md b/models/doc_descriptions/general/base_to_address.md new file mode 100644 index 0000000..4366200 --- /dev/null +++ b/models/doc_descriptions/general/base_to_address.md @@ -0,0 +1,5 @@ +{% docs base_to_address %} + +The receiving address of this transaction. This can be a contract address. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_block_no.md b/models/doc_descriptions/traces/base_traces_block_no.md new file mode 100644 index 0000000..35b8712 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_block_no.md @@ -0,0 +1,5 @@ +{% docs base_traces_block_no %} + +The block number of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_blocktime.md b/models/doc_descriptions/traces/base_traces_blocktime.md new file mode 100644 index 0000000..b90df0a --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_blocktime.md @@ -0,0 +1,5 @@ +{% docs base_traces_blocktime %} + +The block timestamp of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_call_data.md b/models/doc_descriptions/traces/base_traces_call_data.md new file mode 100644 index 0000000..491550f --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_call_data.md @@ -0,0 +1,5 @@ +{% docs base_traces_call_data %} + +The raw JSON data for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_from.md b/models/doc_descriptions/traces/base_traces_from.md new file mode 100644 index 0000000..3f48bd9 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_from.md @@ -0,0 +1,5 @@ +{% docs base_traces_from %} + +The sending address of this trace. This is not necessarily the from address of the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_gas.md b/models/doc_descriptions/traces/base_traces_gas.md new file mode 100644 index 0000000..69cda46 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_gas.md @@ -0,0 +1,5 @@ +{% docs base_traces_gas %} + +The gas supplied for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_gas_used.md b/models/doc_descriptions/traces/base_traces_gas_used.md new file mode 100644 index 0000000..3c5fc91 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_gas_used.md @@ -0,0 +1,5 @@ +{% docs base_traces_gas_used %} + +The gas used for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_identifier.md b/models/doc_descriptions/traces/base_traces_identifier.md new file mode 100644 index 0000000..d2c8497 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_identifier.md @@ -0,0 +1,5 @@ +{% docs base_traces_identifier %} + +This field represents the position and type of the trace within the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_input.md b/models/doc_descriptions/traces/base_traces_input.md new file mode 100644 index 0000000..51c0797 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_input.md @@ -0,0 +1,5 @@ +{% docs base_traces_input %} + +The input data for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_output.md b/models/doc_descriptions/traces/base_traces_output.md new file mode 100644 index 0000000..8d84616 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_output.md @@ -0,0 +1,5 @@ +{% docs base_traces_output %} + +The output data for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_sub.md b/models/doc_descriptions/traces/base_traces_sub.md new file mode 100644 index 0000000..b1318f7 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_sub.md @@ -0,0 +1,5 @@ +{% docs base_traces_sub %} + +The amount of nested sub traces for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_table_doc.md b/models/doc_descriptions/traces/base_traces_table_doc.md new file mode 100644 index 0000000..99300e2 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_table_doc.md @@ -0,0 +1,5 @@ +{% docs base_traces_table_doc %} + +This table contains flattened trace data for internal contract calls on the Base Blockchain. Hex encoded fields can be decoded to integers by using `utils.udf_hex_to_int()`. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_to.md b/models/doc_descriptions/traces/base_traces_to.md new file mode 100644 index 0000000..4567d69 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_to.md @@ -0,0 +1,5 @@ +{% docs base_traces_to %} + +The receiving address of this trace. This is not necessarily the to address of the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_tx_hash.md b/models/doc_descriptions/traces/base_traces_tx_hash.md new file mode 100644 index 0000000..649b1ab --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_tx_hash.md @@ -0,0 +1,5 @@ +{% docs base_traces_tx_hash %} + +The transaction hash for the trace. Please note, this is not necessarily unique in this table as transactions frequently have multiple traces. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_type.md b/models/doc_descriptions/traces/base_traces_type.md new file mode 100644 index 0000000..c439753 --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_type.md @@ -0,0 +1,5 @@ +{% docs base_traces_type %} + +The type of internal transaction. Common trace types are `CALL`, `DELEGATECALL`, and `STATICCALL`. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/base_traces_value.md b/models/doc_descriptions/traces/base_traces_value.md new file mode 100644 index 0000000..eed806c --- /dev/null +++ b/models/doc_descriptions/traces/base_traces_value.md @@ -0,0 +1,5 @@ +{% docs base_traces_value %} + +The amount of ETH transferred in this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_cumulative_gas_used.md b/models/doc_descriptions/transactions/base_cumulative_gas_used.md new file mode 100644 index 0000000..c719f99 --- /dev/null +++ b/models/doc_descriptions/transactions/base_cumulative_gas_used.md @@ -0,0 +1,5 @@ +{% docs base_cumulative_gas_used %} + +The total amount of gas used when this transaction was executed in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_block_hash.md b/models/doc_descriptions/transactions/base_tx_block_hash.md new file mode 100644 index 0000000..a8bbea8 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_block_hash.md @@ -0,0 +1,5 @@ +{% docs base_tx_block_hash %} + +Block hash is a unique 66-character identifier that is generate when a block is produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_fee.md b/models/doc_descriptions/transactions/base_tx_fee.md new file mode 100644 index 0000000..88e1fe2 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_fee.md @@ -0,0 +1,5 @@ +{% docs base_tx_fee %} + +Amount paid to validate the transaction in ETH. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_gas_limit.md b/models/doc_descriptions/transactions/base_tx_gas_limit.md new file mode 100644 index 0000000..d91e1cb --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_gas_limit.md @@ -0,0 +1,5 @@ +{% docs base_tx_gas_limit %} + +Maximum amount of gas allocated for the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_gas_price.md b/models/doc_descriptions/transactions/base_tx_gas_price.md new file mode 100644 index 0000000..ad209b6 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_gas_price.md @@ -0,0 +1,5 @@ +{% docs base_tx_gas_price %} + +Cost per unit of gas in Gwei. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_gas_used.md b/models/doc_descriptions/transactions/base_tx_gas_used.md new file mode 100644 index 0000000..cccfc0e --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_gas_used.md @@ -0,0 +1,5 @@ +{% docs base_tx_gas_used %} + +Gas used by transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_hash.md b/models/doc_descriptions/transactions/base_tx_hash.md new file mode 100644 index 0000000..b573293 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_hash.md @@ -0,0 +1,5 @@ +{% docs base_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_input_data.md b/models/doc_descriptions/transactions/base_tx_input_data.md new file mode 100644 index 0000000..adfe82b --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_input_data.md @@ -0,0 +1,5 @@ +{% docs base_tx_input_data %} + +This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_json.md b/models/doc_descriptions/transactions/base_tx_json.md new file mode 100644 index 0000000..6b00605 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_json.md @@ -0,0 +1,5 @@ +{% docs base_tx_json %} + +This JSON column contains the transaction details, including event logs. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_nonce.md b/models/doc_descriptions/transactions/base_tx_nonce.md new file mode 100644 index 0000000..4e3d7a5 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_nonce.md @@ -0,0 +1,5 @@ +{% docs base_tx_nonce %} + +The number of transactions sent from a given address. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_origin_sig.md b/models/doc_descriptions/transactions/base_tx_origin_sig.md new file mode 100644 index 0000000..a88c6e0 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_origin_sig.md @@ -0,0 +1,5 @@ +{% docs base_tx_origin_sig %} + +The function signature of the contract call. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_position.md b/models/doc_descriptions/transactions/base_tx_position.md new file mode 100644 index 0000000..1495749 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_position.md @@ -0,0 +1,5 @@ +{% docs base_tx_position %} + +The position of the transaction within the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_status.md b/models/doc_descriptions/transactions/base_tx_status.md new file mode 100644 index 0000000..cd28733 --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_status.md @@ -0,0 +1,5 @@ +{% docs base_tx_status %} + +Status of the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_tx_table_doc.md b/models/doc_descriptions/transactions/base_tx_table_doc.md new file mode 100644 index 0000000..23a88cb --- /dev/null +++ b/models/doc_descriptions/transactions/base_tx_table_doc.md @@ -0,0 +1,5 @@ +{% docs base_tx_table_doc %} + +This table contains transaction level data for the Base Blockchain. Each transaction will have a unique transaction hash, along with transactions fees and a ETH value transferred when applicable. Transactions may be native ETH transfers or interactions with contract addresses. For more information, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/) + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/base_value.md b/models/doc_descriptions/transactions/base_value.md new file mode 100644 index 0000000..c798b00 --- /dev/null +++ b/models/doc_descriptions/transactions/base_value.md @@ -0,0 +1,5 @@ +{% docs base_value %} + +The value transacted in ETH. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_eth_amount.md b/models/doc_descriptions/transfers/base_eth_amount.md new file mode 100644 index 0000000..2e2c55e --- /dev/null +++ b/models/doc_descriptions/transfers/base_eth_amount.md @@ -0,0 +1,5 @@ +{% docs base_eth_amount %} + +ETH value transferred. + +{% enddocs %} diff --git a/models/doc_descriptions/transfers/base_eth_amount_usd.md b/models/doc_descriptions/transfers/base_eth_amount_usd.md new file mode 100644 index 0000000..c9e7f8c --- /dev/null +++ b/models/doc_descriptions/transfers/base_eth_amount_usd.md @@ -0,0 +1,6 @@ + +{% docs base_eth_amount_usd %} + +ETH value transferred, in USD. + +{% enddocs %} diff --git a/models/doc_descriptions/transfers/base_ez_eth_transfers_table_doc.md b/models/doc_descriptions/transfers/base_ez_eth_transfers_table_doc.md new file mode 100644 index 0000000..90dceba --- /dev/null +++ b/models/doc_descriptions/transfers/base_ez_eth_transfers_table_doc.md @@ -0,0 +1,5 @@ +{% docs base_ez_eth_transfers_table_doc %} + +This table contains all native ETH transfers, including equivalent USD amounts. The origin addresses correspond to the to and from addresses from the `fact_transactions` table. The `identifier` and `tx_hash` columns relate this table back to `fact_traces`, which contains more details on the transfers. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_ez_transfer_table_doc.md b/models/doc_descriptions/transfers/base_ez_transfer_table_doc.md new file mode 100644 index 0000000..3f3862f --- /dev/null +++ b/models/doc_descriptions/transfers/base_ez_transfer_table_doc.md @@ -0,0 +1,5 @@ +{% docs base_ez_transfers_table_doc %} + +This table will contain all events in the ```fact_token_transfers table```, along with joined columns such as token price, symbol, and decimals where possible that allow for easier analysis of token transfer events. Please note Native ETH transfers are not included here. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_log_id_transfers.md b/models/doc_descriptions/transfers/base_log_id_transfers.md new file mode 100644 index 0000000..4c2922c --- /dev/null +++ b/models/doc_descriptions/transfers/base_log_id_transfers.md @@ -0,0 +1,5 @@ +{% docs base_log_id_transfers %} + +This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the transfer event occurred. This field can be used to find more details on the event within the ```fact_event_logs``` table. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_origin_from.md b/models/doc_descriptions/transfers/base_origin_from.md new file mode 100644 index 0000000..dad97a6 --- /dev/null +++ b/models/doc_descriptions/transfers/base_origin_from.md @@ -0,0 +1,5 @@ +{% docs base_eth_origin_from %} + +The from address at the transaction level. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_origin_to.md b/models/doc_descriptions/transfers/base_origin_to.md new file mode 100644 index 0000000..19fedaa --- /dev/null +++ b/models/doc_descriptions/transfers/base_origin_to.md @@ -0,0 +1,5 @@ +{% docs base_eth_origin_to %} + +The to address at the transaction level. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_amount.md b/models/doc_descriptions/transfers/base_transfer_amount.md new file mode 100644 index 0000000..7a78aa3 --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_amount.md @@ -0,0 +1,5 @@ +{% docs base_transfer_amount %} + +The decimal transformed amount for this token. Tokens without a decimal adjustment will be nulled out here. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_amount_usd.md b/models/doc_descriptions/transfers/base_transfer_amount_usd.md new file mode 100644 index 0000000..6d76cdd --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_amount_usd.md @@ -0,0 +1,5 @@ +{% docs base_transfer_amount_usd %} + +The amount in US dollars for this transfer at the time of the transfer. Tokens without a decimal adjustment or price will be nulled out here. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_contract_address.md b/models/doc_descriptions/transfers/base_transfer_contract_address.md new file mode 100644 index 0000000..5db6b70 --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_contract_address.md @@ -0,0 +1,5 @@ +{% docs base_transfer_contract_address %} + +Contract address of the token being transferred. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_from_address.md b/models/doc_descriptions/transfers/base_transfer_from_address.md new file mode 100644 index 0000000..25f5192 --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_from_address.md @@ -0,0 +1,5 @@ +{% docs base_transfer_from_address %} + +The sending address of this transfer. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_has_decimal.md b/models/doc_descriptions/transfers/base_transfer_has_decimal.md new file mode 100644 index 0000000..54574a6 --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_has_decimal.md @@ -0,0 +1,5 @@ +{% docs base_transfer_has_decimal %} + +Whether or not our contracts model contains the necessary decimal adjustment for this token. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_has_price.md b/models/doc_descriptions/transfers/base_transfer_has_price.md new file mode 100644 index 0000000..9b8d9e4 --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_has_price.md @@ -0,0 +1,5 @@ +{% docs base_transfer_has_price %} + +Whether or not our prices model contains this hourly token price. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_raw_amount.md b/models/doc_descriptions/transfers/base_transfer_raw_amount.md new file mode 100644 index 0000000..b01ecb2 --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_raw_amount.md @@ -0,0 +1,5 @@ +{% docs base_transfer_raw_amount %} + +The amount of tokens transferred. This value is not decimal adjusted. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_table_doc.md b/models/doc_descriptions/transfers/base_transfer_table_doc.md new file mode 100644 index 0000000..de4ccbc --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_table_doc.md @@ -0,0 +1,5 @@ +{% docs base_transfer_table_doc %} + +This table contains events on the Base Blockchain with an event name of 'Transfer'. The contract address is the token transferred, and the raw amount field is the amount of tokens transferred. Please note this amount is not decimal adjusted. This table will not contain transfers of native ETH. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_to_address.md b/models/doc_descriptions/transfers/base_transfer_to_address.md new file mode 100644 index 0000000..b9b954c --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_to_address.md @@ -0,0 +1,5 @@ +{% docs base_transfer_to_address %} + +The receiving address of this transfer. This can be a contract address. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_token_price.md b/models/doc_descriptions/transfers/base_transfer_token_price.md new file mode 100644 index 0000000..f988a4d --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_token_price.md @@ -0,0 +1,5 @@ +{% docs base_transfer_token_price %} + +The price, if available, for this token at the transfer time. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transfers/base_transfer_tx_hash.md b/models/doc_descriptions/transfers/base_transfer_tx_hash.md new file mode 100644 index 0000000..3db0983 --- /dev/null +++ b/models/doc_descriptions/transfers/base_transfer_tx_hash.md @@ -0,0 +1,5 @@ +{% docs base_transfer_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This will not be unique in this table as a transaction could include multiple transfer events. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/base_goerli/goerli__fact_blocks.yml b/models/gold/base_goerli/goerli__fact_blocks.yml index d312dd6..c968e5d 100644 --- a/models/gold/base_goerli/goerli__fact_blocks.yml +++ b/models/gold/base_goerli/goerli__fact_blocks.yml @@ -1,40 +1,4 @@ version: 2 models: - name: goerli__fact_blocks - description: This table contains block level data for the Base Blockchain, Goerli Testnet. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information on Base, please see [Base Documentation](https://docs.base.org/). For more information on EVM transactions, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/). - - columns: - - name: BLOCK_NUMBER - description: Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. - - name: BLOCK_TIMESTAMP - description: The date and time at which the block was produced. - - name: NETWORK - description: The network of the blockchain used by a transaction. - - name: BLOCKCHAIN - description: The blockchain on which transactions are being confirmed. - - name: TX_COUNT - description: Total number of transactions within a block. - - name: DIFFICULTY - description: The effort required to mine the block. - - name: TOTAL_DIFFICULTY - description: Total difficulty of the chain at a given block. - - name: EXTRA_DATA - description: Any data included by the validator for a given block. - - name: GAS_LIMIT - description: Total gas limit provided by all transactions in the block. - - name: GAS_USED - description: Total gas used in the block. - - name: HASH - description: The hash of the block header for a given block. - - name: PARENT_HASH - description: The hash of the block from which a given block is generated. Also known as the parent block. - - name: RECEIPTS_ROOT - description: The root of the state trie. - - name: SHA3_UNCLES - description: The mechanism which Ethereum Javascript RLP encodes an empty string. - - name: SIZE - description: Block size, which is determined by a given block's gas limit. - - name: UNCLE_BLOCKS - description: Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks. - - name: BLOCK_HEADER_JSON - description: This JSON column contains the block header details. \ No newline at end of file + description: Deprecating soon. Please migrate your queries to mainnet. \ No newline at end of file diff --git a/models/gold/base_goerli/goerli__fact_event_logs.yml b/models/gold/base_goerli/goerli__fact_event_logs.yml index 17329e8..2c80bd6 100644 --- a/models/gold/base_goerli/goerli__fact_event_logs.yml +++ b/models/gold/base_goerli/goerli__fact_event_logs.yml @@ -1,44 +1,4 @@ version: 2 models: - name: goerli__fact_event_logs - description: This table contains flattened event logs from transactions on the Base Blockchain, Goerli Testnet. Transactions may have multiple events, which are denoted by the event index for a transaction hash. Therefore, this table is unique on the combination of transaction hash and event index. For more information on Base, please see [Base Documentation](https://docs.base.org/). - - columns: - - name: _LOG_ID - description: This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred. - - name: BLOCK_NUMBER - description: Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. - - name: BLOCK_HASH - description: Block hash is a unique 66-character identifier that is generate when a block is produced. - - name: BLOCK_TIMESTAMP - description: The date and time at which the block was produced. - - name: TX_HASH - description: Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This field will not be unique in this table, as a given transaction can include multiple events. - - name: ORIGIN_FUNCTION_SIGNATURE - description: The function signature of the contract call. - - name: ORIGIN_FROM_ADDRESS - description: The from address of this transaction. - - name: ORIGIN_TO_ADDRESS - description: The to address of this transaction. - - name: EVENT_INDEX - description: Event number within a transaction. - - name: CONTRACT_ADDRESS - description: The address interacted with for a given event. - - name: TOPICS - description: The un-decoded event input topics. - - name: DATA - description: The un-decoded event data. - - name: EVENT_REMOVED - description: Whether the event has been removed from the transaction. - - name: TX_STATUS - description: Status of the transaction. - - name: TX_INDEX - description: The transactions index position from which this log was created from. - - name: GAS_USED - description: Gas used by this event. - - name: CUMULATIVE_GAS_USED - description: The total amount of gas used when this transaction was executed in the block. - - name: EFFECTIVE_GAS_PRICE - description: The total base charge plus tip paid for each unit of gas. - - name: TYPE - description: The type of value. \ No newline at end of file + description: Deprecating soon. Please migrate your queries to mainnet. \ No newline at end of file diff --git a/models/gold/base_goerli/goerli__fact_traces.yml b/models/gold/base_goerli/goerli__fact_traces.yml index 5b3a743..50a26ca 100644 --- a/models/gold/base_goerli/goerli__fact_traces.yml +++ b/models/gold/base_goerli/goerli__fact_traces.yml @@ -1,40 +1,4 @@ version: 2 models: - name: goerli__fact_traces - description: This table contains trace level data for the Base Blockchain, Goerli Testnet. Each trace will have a unique trace identifier, along with the transaction hash, block number, and block timestamp. For more information on Base, please see [Base Documentation](https://docs.base.org/). For more information on EVM traces, please see [The Ethereum Organization - Traces](https://ethereum.org/en/developers/docs/traces/) - - columns: - - name: BLOCK_NUMBER - description: Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. - - name: BLOCK_TIMESTAMP - description: The date and time at which the block was produced. - - name: TX_HASH - description: Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. Please note that this is not unique in this table, as a transaction may have multiple traces. - - name: FROM_ADDRESS - description: The sending address of this trace. - - name: TO_ADDRESS - description: The receiving address of this trace. This can be a contract address. - - name: ETH_VALUE - description: The value transacted in ETH. - - name: GAS - description: Gas supplied by the trace. - - name: GAS_USED - description: Gas used by the trace. - - name: INPUT - description: This column contains additional data for this trace, and is commonly used as part of a contract interaction or as a message to the recipient. - - name: OUTPUT - description: This column contains the output of the trace. - - name: TYPE - description: This column indicates the trace type, such as `CALL`, `STATICCALL`, `CREATE`, `DELEGATECALL', etc. - - name: IDENTIFIER - description: This field represents the position and type of the trace within the transaction. - - name: DATA - description: The raw JSON data for this trace. - - name: TX_STATUS - description: Status of the transaction. - - name: SUB_TRACES - description: The amount of nested sub traces for a given trace. - - - - \ No newline at end of file + description: Deprecating soon. Please migrate your queries to mainnet. \ No newline at end of file diff --git a/models/gold/base_goerli/goerli__fact_transactions.yml b/models/gold/base_goerli/goerli__fact_transactions.yml index e31b6d9..bbe7b53 100644 --- a/models/gold/base_goerli/goerli__fact_transactions.yml +++ b/models/gold/base_goerli/goerli__fact_transactions.yml @@ -1,52 +1,4 @@ version: 2 models: - name: goerli__fact_transactions - description: This table contains transaction level data for the Base Blockchain, Goerli Testnet. Each transaction will have a unique transaction hash, along with transactions fees and an ETH value transferred when applicable. Transactions may be native ETH transfers or interactions with contract addresses. For more information on Base, please see [Base Documentation](https://docs.base.org/). For more information on EVM transactions, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/) - - columns: - - name: BLOCK_NUMBER - description: Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. - - name: BLOCK_TIMESTAMP - description: The date and time at which the block was produced. - - name: TX_HASH - description: Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. - - name: NONCE - description: The number of transactions sent from a given address. - - name: POSITION - description: The position of the transaction within the block. - - name: ORIGIN_FUNCTION_SIGNATURE - description: The function signature of the transaction. - - name: FROM_ADDRESS - description: The sending address of this transaction. - - name: TO_ADDRESS - description: The receiving address of this transaction. This can be a contract address. - - name: ETH_VALUE - description: The value transacted in ETH. - - name: BLOCK_HASH - description: Block hash is a unique 66-character identifier that is generated when a block is produced. - - name: GAS_PRICE - description: Cost per unit of gas in Gwei. - - name: GAS_LIMIT - description: Maximum amount of gas allocated for the transaction. - - name: INPUT_DATA - description: This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient. - - name: STATUS - description: Status of the transaction. - - name: GAS_USED - description: Gas used by the transaction. - - name: CUMULATIVE_GAS_USED - description: The total amount of gas used when this transaction was executed in the block. - - name: L1_FEE_SCALAR - description: This value covers the change in L1 gas price between the time the transaction is submitted and when it is published. - - name: L1_GAS_USED - description: The gas used on L1 to publish the transaction. - - name: L1_GAS_PRICE - description: The gas price for L1 transactions when the transaction was processed. - - name: TX_FEE - description: Amount paid to validate the transaction in ETH. - - name: TX_TYPE - description: This column indicates the transaction type. - - name: IS_SYSTEM_TX - description: This column indicates system transactions. - - name: TX_JSON - description: This JSON column contains the transaction details, including event logs. \ No newline at end of file + description: Deprecating soon. Please migrate your queries to mainnet. \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql new file mode 100644 index 0000000..2fb9685 --- /dev/null +++ b/models/gold/core/core__fact_blocks.sql @@ -0,0 +1,68 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + A.block_number AS block_number, + block_timestamp, + 'mainnet' AS network, + 'base' AS blockchain, + tx_count, + difficulty, + total_difficulty, + extra_data, + gas_limit, + gas_used, + HASH, + parent_hash, + receipts_root, + sha3_uncles, + SIZE, + uncles AS uncle_blocks, + OBJECT_CONSTRUCT( + 'baseFeePerGas', + base_fee_per_gas, + 'difficulty', + difficulty, + 'extraData', + extra_data, + 'gasLimit', + gas_limit, + 'gasUsed', + gas_used, + 'hash', + HASH, + 'logsBloom', + logs_bloom, + 'miner', + miner, + 'nonce', + nonce, + 'number', + NUMBER, + 'parentHash', + parent_hash, + 'receiptsRoot', + receipts_root, + 'sha3Uncles', + sha3_uncles, + 'size', + SIZE, + 'stateRoot', + state_root, + 'timestamp', + block_timestamp, + 'totalDifficulty', + total_difficulty, + 'transactionsRoot', + transactions_root, + 'uncles', + uncles + ) AS block_header_json +FROM + {{ ref('silver__blocks') }} A + LEFT JOIN {{ ref('silver__tx_count') }} + d + ON A.block_number = d.block_number diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml new file mode 100644 index 0000000..ff1d3e7 --- /dev/null +++ b/models/gold/core/core__fact_blocks.yml @@ -0,0 +1,40 @@ +version: 2 +models: + - name: core__fact_blocks + description: '{{ doc("base_blocks_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("base_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("base_block_timestamp") }}' + - name: NETWORK + description: '{{ doc("base_network") }}' + - name: BLOCKCHAIN + description: '{{ doc("base_blockchain") }}' + - name: TX_COUNT + description: '{{ doc("base_tx_count") }}' + - name: DIFFICULTY + description: '{{ doc("base_difficulty") }}' + - name: TOTAL_DIFFICULTY + description: '{{ doc("base_total_difficulty") }}' + - name: EXTRA_DATA + description: '{{ doc("base_extra_data") }}' + - name: GAS_LIMIT + description: '{{ doc("base_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("base_gas_used") }}' + - name: HASH + description: '{{ doc("base_blocks_hash") }}' + - name: PARENT_HASH + description: '{{ doc("base_parent_hash") }}' + - name: RECEIPTS_ROOT + description: '{{ doc("base_receipts_root") }}' + - name: SHA3_UNCLES + description: '{{ doc("base_sha3_uncles") }}' + - name: SIZE + description: '{{ doc("base_size") }}' + - name: UNCLE_BLOCKS + description: '{{ doc("base_uncle_blocks") }}' + - name: BLOCK_HEADER_JSON + description: '{{ doc("base_block_header_json") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_event_logs.sql b/models/gold/core/core__fact_event_logs.sql new file mode 100644 index 0000000..f57c5bf --- /dev/null +++ b/models/gold/core/core__fact_event_logs.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + contract_address, + topics, + DATA, + event_removed, + tx_status, + _log_id +FROM + {{ ref('silver__logs') }} diff --git a/models/gold/core/core__fact_event_logs.yml b/models/gold/core/core__fact_event_logs.yml new file mode 100644 index 0000000..b1e310e --- /dev/null +++ b/models/gold/core/core__fact_event_logs.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: core__fact_event_logs + description: '{{ doc("base_logs_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("base_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("base_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("base_logs_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("base_event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("base_logs_contract_address") }}' + - name: TOPICS + description: '{{ doc("base_topics") }}' + - name: DATA + description: '{{ doc("base_logs_data") }}' + - name: EVENT_REMOVED + description: '{{ doc("base_event_removed") }}' + - name: _LOG_ID + description: '{{ doc("base_log_id_events") }}' + - name: TX_STATUS + description: '{{ doc("base_tx_status") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("base_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("base_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("base_origin_to") }}' diff --git a/models/gold/core/core__fact_token_transfers.sql b/models/gold/core/core__fact_token_transfers.sql new file mode 100644 index 0000000..1b8c258 --- /dev/null +++ b/models/gold/core/core__fact_token_transfers.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + from_address, + to_address, + raw_amount, + _log_id +FROM + {{ ref('silver__transfers') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_token_transfers.yml b/models/gold/core/core__fact_token_transfers.yml new file mode 100644 index 0000000..a555f36 --- /dev/null +++ b/models/gold/core/core__fact_token_transfers.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: core__fact_token_transfers + description: '{{ doc("base_transfer_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("base_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("base_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("base_transfer_tx_hash") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("base_transfer_contract_address") }}' + - name: FROM_ADDRESS + description: '{{ doc("base_transfer_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("base_transfer_to_address") }}' + - name: RAW_AMOUNT + description: '{{ doc("base_transfer_raw_amount") }}' + - name: _LOG_ID + description: '{{ doc("base_log_id_transfers") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("base_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("base_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("base_origin_to") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql new file mode 100644 index 0000000..723b590 --- /dev/null +++ b/models/gold/core/core__fact_traces.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + DATA, + tx_status, + sub_traces, + trace_status, + error_reason, + trace_index +FROM + {{ ref('silver__traces') }} diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml new file mode 100644 index 0000000..db269a8 --- /dev/null +++ b/models/gold/core/core__fact_traces.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: core__fact_traces + description: '{{ doc("base_traces_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("base_traces_block_no") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("base_traces_blocktime") }}' + - name: TX_HASH + description: '{{ doc("base_traces_tx_hash") }}' + - name: FROM_ADDRESS + description: '{{ doc("base_traces_from") }}' + - name: TO_ADDRESS + description: '{{ doc("base_traces_to") }}' + - name: ETH_VALUE + description: '{{ doc("base_traces_value") }}' + - name: GAS + description: '{{ doc("base_traces_gas") }}' + - name: GAS_USED + description: '{{ doc("base_traces_gas_used") }}' + - name: INPUT + description: '{{ doc("base_traces_input") }}' + - name: OUTPUT + description: '{{ doc("base_traces_output") }}' + - name: TYPE + description: '{{ doc("base_traces_type") }}' + - name: IDENTIFIER + description: '{{ doc("base_traces_identifier") }}' + - name: DATA + description: '{{ doc("base_traces_call_data") }}' + - name: TX_STATUS + description: '{{ doc("base_tx_status") }}' + - name: SUB_TRACES + description: '{{ doc("base_traces_sub") }}' + - name: TRACE_STATUS + description: The status of the trace, either `SUCCESS` or `FAIL` + - name: ERROR_REASON + description: The reason for the trace failure, if any. + - name: TRACE_INDEX + description: The index of the trace within the transaction. + + + + \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql new file mode 100644 index 0000000..0d09f96 --- /dev/null +++ b/models/gold/core/core__fact_transactions.sql @@ -0,0 +1,36 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number AS block_number, + block_timestamp, + block_hash, + tx_hash, + nonce, + POSITION, + origin_function_signature, + from_address, + to_address, + VALUE AS eth_value, + tx_fee, + gas_price, + effective_gas_price, + gas AS gas_limit, + gas_used, + l1_gas_price, + l1_gas_used, + l1_fee_scalar, + l1_fee, + cumulative_gas_used, + max_fee_per_gas, + max_priority_fee_per_gas, + input_data, + tx_status AS status, + r, + s, + v +FROM + {{ ref('silver__transactions') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml new file mode 100644 index 0000000..3d7c92d --- /dev/null +++ b/models/gold/core/core__fact_transactions.yml @@ -0,0 +1,60 @@ +version: 2 +models: + - name: core__fact_transactions + description: '{{ doc("base_tx_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("base_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("base_block_timestamp") }}' + - name: BLOCK_HASH + description: '{{ doc("base_tx_block_hash") }}' + - name: TX_HASH + description: '{{ doc("base_tx_hash") }}' + - name: NONCE + description: '{{ doc("base_tx_nonce") }}' + - name: POSITION + description: '{{ doc("base_tx_position") }}' + - name: FROM_ADDRESS + description: '{{ doc("base_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("base_to_address") }}' + - name: ETH_VALUE + description: '{{ doc("base_value") }}' + - name: TX_FEE + description: '{{ doc("base_tx_fee") }}' + - name: GAS_PRICE + description: '{{ doc("base_tx_gas_price") }}' + - name: EFFECTIVE_GAS_PRICE + description: The total base charge plus tip paid for each unit of gas, in Gwei. + - name: GAS_LIMIT + description: '{{ doc("base_tx_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("base_tx_gas_used") }}' + - name: L1_GAS_PRICE + description: '{{ doc("base_l1_gas_price") }}' + - name: L1_GAS_USED + description: '{{ doc("base_l1_gas_used") }}' + - name: L1_FEE_SCALAR + description: '{{ doc("base_l1_fee_scalar") }}' + - name: L1_FEE + description: The L1 portion of fees paid. + - name: CUMULATIVE_GAS_USED + description: '{{ doc("base_cumulative_gas_used") }}' + - name: MAX_FEE_PER_GAS + description: The maximum fee per gas of the transaction, in Gwei. + - name: MAX_PRIORITY_FEE_PER_GAS + description: The maximum priority fee per gas of the transaction, in Gwei. + - name: STATUS + description: '{{ doc("base_tx_status") }}' + - name: INPUT_DATA + description: '{{ doc("base_tx_input_data") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("base_tx_origin_sig") }}' + - name: R + description: The r value of the transaction signature. + - name: S + description: The s value of the transaction signature. + - name: V + description: The v value of the transaction signature. \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql new file mode 100644 index 0000000..2e1bd9c --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -0,0 +1,166 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +blocks AS ( + SELECT + l.block_number, + block_timestamp, + LAG( + l.block_number, + 1 + ) over ( + ORDER BY + l.block_number ASC + ) AS prev_BLOCK_NUMBER + FROM + {{ ref("silver__blocks") }} + l + INNER JOIN block_range b + ON l.block_number = b.block_number + AND l.block_number >= ( + SELECT + MIN(block_number) + FROM + block_range + ) +), +block_gen AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + MIN(block_number) + FROM + blocks + ) + AND ( + SELECT + MAX(block_number) + FROM + blocks + ) +) +SELECT + 'blocks' AS test_name, + MIN( + b.block_number + ) AS min_block, + MAX( + b.block_number + ) AS max_block, + MIN( + b.block_timestamp + ) AS min_block_timestamp, + MAX( + b.block_timestamp + ) AS max_block_timestamp, + COUNT(1) AS blocks_tested, + COUNT( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) AS blocks_impacted_count, + ARRAY_AGG( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) within GROUP ( + ORDER BY + A.block_number + ) AS blocks_impacted_array, + CURRENT_TIMESTAMP AS test_timestamp +FROM + block_gen A + LEFT JOIN blocks b + ON A.block_number = b.block_number + LEFT JOIN blocks C + ON A.block_number > C.prev_block_number + AND A.block_number < C.block_number + AND C.block_number - C.prev_block_number <> 1 +WHERE + COALESCE( + b.block_number, + C.block_number + ) IS NOT NULL diff --git a/models/silver/_observability/silver_observability__logs_completeness.sql b/models/silver/_observability/silver_observability__logs_completeness.sql new file mode 100644 index 0000000..f305ee4 --- /dev/null +++ b/models/silver/_observability/silver_observability__logs_completeness.sql @@ -0,0 +1,121 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__receipts") }} + r + LEFT JOIN {{ ref("silver__logs") }} + l USING ( + block_number, + tx_hash + ) + JOIN block_range USING (block_number) + WHERE + l.tx_hash IS NULL + AND ARRAY_SIZE( + r.logs + ) > 0 +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'event_logs' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__receipts_completeness.sql b/models/silver/_observability/silver_observability__receipts_completeness.sql new file mode 100644 index 0000000..8ac50a3 --- /dev/null +++ b/models/silver/_observability/silver_observability__receipts_completeness.sql @@ -0,0 +1,119 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__transactions") }} + t + LEFT JOIN {{ ref("silver__receipts") }} + r USING ( + block_number, + tx_hash, + block_hash + ) + JOIN block_range USING (block_number) + WHERE + r.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'receipts' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__traces_completeness.sql b/models/silver/_observability/silver_observability__traces_completeness.sql new file mode 100644 index 0000000..5c4c597 --- /dev/null +++ b/models/silver/_observability/silver_observability__traces_completeness.sql @@ -0,0 +1,118 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__transactions") }} + tx + LEFT JOIN {{ ref("silver__traces") }} + tr USING ( + block_number, + tx_hash + ) + JOIN block_range USING (block_number) + WHERE + tr.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'traces' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql new file mode 100644 index 0000000..013b85a --- /dev/null +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -0,0 +1,119 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ source( + 'silver_crosschain', + 'number_sequence' + ) }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__confirmed_blocks") }} + b + LEFT JOIN {{ ref("silver__transactions") }} + t USING ( + block_number, + tx_hash, + block_hash + ) + JOIN block_range USING (block_number) + WHERE + t.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'transactions' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql new file mode 100644 index 0000000..575b298 --- /dev/null +++ b/models/silver/core/silver__blocks.sql @@ -0,0 +1,68 @@ +-- depends_on: {{ ref('bronze__streamline_blocks') }} +{{ config( + materialized = 'incremental', + unique_key = "block_number", + cluster_by = "block_timestamp::date", + tags = ['core'], + full_refresh = false +) }} + +SELECT + block_number, + utils.udf_hex_to_int( + DATA :baseFeePerGas :: STRING + ) :: INT AS base_fee_per_gas, + utils.udf_hex_to_int( + DATA :difficulty :: STRING + ) :: INT AS difficulty, + DATA :extraData :: STRING AS extra_data, + utils.udf_hex_to_int( + DATA :gasLimit :: STRING + ) :: INT AS gas_limit, + utils.udf_hex_to_int( + DATA :gasUsed :: STRING + ) :: INT AS gas_used, + DATA :hash :: STRING AS HASH, + DATA :logsBloom :: STRING AS logs_bloom, + DATA :miner :: STRING AS miner, + DATA :mixHash :: STRING AS mixHash, + utils.udf_hex_to_int( + DATA :nonce :: STRING + ) :: INT AS nonce, + utils.udf_hex_to_int( + DATA :number :: STRING + ) :: INT AS NUMBER, + DATA :parentHash :: STRING AS parent_hash, + DATA :receiptsRoot :: STRING AS receipts_root, + DATA :sha3Uncles :: STRING AS sha3_uncles, + utils.udf_hex_to_int( + DATA :size :: STRING + ) :: INT AS SIZE, + DATA :stateRoot :: STRING AS state_root, + utils.udf_hex_to_int( + DATA :timestamp :: STRING + ) :: TIMESTAMP AS block_timestamp, + utils.udf_hex_to_int( + DATA :totalDifficulty :: STRING + ) :: INT AS total_difficulty, + DATA :transactionsRoot :: STRING AS transactions_root, + DATA :uncles AS uncles, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__confirmed_blocks.sql b/models/silver/core/silver__confirmed_blocks.sql new file mode 100644 index 0000000..3a24502 --- /dev/null +++ b/models/silver/core/silver__confirmed_blocks.sql @@ -0,0 +1,50 @@ +-- depends_on: {{ ref('bronze__streamline_confirm_blocks') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "round(block_number,-3)", + full_refresh = false +) }} + +WITH base AS ( + + SELECT + block_number, + DATA :result :hash :: STRING AS block_hash, + DATA :result :transactions txs, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_confirm_blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + IFNULL( + MAX( + _inserted_timestamp + ), + '1970-01-01' :: TIMESTAMP + ) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_confirm_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 +) +SELECT + block_number, + block_hash, + VALUE :: STRING AS tx_hash, + _inserted_timestamp +FROM + base, + LATERAL FLATTEN ( + input => txs + ) diff --git a/models/silver/core/silver__logs.sql b/models/silver/core/silver__logs.sql new file mode 100644 index 0000000..135041d --- /dev/null +++ b/models/silver/core/silver__logs.sql @@ -0,0 +1,183 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'], + full_refresh = false +) }} + +WITH base AS ( + + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + tx_status, + logs, + _inserted_timestamp + FROM + {{ ref('silver__receipts') }} + WHERE + ARRAY_SIZE(logs) > 0 + +{% if is_incremental() %} +AND _INSERTED_TIMESTAMP >= ( + SELECT + MAX(_INSERTED_TIMESTAMP) _INSERTED_TIMESTAMP + FROM + {{ this }} +) +{% endif %} +), +flat_logs AS ( + SELECT + block_number, + tx_hash, + origin_from_address, + origin_to_address, + tx_status, + VALUE :address :: STRING AS contract_address, + VALUE :blockHash :: STRING AS block_hash, + VALUE :data :: STRING AS DATA, + utils.udf_hex_to_int( + VALUE :logIndex :: STRING + ) :: INT AS event_index, + VALUE :removed :: BOOLEAN AS event_removed, + VALUE :topics AS topics, + _inserted_timestamp + FROM + base, + LATERAL FLATTEN( + input => logs + ) +), +new_records AS ( + SELECT + l.block_number, + txs.block_timestamp, + l.tx_hash, + l.origin_from_address, + l.origin_to_address, + txs.origin_function_signature, + l.tx_status, + l.contract_address, + l.block_hash, + l.data, + l.event_index, + l.event_removed, + l.topics, + l._inserted_timestamp, + CASE + WHEN txs.block_timestamp IS NULL + OR txs.origin_function_signature IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id + FROM + flat_logs l + LEFT OUTER JOIN {{ ref('silver__transactions') }} + txs + ON l.block_number = txs.block_number + AND l.tx_hash = txs.tx_hash + +{% if is_incremental() %} +AND txs._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + txs.block_timestamp, + t.tx_hash, + t.origin_from_address, + t.origin_to_address, + txs.origin_function_signature, + t.tx_status, + t.contract_address, + t.block_hash, + t.data, + t.event_index, + t.event_removed, + t.topics, + GREATEST( + t._inserted_timestamp, + txs._inserted_timestamp + ) AS _inserted_timestamp, + _log_id, + FALSE AS is_pending + FROM + {{ this }} + t + INNER JOIN {{ ref('silver__transactions') }} + txs USING ( + block_number, + tx_hash + ) + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending +FROM + missing_data +{% endif %} +) +SELECT + * +FROM + FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, event_index +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/silver/core/silver__receipts.sql b/models/silver/core/silver__receipts.sql new file mode 100644 index 0000000..71a42db --- /dev/null +++ b/models/silver/core/silver__receipts.sql @@ -0,0 +1,124 @@ +-- depends_on: {{ ref('bronze__streamline_receipts') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", + tags = ['core'], + full_refresh = false +) }} + +WITH base AS ( + + SELECT + block_number, + DATA, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_receipts') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) + AND IS_OBJECT(DATA) +{% else %} + {{ ref('bronze__streamline_FR_receipts') }} +WHERE + IS_OBJECT(DATA) +{% endif %} +), +FINAL AS ( + SELECT + block_number, + DATA :blockHash :: STRING AS block_hash, + utils.udf_hex_to_int( + DATA :blockNumber :: STRING + ) :: INT AS blockNumber, + DATA :contractAddress :: STRING AS contractAddress, + utils.udf_hex_to_int( + DATA :cumulativeGasUsed :: STRING + ) :: INT AS cumulative_gas_used, + utils.udf_hex_to_int( + DATA :effectiveGasPrice :: STRING + ) :: INT / pow( + 10, + 9 + ) AS effective_gas_price, + DATA :from :: STRING AS from_address, + COALESCE( + utils.udf_hex_to_int( + DATA :gasUsed :: STRING + ) :: INT, + 0 + ) AS gas_used, + COALESCE( + utils.udf_hex_to_int( + DATA :l1Fee :: STRING + ) :: FLOAT, + 0 + ) AS l1_fee, + COALESCE( + ( + DATA :l1FeeScalar :: STRING + ) :: FLOAT, + 0 + ) AS l1_fee_scalar, + COALESCE( + utils.udf_hex_to_int( + DATA :l1GasUsed :: STRING + ) :: FLOAT, + 0 + ) AS l1_gas_used, + COALESCE( + utils.udf_hex_to_int( + DATA :l1GasPrice :: STRING + ) :: FLOAT, + 0 + ) AS l1_gas_price, + DATA :logs AS logs, + DATA :logsBloom :: STRING AS logs_bloom, + utils.udf_hex_to_int( + DATA :status :: STRING + ) :: INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA :to :: STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA :transactionHash :: STRING AS tx_hash, + CASE + WHEN block_number <> blockNumber THEN NULL + ELSE utils.udf_hex_to_int( + DATA :transactionIndex :: STRING + ) :: INT + END AS POSITION, + utils.udf_hex_to_int( + DATA :type :: STRING + ) :: INT AS TYPE, + _inserted_timestamp + FROM + base +) +SELECT + * +FROM + FINAL +WHERE + tx_hash IS NOT NULL + AND POSITION IS NOT NULL qualify(ROW_NUMBER() over (PARTITION BY block_number, POSITION +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql new file mode 100644 index 0000000..ccd60f8 --- /dev/null +++ b/models/silver/core/silver__traces.sql @@ -0,0 +1,369 @@ +-- depends_on: {{ ref('bronze__streamline_traces') }} +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'], + full_refresh = false +) }} + +WITH traces_txs AS ( + + SELECT + block_number, + VALUE :array_index :: INT AS tx_position, + DATA :result AS full_traces, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_traces') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_traces') }} +WHERE + _partition_by_block_id <= 5000000 +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position +ORDER BY + _inserted_timestamp DESC)) = 1 +), +base_table AS ( + SELECT + CASE + WHEN POSITION( + '.', + path :: STRING + ) > 0 THEN REPLACE( + REPLACE( + path :: STRING, + SUBSTR(path :: STRING, len(path :: STRING) - POSITION('.', REVERSE(path :: STRING)) + 1, POSITION('.', REVERSE(path :: STRING))), + '' + ), + '.', + '__' + ) + ELSE '__' + END AS id, + OBJECT_AGG( + DISTINCT key, + VALUE + ) AS DATA, + txs.tx_position AS tx_position, + txs.block_number AS block_number, + txs._inserted_timestamp AS _inserted_timestamp + FROM + traces_txs txs, + TABLE( + FLATTEN( + input => PARSE_JSON( + txs.full_traces + ), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + GROUP BY + tx_position, + id, + block_number, + _inserted_timestamp +), +flattened_traces AS ( + SELECT + DATA :from :: STRING AS from_address, + utils.udf_hex_to_int( + DATA :gas :: STRING + ) AS gas, + utils.udf_hex_to_int( + DATA :gasUsed :: STRING + ) AS gas_used, + DATA :input :: STRING AS input, + DATA :output :: STRING AS output, + DATA :error :: STRING AS error_reason, + DATA :to :: STRING AS to_address, + DATA :type :: STRING AS TYPE, + CASE + WHEN DATA :type :: STRING = 'CALL' THEN utils.udf_hex_to_int( + DATA :value :: STRING + ) / pow( + 10, + 18 + ) + ELSE 0 + END AS eth_value, + CASE + WHEN id = '__' THEN CONCAT( + DATA :type :: STRING, + '_ORIGIN' + ) + ELSE CONCAT( + DATA :type :: STRING, + '_', + REPLACE( + REPLACE(REPLACE(REPLACE(id, 'calls', ''), '[', ''), ']', ''), + '__', + '_' + ) + ) + END AS identifier, + concat_ws( + '-', + block_number, + tx_position, + identifier + ) AS _call_id, + SPLIT( + identifier, + '_' + ) AS id_split, + ARRAY_SLICE(id_split, 1, ARRAY_SIZE(id_split)) AS levels, + ARRAY_TO_STRING( + levels, + '_' + ) AS LEVEL, + CASE + WHEN ARRAY_SIZE(levels) = 1 + AND levels [0] :: STRING = 'ORIGIN' THEN NULL + WHEN ARRAY_SIZE(levels) = 1 THEN 'ORIGIN' + ELSE ARRAY_TO_STRING(ARRAY_SLICE(levels, 0, ARRAY_SIZE(levels) -1), '_')END AS parent_level, + COUNT(parent_level) over ( + PARTITION BY block_number, + tx_position, + parent_level + ) AS sub_traces,* + FROM + base_table + ), + group_sub_traces AS ( + SELECT + tx_position, + block_number, + parent_level, + sub_traces + FROM + flattened_traces + GROUP BY + tx_position, + block_number, + parent_level, + sub_traces + ), + add_sub_traces AS ( + SELECT + flattened_traces.tx_position AS tx_position, + flattened_traces.block_number :: INTEGER AS block_number, + flattened_traces.error_reason AS error_reason, + flattened_traces.from_address AS from_address, + flattened_traces.to_address AS to_address, + flattened_traces.eth_value :: FLOAT AS eth_value, + flattened_traces.gas :: FLOAT AS gas, + flattened_traces.gas_used :: FLOAT AS gas_used, + flattened_traces.input AS input, + flattened_traces.output AS output, + flattened_traces.type AS TYPE, + flattened_traces.identifier AS identifier, + flattened_traces._call_id AS _call_id, + flattened_traces.data AS DATA, + group_sub_traces.sub_traces AS sub_traces, + ROW_NUMBER() over( + PARTITION BY flattened_traces.block_number, + flattened_traces.tx_position + ORDER BY + flattened_traces.gas :: FLOAT DESC, + flattened_traces.eth_value :: FLOAT ASC, + flattened_traces.to_address + ) AS trace_index, + flattened_traces._inserted_timestamp AS _inserted_timestamp + FROM + flattened_traces + LEFT OUTER JOIN group_sub_traces + ON flattened_traces.tx_position = group_sub_traces.tx_position + AND flattened_traces.level = group_sub_traces.parent_level + AND flattened_traces.block_number = group_sub_traces.block_number + ), + final_traces AS ( + SELECT + tx_position, + trace_index, + block_number, + error_reason, + from_address, + to_address, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + _call_id, + _inserted_timestamp, + DATA, + sub_traces + FROM + add_sub_traces + WHERE + identifier IS NOT NULL + ), + new_records AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address, + f.to_address, + f.eth_value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + CASE + WHEN f.error_reason IS NULL THEN 'SUCCESS' + ELSE 'FAIL' + END AS trace_status, + f.data, + CASE + WHEN t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + f._call_id, + f._inserted_timestamp + FROM + final_traces f + LEFT OUTER JOIN {{ ref('silver__transactions') }} + t + ON f.tx_position = t.position + AND f.block_number = t.block_number + +{% if is_incremental() %} +AND t._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + txs.tx_hash, + txs.block_timestamp, + txs.tx_status, + t.tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.eth_value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.identifier, + t.sub_traces, + t.error_reason, + t.trace_status, + t.data, + FALSE AS is_pending, + t._call_id, + GREATEST( + t._inserted_timestamp, + txs._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ this }} + t + INNER JOIN {{ ref('silver__transactions') }} + txs + ON t.tx_position = txs.position + AND t.block_number = txs.block_number + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + tx_hash, + block_timestamp, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + trace_status, + DATA, + is_pending, + _call_id, + _inserted_timestamp +FROM + missing_data +{% endif %} +) +SELECT + * +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql new file mode 100644 index 0000000..671a43e --- /dev/null +++ b/models/silver/core/silver__transactions.sql @@ -0,0 +1,363 @@ +-- depends_on: {{ ref('bronze__streamline_transactions') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'], + full_refresh = false +) }} + +WITH base AS ( + + SELECT + block_number, + DATA, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_transactions') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) + AND IS_OBJECT(DATA) +{% else %} + {{ ref('bronze__streamline_FR_transactions') }} +WHERE + IS_OBJECT(DATA) +{% endif %} +), +base_tx AS ( + SELECT + A.block_number AS block_number, + A.data :blockHash :: STRING AS block_hash, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :blockNumber :: STRING + ) + ) AS blockNumber, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :chainId :: STRING + ) + ) AS chain_id, + A.data :from :: STRING AS from_address, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :gas :: STRING + ) + ) AS gas, + COALESCE( + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :gasPrice :: STRING + ) + ), + 0 + ) AS gas_price, + A.data :hash :: STRING AS tx_hash, + A.data :input :: STRING AS input_data, + SUBSTR( + input_data, + 1, + 10 + ) AS origin_function_signature, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :maxFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_fee_per_gas, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :maxPriorityFeePerGas :: STRING + ) + ) / pow( + 10, + 9 + ) AS max_priority_fee_per_gas, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :nonce :: STRING + ) + ) AS nonce, + A.data :r :: STRING AS r, + A.data :s :: STRING AS s, + A.data :to :: STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :transactionIndex :: STRING + ) + ) AS POSITION, + A.data :type :: STRING AS TYPE, + A.data :v :: STRING AS v, + TRY_TO_NUMBER( + utils.udf_hex_to_int( + A.data :value :: STRING + ) + ) / pow( + 10, + 18 + ) :: FLOAT AS VALUE, + A._INSERTED_TIMESTAMP + FROM + base A +), +new_records AS ( + SELECT + t.block_number, + t.block_hash, + t.chain_id, + t.from_address, + t.gas, + t.gas_price / pow( + 10, + 9 + ) AS gas_price, + t.gas_price AS gas_price1, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.nonce, + t.r, + t.s, + t.to_address, + t.position, + t.type, + t.v, + t.value, + block_timestamp, + CASE + WHEN block_timestamp IS NULL + OR tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + r.gas_used, + r.l1_fee / pow( + 10, + 18 + ) AS l1_fee, + r.l1_fee_scalar, + r.l1_gas_used, + r.l1_gas_price / pow( + 10, + 9 + ) AS l1_gas_price, + r.l1_gas_price AS l1_gas_price1, + COALESCE( + ((r.gas_used * gas_price1) + (l1_gas_price1 * r.l1_gas_used * r.l1_fee_scalar)) / pow( + 10, + 18 + ), + 0 + ) AS tx_fee, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + r.type AS tx_type, + t._inserted_timestamp + FROM + base_tx t + LEFT OUTER JOIN {{ ref('silver__blocks') }} + b + ON t.block_number = b.block_number + LEFT OUTER JOIN {{ ref('silver__receipts') }} + r + ON t.block_number = r.block_number + AND t.tx_hash = r.tx_hash + +{% if is_incremental() %} +AND r._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + t.block_hash, + t.chain_id, + t.from_address, + t.gas, + t.gas_price, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.nonce, + t.r, + t.s, + t.to_address, + t.position, + t.type, + t.v, + t.value, + b.block_timestamp, + FALSE AS is_pending, + r.gas_used, + r.tx_success, + r.tx_status, + r.cumulative_gas_used, + r.effective_gas_price, + t.l1_fee, + t.l1_fee_scalar, + t.l1_gas_used, + t.l1_gas_price, + t.tx_fee, + r.type AS tx_type, + GREATEST( + t._inserted_timestamp, + b._inserted_timestamp, + r._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ this }} + t + INNER JOIN {{ ref('silver__blocks') }} + b + ON t.block_number = b.block_number + INNER JOIN {{ ref('silver__receipts') }} + r + ON t.tx_hash = r.tx_hash + AND t.block_number = r.block_number + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + to_address, + POSITION, + TYPE, + v, + VALUE, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + l1_fee, + l1_fee_scalar, + l1_gas_used, + l1_gas_price, + tx_fee, + tx_type, + _inserted_timestamp + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + to_address, + POSITION, + TYPE, + v, + VALUE, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + l1_fee, + l1_fee_scalar, + l1_gas_used, + l1_gas_price, + tx_fee, + tx_type, + _inserted_timestamp +FROM + missing_data +{% endif %} +) +SELECT + block_number, + block_hash, + chain_id, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + to_address, + POSITION, + TYPE, + v, + VALUE, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + effective_gas_price, + l1_fee, + l1_fee_scalar, + l1_gas_used, + l1_gas_price, + tx_fee, + tx_type, + _inserted_timestamp +FROM + FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, POSITION +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql new file mode 100644 index 0000000..7a248b8 --- /dev/null +++ b/models/silver/core/silver__transfers.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + unique_key = '_log_id', + cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'] +) }} + +WITH logs AS ( + + SELECT + _log_id, + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address :: STRING AS contract_address, + CONCAT('0x', SUBSTR(topics [1], 27, 40)) :: STRING AS from_address, + CONCAT('0x', SUBSTR(topics [2], 27, 40)) :: STRING AS to_address, + utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) :: FLOAT AS raw_amount, + event_index, + _inserted_timestamp + FROM + {{ ref('silver__logs') }} + WHERE + topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + AND tx_status = 'SUCCESS' + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} +) +SELECT + _log_id, + block_number, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + block_timestamp, + contract_address, + from_address, + to_address, + raw_amount, + _inserted_timestamp, + event_index +FROM + logs +WHERE + raw_amount IS NOT NULL + AND to_address IS NOT NULL + AND from_address IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/silver/core/silver__transfers.yml b/models/silver/core/silver__transfers.yml new file mode 100644 index 0000000..7b597db --- /dev/null +++ b/models/silver/core/silver__transfers.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: silver__transfers + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: BLOCK_TIMESTAMP > '2021-08-01' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: RAW_AMOUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ \ No newline at end of file diff --git a/models/silver/core/silver__tx_count.sql b/models/silver/core/silver__tx_count.sql new file mode 100644 index 0000000..0144d01 --- /dev/null +++ b/models/silver/core/silver__tx_count.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'incremental', + unique_key = "block_number" +) }} + +SELECT + block_number, + MIN(_inserted_timestamp) AS _inserted_timestamp, + COUNT(*) AS tx_count +FROM + {{ ref('silver__transactions') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% endif %} +GROUP BY + block_number \ No newline at end of file diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.sql b/models/silver/core/tests/blocks/test_silver__blocks_full.sql new file mode 100644 index 0000000..25dc03b --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__blocks') }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.yml b/models/silver/core/tests/blocks/test_silver__blocks_full.yml new file mode 100644 index 0000000..f90d39c --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.yml @@ -0,0 +1,95 @@ +version: 2 +models: + - name: test_silver__blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EXTRA_DATA + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SHA3_UNCLES + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql new file mode 100644 index 0000000..c27a8a9 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.yml b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml new file mode 100644 index 0000000..ceb767f --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: test_silver__blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + \ No newline at end of file diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.sql b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.sql new file mode 100644 index 0000000..6d93a21 --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__confirmed_blocks') }} diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.yml b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.yml new file mode 100644 index 0000000..81a9db0 --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: test_silver__confirmed_blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ \ No newline at end of file diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql new file mode 100644 index 0000000..0808f4a --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__confirmed_blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.yml b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.yml new file mode 100644 index 0000000..460ddf1 --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: test_silver__confirmed_blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.sql b/models/silver/core/tests/event_logs/test_silver__logs_full.sql new file mode 100644 index 0000000..e117298 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__logs') }} diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.yml b/models/silver/core/tests/event_logs/test_silver__logs_full.yml new file mode 100644 index 0000000..4b5140c --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.yml @@ -0,0 +1,77 @@ +version: 2 +models: + - name: test_silver__logs_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - fsc_utils.tx_block_count: + config: + severity: error + error_if: "!=0" + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.sql b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql new file mode 100644 index 0000000..faa40cf --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__logs') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.yml b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml new file mode 100644 index 0000000..4a2b8a6 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: test_silver__logs_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.sql b/models/silver/core/tests/receipts/test_silver__receipts_full.sql new file mode 100644 index 0000000..56f39ae --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__receipts') }} diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.yml b/models/silver/core/tests/receipts/test_silver__receipts_full.yml new file mode 100644 index 0000000..9a18685 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.yml @@ -0,0 +1,82 @@ +version: 2 +models: + - name: test_silver__receipts_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + - name: TYPE + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + + diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.sql b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql new file mode 100644 index 0000000..0b5a901 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__receipts') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.yml b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml new file mode 100644 index 0000000..1e8cfa1 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: test_silver__receipts_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + + diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql new file mode 100644 index 0000000..be52e71 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__traces') }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml new file mode 100644 index 0000000..639f77b --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.yml @@ -0,0 +1,59 @@ +version: 2 +models: + - name: test_silver__traces_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: IDENTIFIER + tests: + - not_null + - name: ETH_VALUE + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + + diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql new file mode 100644 index 0000000..7c3aae3 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__traces') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml new file mode 100644 index 0000000..51db5b0 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: test_silver__traces_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.sql b/models/silver/core/tests/transactions/test_silver__transactions_full.sql new file mode 100644 index 0000000..df58d17 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__transactions') }} diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.yml b/models/silver/core/tests/transactions/test_silver__transactions_full.yml new file mode 100644 index 0000000..fc88627 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.yml @@ -0,0 +1,119 @@ +version: 2 +models: + - name: test_silver__transactions_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: VALUE + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS + tests: + - not_null + - name: INPUT_DATA + tests: + - not_null + - name: TX_STATUS + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + where: NOT IS_PENDING + - name: GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CUMULATIVE_GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_FEE + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + + diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql new file mode 100644 index 0000000..a7b0802 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__transactions') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.yml b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml new file mode 100644 index 0000000..3539ac2 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml @@ -0,0 +1,22 @@ +version: 2 +models: + - name: test_silver__transactions_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + \ No newline at end of file diff --git a/models/silver/base_goerli/methods/silver_goerli__blocks_method.sql b/models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.sql similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__blocks_method.sql rename to models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.sql diff --git a/models/silver/base_goerli/methods/silver_goerli__blocks_method.yml b/models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.yml similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__blocks_method.yml rename to models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.yml diff --git a/models/silver/base_goerli/methods/silver_goerli__receipts_method.sql b/models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.sql similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__receipts_method.sql rename to models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.sql diff --git a/models/silver/base_goerli/methods/silver_goerli__receipts_method.yml b/models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.yml similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__receipts_method.yml rename to models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.yml diff --git a/models/silver/base_goerli/methods/silver_goerli__traces_method.sql b/models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.sql similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__traces_method.sql rename to models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.sql diff --git a/models/silver/base_goerli/methods/silver_goerli__traces_method.yml b/models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.yml similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__traces_method.yml rename to models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.yml diff --git a/models/silver/base_goerli/methods/silver_goerli__tx_method.sql b/models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.sql similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__tx_method.sql rename to models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.sql diff --git a/models/silver/base_goerli/methods/silver_goerli__tx_method.yml b/models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.yml similarity index 100% rename from models/silver/base_goerli/methods/silver_goerli__tx_method.yml rename to models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.yml diff --git a/models/silver/base_goerli/silver_goerli__blocks.sql b/models/silver/goerli/base_goerli/silver_goerli__blocks.sql similarity index 100% rename from models/silver/base_goerli/silver_goerli__blocks.sql rename to models/silver/goerli/base_goerli/silver_goerli__blocks.sql diff --git a/models/silver/base_goerli/silver_goerli__blocks.yml b/models/silver/goerli/base_goerli/silver_goerli__blocks.yml similarity index 100% rename from models/silver/base_goerli/silver_goerli__blocks.yml rename to models/silver/goerli/base_goerli/silver_goerli__blocks.yml diff --git a/models/silver/base_goerli/silver_goerli__logs.sql b/models/silver/goerli/base_goerli/silver_goerli__logs.sql similarity index 100% rename from models/silver/base_goerli/silver_goerli__logs.sql rename to models/silver/goerli/base_goerli/silver_goerli__logs.sql diff --git a/models/silver/base_goerli/silver_goerli__logs.yml b/models/silver/goerli/base_goerli/silver_goerli__logs.yml similarity index 100% rename from models/silver/base_goerli/silver_goerli__logs.yml rename to models/silver/goerli/base_goerli/silver_goerli__logs.yml diff --git a/models/silver/base_goerli/silver_goerli__traces.sql b/models/silver/goerli/base_goerli/silver_goerli__traces.sql similarity index 100% rename from models/silver/base_goerli/silver_goerli__traces.sql rename to models/silver/goerli/base_goerli/silver_goerli__traces.sql diff --git a/models/silver/base_goerli/silver_goerli__traces.yml b/models/silver/goerli/base_goerli/silver_goerli__traces.yml similarity index 100% rename from models/silver/base_goerli/silver_goerli__traces.yml rename to models/silver/goerli/base_goerli/silver_goerli__traces.yml diff --git a/models/silver/base_goerli/silver_goerli__transactions.sql b/models/silver/goerli/base_goerli/silver_goerli__transactions.sql similarity index 100% rename from models/silver/base_goerli/silver_goerli__transactions.sql rename to models/silver/goerli/base_goerli/silver_goerli__transactions.sql diff --git a/models/silver/base_goerli/silver_goerli__transactions.yml b/models/silver/goerli/base_goerli/silver_goerli__transactions.yml similarity index 100% rename from models/silver/base_goerli/silver_goerli__transactions.yml rename to models/silver/goerli/base_goerli/silver_goerli__transactions.yml diff --git a/models/silver/eth_goerli/silver__eth_goerli_block_ranges.sql b/models/silver/goerli/eth_goerli/silver__eth_goerli_block_ranges.sql similarity index 100% rename from models/silver/eth_goerli/silver__eth_goerli_block_ranges.sql rename to models/silver/goerli/eth_goerli/silver__eth_goerli_block_ranges.sql diff --git a/models/silver/eth_goerli/silver__eth_goerli_contract_reads.sql b/models/silver/goerli/eth_goerli/silver__eth_goerli_contract_reads.sql similarity index 100% rename from models/silver/eth_goerli/silver__eth_goerli_contract_reads.sql rename to models/silver/goerli/eth_goerli/silver__eth_goerli_contract_reads.sql diff --git a/models/silver/eth_goerli/silver__eth_goerli_contracts.sql b/models/silver/goerli/eth_goerli/silver__eth_goerli_contracts.sql similarity index 100% rename from models/silver/eth_goerli/silver__eth_goerli_contracts.sql rename to models/silver/goerli/eth_goerli/silver__eth_goerli_contracts.sql diff --git a/models/silver/eth_goerli/silver__eth_goerli_logs.sql b/models/silver/goerli/eth_goerli/silver__eth_goerli_logs.sql similarity index 100% rename from models/silver/eth_goerli/silver__eth_goerli_logs.sql rename to models/silver/goerli/eth_goerli/silver__eth_goerli_logs.sql diff --git a/models/silver/eth_goerli/silver__eth_goerli_logs.yml b/models/silver/goerli/eth_goerli/silver__eth_goerli_logs.yml similarity index 100% rename from models/silver/eth_goerli/silver__eth_goerli_logs.yml rename to models/silver/goerli/eth_goerli/silver__eth_goerli_logs.yml diff --git a/models/silver/streamline/_max_block_by_date.sql b/models/silver/streamline/_max_block_by_date.sql new file mode 100644 index 0000000..a56cf82 --- /dev/null +++ b/models/silver/streamline/_max_block_by_date.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "ephemeral", + unique_key = "block_number", +) }} + +WITH base AS ( + + SELECT + block_timestamp :: DATE AS block_date, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + GROUP BY + block_timestamp :: DATE +) +SELECT + block_date, + block_number +FROM + base +WHERE + block_date <> ( + SELECT + MAX(block_date) + FROM + base + ) \ No newline at end of file diff --git a/models/silver/streamline/_max_block_by_hour.sql b/models/silver/streamline/_max_block_by_hour.sql new file mode 100644 index 0000000..80e3d97 --- /dev/null +++ b/models/silver/streamline/_max_block_by_hour.sql @@ -0,0 +1,37 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH base AS ( + + SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_hour, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + WHERE + block_timestamp > DATEADD( + 'day', + -5, + CURRENT_DATE + ) + GROUP BY + 1 +) +SELECT + block_hour, + block_number +FROM + base +WHERE + block_hour <> ( + SELECT + MAX( + block_hour + ) + FROM + base + ) diff --git a/models/silver/streamline/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql b/models/silver/streamline/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql index 56f3834..a6630e5 100644 --- a/models/silver/streamline/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql +++ b/models/silver/streamline/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql @@ -6,8 +6,8 @@ ) ) }} +WITH blocks AS ( -with blocks AS ( SELECT block_number FROM @@ -17,6 +17,27 @@ with blocks AS ( block_number FROM {{ ref("streamline__complete_debug_traceBlockByNumber") }} +), +all_blocks AS ( + SELECT + block_number + FROM + blocks + UNION + SELECT + block_number + FROM + ( + SELECT + block_number + FROM + {{ ref("_missing_traces") }} + UNION + SELECT + block_number + FROM + {{ ref("_unconfirmed_blocks") }} + ) ) SELECT PARSE_JSON( @@ -42,6 +63,6 @@ SELECT ) ) AS request FROM - blocks + all_blocks ORDER BY block_number ASC diff --git a/models/silver/streamline/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql b/models/silver/streamline/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql index 7983147..be21884 100644 --- a/models/silver/streamline/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql +++ b/models/silver/streamline/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql @@ -6,7 +6,8 @@ ) ) }} -with blocks AS ( +WITH blocks AS ( + SELECT block_number :: STRING AS block_number FROM @@ -16,6 +17,32 @@ with blocks AS ( block_number :: STRING FROM {{ ref("streamline__complete_qn_getBlockWithReceipts") }} +), +all_blocks AS ( + SELECT + block_number + FROM + blocks + UNION + SELECT + block_number + FROM + ( + SELECT + block_number + FROM + {{ ref("_missing_receipts") }} + UNION + SELECT + block_number + FROM + {{ ref("_missing_txs") }} + UNION + SELECT + block_number + FROM + {{ ref("_unconfirmed_blocks") }} + ) ) SELECT PARSE_JSON( @@ -40,6 +67,6 @@ SELECT ) ) AS request FROM - blocks + all_blocks ORDER BY block_number ASC diff --git a/models/silver/streamline/core/retry/_missing_receipts.sql b/models/silver/streamline/core/retry/_missing_receipts.sql new file mode 100644 index 0000000..c7ab41c --- /dev/null +++ b/models/silver/streamline/core/retry/_missing_receipts.sql @@ -0,0 +1,32 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + MAX(block_number) AS block_number + FROM + {{ ref("silver__blocks") }} + WHERE + block_timestamp :: DATE = CURRENT_DATE() - 3 +) +SELECT + DISTINCT t.block_number AS block_number +FROM + {{ ref("silver__transactions") }} + t + LEFT JOIN {{ ref("silver__receipts") }} + r USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + r.tx_hash IS NULL + AND t.block_number >= ( + SELECT + block_number + FROM + lookback + ) diff --git a/models/silver/streamline/core/retry/_missing_traces.sql b/models/silver/streamline/core/retry/_missing_traces.sql new file mode 100644 index 0000000..0830b89 --- /dev/null +++ b/models/silver/streamline/core/retry/_missing_traces.sql @@ -0,0 +1,25 @@ +{{ config ( + materialized = "ephemeral" +) }} + +SELECT + DISTINCT tx.block_number block_number +FROM + {{ ref("silver__transactions") }} + tx + LEFT JOIN {{ ref("silver__traces") }} + tr + ON tx.block_number = tr.block_number + AND tx.tx_hash = tr.tx_hash + AND tr.block_timestamp >= DATEADD( + 'day', + -2, + CURRENT_DATE + ) +WHERE + tx.block_timestamp >= DATEADD( + 'day', + -2, + CURRENT_DATE + ) + AND tr.tx_hash IS NULL diff --git a/models/silver/streamline/core/retry/_missing_txs.sql b/models/silver/streamline/core/retry/_missing_txs.sql new file mode 100644 index 0000000..9d1fecb --- /dev/null +++ b/models/silver/streamline/core/retry/_missing_txs.sql @@ -0,0 +1,32 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH transactions AS ( + + SELECT + block_number, + POSITION, + LAG( + POSITION, + 1 + ) over ( + PARTITION BY block_number + ORDER BY + POSITION ASC + ) AS prev_POSITION + FROM + {{ ref("silver__transactions") }} + WHERE + block_timestamp >= DATEADD( + 'day', + -2, + CURRENT_DATE + ) +) +SELECT + DISTINCT block_number AS block_number +FROM + transactions +WHERE + POSITION - prev_POSITION <> 1 diff --git a/models/silver/streamline/core/retry/_unconfirmed_blocks.sql b/models/silver/streamline/core/retry/_unconfirmed_blocks.sql new file mode 100644 index 0000000..10e9ecc --- /dev/null +++ b/models/silver/streamline/core/retry/_unconfirmed_blocks.sql @@ -0,0 +1,32 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + MAX(block_number) AS block_number + FROM + {{ ref("silver__blocks") }} + WHERE + block_timestamp :: DATE = CURRENT_DATE() - 3 +) +SELECT + DISTINCT cb.block_number AS block_number +FROM + {{ ref("silver__confirmed_blocks") }} + cb + LEFT JOIN {{ ref("silver__transactions") }} + txs USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + txs.tx_hash IS NULL + AND cb.block_number >= ( + SELECT + block_number + FROM + lookback + ) diff --git a/models/sources.yml b/models/sources.yml index 9544b03..f9275be 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -26,3 +26,8 @@ sources: schema: streamline tables: - name: node_mapping + - name: silver_crosschain + database: crosschain + schema: silver + tables: + - name: number_sequence diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/base/test_silver__confirmed_blocks.sql b/tests/base/test_silver__confirmed_blocks.sql new file mode 100644 index 0000000..8cbac8b --- /dev/null +++ b/tests/base/test_silver__confirmed_blocks.sql @@ -0,0 +1 @@ +{{ missing_confirmed_txs(ref("test_silver__confirmed_blocks_full"), ref("test_silver__transactions_full")) }} diff --git a/tests/base/test_silver__missing_receipts.sql b/tests/base/test_silver__missing_receipts.sql new file mode 100644 index 0000000..8481599 --- /dev/null +++ b/tests/base/test_silver__missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__receipts_full")) }} diff --git a/tests/base/test_silver__missing_traces.sql b/tests/base/test_silver__missing_traces.sql new file mode 100644 index 0000000..c96256f --- /dev/null +++ b/tests/base/test_silver__missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__traces_full")) }} diff --git a/tests/base/test_silver__recent_confirmed_blocks.sql b/tests/base/test_silver__recent_confirmed_blocks.sql new file mode 100644 index 0000000..79cdd66 --- /dev/null +++ b/tests/base/test_silver__recent_confirmed_blocks.sql @@ -0,0 +1 @@ +{{ missing_confirmed_txs(ref("test_silver__confirmed_blocks_recent"), ref("test_silver__transactions_recent")) }} diff --git a/tests/base/test_silver__recent_missing_receipts.sql b/tests/base/test_silver__recent_missing_receipts.sql new file mode 100644 index 0000000..f657fd3 --- /dev/null +++ b/tests/base/test_silver__recent_missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__receipts_recent")) }} diff --git a/tests/base/test_silver__recent_missing_traces.sql b/tests/base/test_silver__recent_missing_traces.sql new file mode 100644 index 0000000..26a69d4 --- /dev/null +++ b/tests/base/test_silver__recent_missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__traces_recent")) }}