From fbc56fd4233b7bb49827b86d7b6ff082afe3537f Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Tue, 20 May 2025 10:51:08 -0600 Subject: [PATCH] udpated sources with freshness tests, added minimum S/G tests and docs, removed transfers (TODO), modified msgs schema --- .../core/gold/core/_gold_core_tests.yml | 172 +++++++++++++++ .../core/gold/core/core__fact_transfers.sql | 0 .../core/silver/core/_silver_core_tests.yml | 200 ++++++++++++++++++ .../core/silver/core/silver__msgs.sql | 3 - .../core/silver/core/silver__transfers.sql | 0 models/sources.yml | 9 + 6 files changed, 381 insertions(+), 3 deletions(-) create mode 100644 models/main_package/core/gold/core/_gold_core_tests.yml delete mode 100644 models/main_package/core/gold/core/core__fact_transfers.sql create mode 100644 models/main_package/core/silver/core/_silver_core_tests.yml delete mode 100644 models/main_package/core/silver/core/silver__transfers.sql diff --git a/models/main_package/core/gold/core/_gold_core_tests.yml b/models/main_package/core/gold/core/_gold_core_tests.yml new file mode 100644 index 0000000..e5747c5 --- /dev/null +++ b/models/main_package/core/gold/core/_gold_core_tests.yml @@ -0,0 +1,172 @@ +version: 2 + +models: + - name: core__fact_blocks + description: '{{ doc("core__fact_blocks") }}' + data_tests: + - dbt_utils.recency: + datepart: hour + field: modified_timestamp + interval: 3 + severity: error + tags: ['test_recency'] + columns: + - name: blockchain + description: '{{ doc("blockchain") }}' + - name: block_id + description: '{{ doc("block_id") }}' + - name: block_timestamp + description: '{{ doc("block_timestamp") }}' + - name: chain_id + description: '{{ doc("chain_id") }}' + - name: tx_count + description: '{{ doc("tx_count") }}' + - name: proposer_address + description: '{{ doc("proposer_address") }}' + - name: validator_hash + description: '{{ doc("validator_hash") }}' + - name: fact_blocks_id + description: '{{ doc("pk") }}' + - name: inserted_timestamp + description: '{{ doc("inserted_timestamp") }}' + - name: modified_timestamp + description: '{{ doc("modified_timestamp") }}' + + - name: core__fact_msg_attributes + description: '{{ doc("core__fact_msg_attributes") }}' + data_tests: + - dbt_utils.recency: + datepart: hour + field: modified_timestamp + interval: 3 + severity: error + tags: ['test_recency'] + columns: + - name: block_id + description: '{{ doc("block_id") }}' + - name: block_timestamp + description: '{{ doc("block_timestamp") }}' + - name: tx_id + description: '{{ doc("tx_id") }}' + - name: tx_succeeded + description: '{{ doc("tx_succeeded") }}' + - name: msg_group + description: '{{ doc("msg_group") }}' + - name: msg_index + description: '{{ doc("msg_index") }}' + - name: msg_type + description: '{{ doc("msg_type") }}' + - name: attribute_index + description: '{{ doc("attribute_index") }}' + - name: attribute_key + description: '{{ doc("attribute_key") }}' + - name: attribute_value + description: '{{ doc("attribute_value") }}' + - name: fact_msg_attributes_id + description: '{{ doc("pk") }}' + - name: inserted_timestamp + description: '{{ doc("inserted_timestamp") }}' + - name: modified_timestamp + description: '{{ doc("modified_timestamp") }}' + + - name: core__fact_msgs + description: '{{ doc("core__fact_msgs") }}' + data_tests: + - dbt_utils.recency: + datepart: hour + field: modified_timestamp + interval: 3 + severity: error + tags: ['test_recency'] + columns: + - name: block_id + description: '{{ doc("block_id") }}' + - name: block_timestamp + description: '{{ doc("block_timestamp") }}' + - name: tx_id + description: '{{ doc("tx_id") }}' + - name: tx_succeeded + description: '{{ doc("tx_succeeded") }}' + - name: msg_group + description: '{{ doc("msg_group") }}' + - name: msg_type + description: '{{ doc("msg_type") }}' + - name: msg_index + description: '{{ doc("msg_index") }}' + - name: msg + description: '{{ doc("msg") }}' + - name: fact_msgs_id + description: '{{ doc("pk") }}' + - name: inserted_timestamp + description: '{{ doc("inserted_timestamp") }}' + - name: modified_timestamp + description: '{{ doc("modified_timestamp") }}' + + - name: core__fact_transactions + description: '{{ doc("core__fact_transactions") }}' + data_tests: + - dbt_utils.recency: + datepart: hour + field: modified_timestamp + interval: 3 + severity: error + tags: ['test_recency'] + columns: + - name: block_id + description: '{{ doc("block_id") }}' + - name: block_timestamp + description: '{{ doc("block_timestamp") }}' + - name: codespace + description: '{{ doc("codespace") }}' + - name: tx_id + description: '{{ doc("tx_id") }}' + - name: tx_succeeded + description: '{{ doc("tx_succeeded") }}' + - name: tx_code + description: '{{ doc("tx_code") }}' + - name: tx_log + description: '{{ doc("tx_log") }}' + - name: gas_used + description: '{{ doc("gas_used") }}' + - name: gas_wanted + description: '{{ doc("gas_wanted") }}' + - name: fact_transactions_id + description: '{{ doc("pk") }}' + - name: inserted_timestamp + description: '{{ doc("inserted_timestamp") }}' + - name: modified_timestamp + description: '{{ doc("modified_timestamp") }}' + + - name: core__fact_transactions_logs + description: '{{ doc("core__fact_transactions_logs") }}' + data_tests: + - dbt_utils.recency: + datepart: hour + field: modified_timestamp + interval: 3 + severity: error + tags: ['test_recency'] + columns: + - name: block_id + description: '{{ doc("block_id") }}' + - name: block_timestamp + description: '{{ doc("block_timestamp") }}' + - name: tx_id + description: '{{ doc("tx_id") }}' + - name: tx_succeeded + description: '{{ doc("tx_succeeded") }}' + - name: tx_code + description: '{{ doc("tx_code") }}' + - name: codespace + description: '{{ doc("codespace") }}' + - name: tx_log + description: '{{ doc("tx_log") }}' + - name: transactions_logs_id + description: '{{ doc("pk") }}' + - name: inserted_timestamp + description: '{{ doc("inserted_timestamp") }}' + - name: modified_timestamp + description: '{{ doc("modified_timestamp") }}' + + + \ No newline at end of file diff --git a/models/main_package/core/gold/core/core__fact_transfers.sql b/models/main_package/core/gold/core/core__fact_transfers.sql deleted file mode 100644 index e69de29..0000000 diff --git a/models/main_package/core/silver/core/_silver_core_tests.yml b/models/main_package/core/silver/core/_silver_core_tests.yml new file mode 100644 index 0000000..22ee60b --- /dev/null +++ b/models/main_package/core/silver/core/_silver_core_tests.yml @@ -0,0 +1,200 @@ +version: 2 + +models: + - name: silver__blocks + config: + contract: + enforced: true + data_tests: + - dbt_utils.sequential_values: + column_name: block_id + interval: 1 + config: + error_if: ">100" + tags: ['test_recency'] + columns: + - name: block_id + data_type: NUMBER + data_tests: + - not_null: + tags: ['test_quality'] + - name: block_timestamp + data_type: TIMESTAMP_NTZ + data_tests: + - not_null: + tags: ['test_quality'] + - name: chain_id + data_type: VARCHAR + - name: tx_count + data_type: NUMBER + data_tests: + - dbt_utils.expression_is_true: + expression: ">=0" + tags: ['test_quality'] + - name: proposer_address + data_type: VARCHAR + data_tests: + - not_null: + tags: ['test_quality'] + - name: validator_hash + data_type: VARCHAR + data_tests: + - not_null: + tags: ['test_quality'] + - name: header + data_type: VARIANT + - name: _inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: blocks_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__msg_attributes + columns: + - name: block_id + data_type: NUMBER + - name: block_timestamp + data_type: TIMESTAMP_NTZ + - name: tx_id + data_type: VARCHAR + - name: tx_succeeded + data_type: BOOLEAN + - name: msg_group + data_type: NUMBER + - name: msg_sub_group + data_type: NUMBER + - name: msg_index + data_type: NUMBER + - name: msg_type + data_type: VARCHAR + - name: attribute_index + data_type: NUMBER + - name: attribute_key + data_type: VARCHAR + - name: attribute_value + data_type: VARCHAR + - name: msg_attributes_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__msgs + columns: + - name: block_id + data_type: NUMBER + - name: block_timestamp + data_type: TIMESTAMP_NTZ + - name: tx_id + data_type: VARCHAR + - name: tx_succeeded + data_type: BOOLEAN + - name: msg_group + data_type: NUMBER + - name: msg_sub_group + data_type: NUMBER + - name: msg_index + data_type: NUMBER + - name: msg_type + data_type: VARCHAR + - name: msg + data_type: VARIANT + - name: msgs_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__transactions_logs + columns: + - name: block_id + data_type: NUMBER + - name: block_timestamp + data_type: TIMESTAMP_NTZ + - name: tx_id + data_type: VARCHAR + - name: tx_succeeded + data_type: BOOLEAN + - name: tx_code + data_type: NUMBER + - name: codespace + data_type: VARIANT + - name: tx_log + data_type: VARIANT + - name: transactions_logs_id + data_type: VARCHAR + - name: _inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__transactions + config: + contract: + enforced: true + columns: + - name: block_id + data_type: NUMBER + data_tests: + - not_null: + tags: ['test_quality'] + - name: block_timestamp + data_type: TIMESTAMP_NTZ + data_tests: + - not_null: + tags: ['test_quality'] + - name: codespace + data_type: VARIANT + - name: tx_id + data_type: VARCHAR + data_tests: + - not_null: + tags: ['test_quality'] + - name: tx_index + data_type: NUMBER + - name: tx_log + data_type: VARCHAR + - name: tx_succeeded + data_type: BOOLEAN + - name: gas_used + data_type: NUMBER + - name: gas_wanted + data_type: NUMBER + - name: tx_code + data_type: NUMBER + - name: DATA + data_type: VARIANT + - name: partition_key + data_type: VARCHAR + - name: block_id_requested + data_type: NUMBER + - name: _inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: transactions_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + - name: silver__transfers diff --git a/models/main_package/core/silver/core/silver__msgs.sql b/models/main_package/core/silver/core/silver__msgs.sql index 86ddcce..c67896b 100644 --- a/models/main_package/core/silver/core/silver__msgs.sql +++ b/models/main_package/core/silver/core/silver__msgs.sql @@ -127,8 +127,6 @@ msgs AS ( concat_ws( '-', bronze_msgs.tx_id, - bronze_msgs.msg_index - ) AS unique_key, _inserted_timestamp FROM bronze_msgs @@ -146,7 +144,6 @@ SELECT msg_index, msg_type, msg :: OBJECT AS msg, - unique_key, {{ dbt_utils.generate_surrogate_key( ['tx_id','msg_index'] ) }} AS msgs_id, diff --git a/models/main_package/core/silver/core/silver__transfers.sql b/models/main_package/core/silver/core/silver__transfers.sql deleted file mode 100644 index e69de29..0000000 diff --git a/models/sources.yml b/models/sources.yml index 760a87c..4fcd957 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -11,8 +11,17 @@ sources: schema: streamline tables: - name: blocks + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 4, period: hour} - name: tx_counts + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 4, period: hour} - name: transactions + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 4, period: hour} - name: crosschain_silver {{ 'CROSSCHAIN_DEV' if '_DEV' in target.database.upper() else 'CROSSCHAIN' }} schema: silver