diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index a9943e9..a7fd203 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -14,23 +14,23 @@ SELECT A.block_timestamp, A.version, A.tx_hash, - A.data :success :: BOOLEAN AS success, + A.success, A.tx_type, - A.data :sender :: STRING AS sender, - A.data :signature :: STRING AS signature, - A.data :payload AS payload, - A.data :payload :function :: STRING AS payload_function, - A.data :changes AS changes, - A.data :events AS events, - A.data: gas_unit_price :: bigint AS gas_unit_price, - A.data :gas_used :: INT AS gas_used, - A.data :max_gas_amount :: bigint AS max_gas_amount, - A.data :expiration_timestamp_secs :: bigint AS expiration_timestamp_secs, - A.data :vm_status :: STRING AS vm_status, - A.data :state_change_hash :: STRING AS state_change_hash, - A.data :accumulator_root_hash :: STRING AS accumulator_root_hash, - A.data :event_root_hash :: STRING AS event_root_hash, - A.data :state_checkpoint_hash :: STRING AS state_checkpoint_hash, + A.sender, + A.signature, + A.payload, + A.payload_function, + A.changes, + A.events, + A.gas_unit_price, + A.gas_used, + A.max_gas_amount, + A.expiration_timestamp_secs, + A.vm_status, + A.state_change_hash, + A.accumulator_root_hash, + A.event_root_hash, + A.state_checkpoint_hash, {{ dbt_utils.generate_surrogate_key( ['tx_hash'] ) }} AS fact_transactions_id, diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.yml b/models/gold/stats/stats__ez_core_metrics_hourly.yml index d65016a..a776a48 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.yml +++ b/models/gold/stats/stats__ez_core_metrics_hourly.yml @@ -2,33 +2,94 @@ version: 2 models: - name: stats__ez_core_metrics_hourly description: '{{ doc("ez_core_metrics_hourly_table_doc") }}' + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP_HOUR + - dbt_utils.expression_is_true: + expression: "block_number_max >= block_number_min" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "transaction_count = transaction_count_success + transaction_count_failed" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "block_count <= transaction_count" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "total_fees_native >= 0" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "total_fees_usd >= 0" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} columns: - name: BLOCK_TIMESTAMP_HOUR description: '{{ doc("block_timestamp_hour") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ - name: BLOCK_NUMBER_MIN description: '{{ doc("block_number_min") }}' + tests: + - not_null - name: BLOCK_NUMBER_MAX description: '{{ doc("block_number_max") }}' + tests: + - not_null - name: BLOCK_COUNT description: '{{ doc("block_count") }}' + tests: + - not_null - name: TRANSACTION_COUNT description: '{{ doc("transaction_count") }}' + tests: + - not_null - name: TRANSACTION_COUNT_SUCCESS description: '{{ doc("transaction_count_success") }}' + tests: + - not_null - name: TRANSACTION_COUNT_FAILED description: '{{ doc("transaction_count_failed") }}' + tests: + - not_null - name: UNIQUE_SENDER_COUNT description: '{{ doc("unique_sender_count") }}' + tests: + - not_null - name: UNIQUE_PAYLOAD_FUNCTION_COUNT description: '{{ doc("unique_payload_function_count") }}' + tests: + - not_null - name: TOTAL_FEES_NATIVE description: '{{ doc("total_fees_native") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DECIMAL(38,8) - name: TOTAL_FEES_USD description: '{{ doc("total_fees_usd") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DECIMAL(38,8) - name: EZ_CORE_METRICS_HOURLY_ID description: '{{ doc("pk") }}' + tests: + - not_null + - unique - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' \ No newline at end of file + description: '{{ doc("modified_timestamp") }}' + tests: + - not_null \ No newline at end of file diff --git a/models/silver/core/silver__blocks.yml b/models/silver/core/silver__blocks.yml index a4a4bcc..343c8c7 100644 --- a/models/silver/core/silver__blocks.yml +++ b/models/silver/core/silver__blocks.yml @@ -1,39 +1,25 @@ version: 2 models: - name: silver__blocks - tests: - - dbt_constraints.primary_key: - column_name: BLOCKS_ID - + - sequence_gaps: + column: block_number columns: - name: block_number tests: - - not_null - - name: BLOCK_TIMESTAMP_NUM + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - name: block_hash + - name: block_timestamp_num + - name: block_timestamp tests: - - not_null - - name: BLOCK_TIMESTAMP - tests: - - not_null - - name: BLOCK_HASH - tests: - - not_null - - name: FIRST_VERSION - tests: - - not_null - - name: LAST_VERSION - tests: - - not_null - - name: TX_COUNT_FROM_TRANSACTIONS_ARRAY - tests: - - not_null - - name: TX_COUNT_FROM_VERSIONS - tests: - - not_null - - name: BLOCKS_ID - tests: - - not_null - - name: INSERTED_TIMESTAMP - - name: MODIFIED_TIMESTAMP - - name: _INVOCATION_ID + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - name: first_version + - name: last_version + - name: tx_count_from_transactions_array + - name: tx_count_from_versions + - name: blocks_id + - name: inserted_timestamp + - name: modified_timestamp + - name: _invocation_id diff --git a/models/silver/core/silver__changes.yml b/models/silver/core/silver__changes.yml index 8746564..37d0ad4 100644 --- a/models/silver/core/silver__changes.yml +++ b/models/silver/core/silver__changes.yml @@ -1,26 +1,34 @@ version: 2 models: - name: silver__changes - + tests: + - dbt_utils.expression_is_true: + expression: "b.value:type IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "b.value:data: 'type' IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} columns: - name: block_number - - name: BLOCK_TIMESTAMP - - name: TX_HASH - - name: VERSION - - name: SUCCESS - - name: TX_TYPE - - name: CHANGE_INDEX - - name: CHANGE_DATA - - name: CHANGE_TYPE - - name: ADDRESS - - name: HANDLE - - name: INNER_CHANGE_TYPE - - name: CHANGE_ADDRESS - - name: CHANGE_MODULE - - name: CHANGE_RESOURCE - - name: KEY - - name: VALUE - - name: STATE_KEY_HASH - - name: CHANGES_ID - - name: INSERTED_TIMESTAMP - - name: MODIFIED_TIMESTAMP \ No newline at end of file + - name: block_timestamp + - name: tx_hash + - name: version + - name: success + - name: tx_type + - name: payload_function + - name: change_index + - name: change_data + - name: change_type + - name: address + - name: handle + - name: inner_change_type + - name: change_address + - name: change_module + - name: change_resource + - name: key + - name: value + - name: state_key_hash + - name: changes_id + - name: inserted_timestamp + - name: modified_timestamp + - name: _invocation_id \ No newline at end of file diff --git a/models/silver/core/silver__events.yml b/models/silver/core/silver__events.yml index b7e53bc..df855a7 100644 --- a/models/silver/core/silver__events.yml +++ b/models/silver/core/silver__events.yml @@ -1,23 +1,31 @@ version: 2 models: - name: silver__events - + tests: + - dbt_utils.expression_is_true: + expression: "b.value:type IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "b.value:guid:account_address IS NOT NULL AND b.value:guid:creation_number IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} columns: - name: block_number - - name: BLOCK_TIMESTAMP - - name: TX_HASH - - name: VERSION - - name: SUCCESS - - name: TX_TYPE - - name: EVENT_INDEX - - name: EVENT_TYPE - - name: EVENT_ADDRESS - - name: EVENT_MODULE - - name: EVENT_RESOURCE - - name: EVENT_DATA - - name: ACCOUNT_ADDRESS - - name: CREATION_NUMBER - - name: SEQUENCE_NUMBER - - name: EVENTS_ID - - name: INSERTED_TIMESTAMP - - name: MODIFIED_TIMESTAMP + - name: block_timestamp + - name: tx_hash + - name: version + - name: success + - name: tx_type + - name: payload_function + - name: event_index + - name: event_type + - name: event_address + - name: event_module + - name: event_resource + - name: event_data + - name: account_address + - name: creation_number + - name: sequence_number + - name: events_id + - name: inserted_timestamp + - name: modified_timestamp + - name: _invocation_id diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 26646aa..e2b6368 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -86,15 +86,58 @@ combo AS ( file_last_updated FROM from_transactions A +), +transformed AS ( + SELECT + COALESCE( + block_timestamp, + '1970-01-01 00:00:00.000' + ) AS block_timestamp, + tx_hash, + version, + tx_type, + DATA :success :: BOOLEAN AS success, + DATA :sender :: STRING AS sender, + DATA :signature :: STRING AS signature, + DATA :payload AS payload, + DATA :payload :function :: STRING AS payload_function, + DATA :changes AS changes, + DATA :events AS events, + DATA :gas_unit_price :: bigint AS gas_unit_price, + DATA :gas_used :: INT AS gas_used, + DATA :max_gas_amount :: bigint AS max_gas_amount, + DATA :expiration_timestamp_secs :: bigint AS expiration_timestamp_secs, + DATA :vm_status :: STRING AS vm_status, + DATA :state_change_hash :: STRING AS state_change_hash, + DATA :accumulator_root_hash :: STRING AS accumulator_root_hash, + DATA :event_root_hash :: STRING AS event_root_hash, + DATA :state_checkpoint_hash :: STRING AS state_checkpoint_hash, + DATA, + file_last_updated + FROM + combo ) SELECT - COALESCE( - block_timestamp, - '1970-01-01 00:00:00.000' - ) AS block_timestamp, + block_timestamp, tx_hash, version, tx_type, + success, + sender, + signature, + payload, + payload_function, + changes, + events, + gas_unit_price, + gas_used, + max_gas_amount, + expiration_timestamp_secs, + vm_status, + state_change_hash, + accumulator_root_hash, + event_root_hash, + state_checkpoint_hash, DATA, {{ dbt_utils.generate_surrogate_key( ['tx_hash'] @@ -103,6 +146,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - combo qualify(ROW_NUMBER() over (PARTITION BY tx_hash + transformed qualify(ROW_NUMBER() over (PARTITION BY tx_hash ORDER BY file_last_updated DESC)) = 1 diff --git a/models/silver/core/silver__transactions.yml b/models/silver/core/silver__transactions.yml index 48c448d..1b080a0 100644 --- a/models/silver/core/silver__transactions.yml +++ b/models/silver/core/silver__transactions.yml @@ -1,37 +1,47 @@ version: 2 models: - name: silver__transactions - tests: - - dbt_constraints.primary_key: - column_name: TRANSACTIONS_ID - + - dbt_utils.expression_is_true: + expression: "b.value:timestamp IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "b.value:hash IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "b.value:version IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "b.value:type IS NOT NULL" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} columns: - - name: BLOCK_TIMESTAMP + - name: block_timestamp + - name: tx_hash tests: - - not_null - - name: TX_HASH - tests: - - not_null - - name: VERSION - tests: - - not_null - - name: TX_TYPE - tests: - - not_null - - name: TRANSACTIONS_ID - tests: - - not_null - - name: INSERTED_TIMESTAMP - tests: - - not_null - - name: MODIFIED_TIMESTAMP - tests: - - not_null - - - name: _INVOCATION_ID - tests: - - name: not_null_silver__transactions_INVOCATION_ID - test_name: not_null + - unique: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - name: version + - name: tx_type + - name: success + - name: sender + - name: signature + - name: payload + - name: payload_function + - name: changes + - name: events + - name: gas_unit_price + - name: gas_used + - name: max_gas_amount + - name: expiration_timestamp_secs + - name: vm_status + - name: state_change_hash + - name: accumulator_root_hash + - name: event_root_hash + - name: state_checkpoint_hash + - name: data + - name: transactions_id + - name: inserted_timestamp + - name: modified_timestamp + - name: _invocation_id \ No newline at end of file diff --git a/models/silver/stats/silver__core_metrics_hourly.yml b/models/silver/stats/silver__core_metrics_hourly.yml index 2d89660..992743e 100644 --- a/models/silver/stats/silver__core_metrics_hourly.yml +++ b/models/silver/stats/silver__core_metrics_hourly.yml @@ -5,64 +5,47 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - BLOCK_TIMESTAMP_HOUR + - dbt_utils.expression_is_true: + expression: "block_number_max >= block_number_min" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "transaction_count = transaction_count_success + transaction_count_failed" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "block_count <= transaction_count" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_utils.expression_is_true: + expression: "total_fees >= 0" + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + columns: - name: BLOCK_TIMESTAMP_HOUR tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ - name: block_number_MIN tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - name: block_number_MAX tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - name: BLOCK_COUNT tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - name: TRANSACTION_COUNT tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - name: TRANSACTION_COUNT_SUCCESS tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - name: TRANSACTION_COUNT_FAILED tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - name: TOTAL_FEES tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - DECIMAL - - FLOAT - - NUMBER - name: _INSERTED_TIMESTAMP tests: - dbt_expectations.expect_row_values_to_have_recent_data: