flip/movement-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/movement-models.git synced 2026-02-06 11:46:43 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

updating testing and refactoring transactions

Browse Source
This commit is contained in:
Mike Stepanovic 2025-03-04 12:42:17 -07:00
parent 05585ff7dd
commit 6fddc417d2
8 changed files with 250 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
models/gold/core/core__fact_transactions.sql
View File

@ -14,23 +14,23 @@ SELECT
A.block_timestamp,
A.version,
A.tx_hash,
A.data :success :: BOOLEAN AS success,
A.success,
A.tx_type,
A.data :sender :: STRING AS sender,
A.data :signature :: STRING AS signature,
A.data :payload AS payload,
A.data :payload :function :: STRING AS payload_function,
A.data :changes AS changes,
A.data :events AS events,
A.data: gas_unit_price :: bigint AS gas_unit_price,
A.data :gas_used :: INT AS gas_used,
A.data :max_gas_amount :: bigint AS max_gas_amount,
A.data :expiration_timestamp_secs :: bigint AS expiration_timestamp_secs,
A.data :vm_status :: STRING AS vm_status,
A.data :state_change_hash :: STRING AS state_change_hash,
A.data :accumulator_root_hash :: STRING AS accumulator_root_hash,
A.data :event_root_hash :: STRING AS event_root_hash,
A.data :state_checkpoint_hash :: STRING AS state_checkpoint_hash,
A.sender,
A.signature,
A.payload,
A.payload_function,
A.changes,
A.events,
A.gas_unit_price,
A.gas_used,
A.max_gas_amount,
A.expiration_timestamp_secs,
A.vm_status,
A.state_change_hash,
A.accumulator_root_hash,
A.event_root_hash,
A.state_checkpoint_hash,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS fact_transactions_id,

61
models/gold/stats/stats__ez_core_metrics_hourly.yml
View File

@ -2,33 +2,94 @@ version: 2
models:
- name: stats__ez_core_metrics_hourly
description: '{{ doc("ez_core_metrics_hourly_table_doc") }}'
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- BLOCK_TIMESTAMP_HOUR
- dbt_utils.expression_is_true:
expression: "block_number_max >= block_number_min"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "transaction_count = transaction_count_success + transaction_count_failed"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "block_count <= transaction_count"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "total_fees_native >= 0"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "total_fees_usd >= 0"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
columns:
- name: BLOCK_TIMESTAMP_HOUR
description: '{{ doc("block_timestamp_hour") }}'
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: BLOCK_NUMBER_MIN
description: '{{ doc("block_number_min") }}'
tests:
- not_null
- name: BLOCK_NUMBER_MAX
description: '{{ doc("block_number_max") }}'
tests:
- not_null
- name: BLOCK_COUNT
description: '{{ doc("block_count") }}'
tests:
- not_null
- name: TRANSACTION_COUNT
description: '{{ doc("transaction_count") }}'
tests:
- not_null
- name: TRANSACTION_COUNT_SUCCESS
description: '{{ doc("transaction_count_success") }}'
tests:
- not_null
- name: TRANSACTION_COUNT_FAILED
description: '{{ doc("transaction_count_failed") }}'
tests:
- not_null
- name: UNIQUE_SENDER_COUNT
description: '{{ doc("unique_sender_count") }}'
tests:
- not_null
- name: UNIQUE_PAYLOAD_FUNCTION_COUNT
description: '{{ doc("unique_payload_function_count") }}'
tests:
- not_null
- name: TOTAL_FEES_NATIVE
description: '{{ doc("total_fees_native") }}'
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- DECIMAL(38,8)
- name: TOTAL_FEES_USD
description: '{{ doc("total_fees_usd") }}'
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- DECIMAL(38,8)
- name: EZ_CORE_METRICS_HOURLY_ID
description: '{{ doc("pk") }}'
tests:
- not_null
- unique
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: day
interval: 1
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'
tests:
- not_null

48
models/silver/core/silver__blocks.yml
View File

@ -1,39 +1,25 @@
version: 2
models:
- name: silver__blocks
tests:
- dbt_constraints.primary_key:
column_name: BLOCKS_ID
- sequence_gaps:
column: block_number
columns:
- name: block_number
tests:
- not_null
- name: BLOCK_TIMESTAMP_NUM
- not_null:
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: block_hash
- name: block_timestamp_num
- name: block_timestamp
tests:
- not_null
- name: BLOCK_TIMESTAMP
tests:
- not_null
- name: BLOCK_HASH
tests:
- not_null
- name: FIRST_VERSION
tests:
- not_null
- name: LAST_VERSION
tests:
- not_null
- name: TX_COUNT_FROM_TRANSACTIONS_ARRAY
tests:
- not_null
- name: TX_COUNT_FROM_VERSIONS
tests:
- not_null
- name: BLOCKS_ID
tests:
- not_null
- name: INSERTED_TIMESTAMP
- name: MODIFIED_TIMESTAMP
- name: _INVOCATION_ID
- not_null:
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: first_version
- name: last_version
- name: tx_count_from_transactions_array
- name: tx_count_from_versions
- name: blocks_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id

50
models/silver/core/silver__changes.yml
View File

@ -1,26 +1,34 @@
version: 2
models:
- name: silver__changes
tests:
- dbt_utils.expression_is_true:
expression: "b.value:type IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "b.value:data: 'type' IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
columns:
- name: block_number
- name: BLOCK_TIMESTAMP
- name: TX_HASH
- name: VERSION
- name: SUCCESS
- name: TX_TYPE
- name: CHANGE_INDEX
- name: CHANGE_DATA
- name: CHANGE_TYPE
- name: ADDRESS
- name: HANDLE
- name: INNER_CHANGE_TYPE
- name: CHANGE_ADDRESS
- name: CHANGE_MODULE
- name: CHANGE_RESOURCE
- name: KEY
- name: VALUE
- name: STATE_KEY_HASH
- name: CHANGES_ID
- name: INSERTED_TIMESTAMP
- name: MODIFIED_TIMESTAMP
- name: block_timestamp
- name: tx_hash
- name: version
- name: success
- name: tx_type
- name: payload_function
- name: change_index
- name: change_data
- name: change_type
- name: address
- name: handle
- name: inner_change_type
- name: change_address
- name: change_module
- name: change_resource
- name: key
- name: value
- name: state_key_hash
- name: changes_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id

44
models/silver/core/silver__events.yml
View File

@ -1,23 +1,31 @@
version: 2
models:
- name: silver__events
tests:
- dbt_utils.expression_is_true:
expression: "b.value:type IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "b.value:guid:account_address IS NOT NULL AND b.value:guid:creation_number IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
columns:
- name: block_number
- name: BLOCK_TIMESTAMP
- name: TX_HASH
- name: VERSION
- name: SUCCESS
- name: TX_TYPE
- name: EVENT_INDEX
- name: EVENT_TYPE
- name: EVENT_ADDRESS
- name: EVENT_MODULE
- name: EVENT_RESOURCE
- name: EVENT_DATA
- name: ACCOUNT_ADDRESS
- name: CREATION_NUMBER
- name: SEQUENCE_NUMBER
- name: EVENTS_ID
- name: INSERTED_TIMESTAMP
- name: MODIFIED_TIMESTAMP
- name: block_timestamp
- name: tx_hash
- name: version
- name: success
- name: tx_type
- name: payload_function
- name: event_index
- name: event_type
- name: event_address
- name: event_module
- name: event_resource
- name: event_data
- name: account_address
- name: creation_number
- name: sequence_number
- name: events_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id

49
models/silver/core/silver__transactions.sql
View File

@ -86,8 +86,9 @@ combo AS (
file_last_updated
FROM
from_transactions A
)
SELECT
),
transformed AS (
SELECT
COALESCE(
block_timestamp,
'1970-01-01 00:00:00.000'
@ -95,6 +96,48 @@ SELECT
tx_hash,
version,
tx_type,
DATA :success :: BOOLEAN AS success,
DATA :sender :: STRING AS sender,
DATA :signature :: STRING AS signature,
DATA :payload AS payload,
DATA :payload :function :: STRING AS payload_function,
DATA :changes AS changes,
DATA :events AS events,
DATA :gas_unit_price :: bigint AS gas_unit_price,
DATA :gas_used :: INT AS gas_used,
DATA :max_gas_amount :: bigint AS max_gas_amount,
DATA :expiration_timestamp_secs :: bigint AS expiration_timestamp_secs,
DATA :vm_status :: STRING AS vm_status,
DATA :state_change_hash :: STRING AS state_change_hash,
DATA :accumulator_root_hash :: STRING AS accumulator_root_hash,
DATA :event_root_hash :: STRING AS event_root_hash,
DATA :state_checkpoint_hash :: STRING AS state_checkpoint_hash,
DATA,
file_last_updated
FROM
combo
)
SELECT
block_timestamp,
tx_hash,
version,
tx_type,
success,
sender,
signature,
payload,
payload_function,
changes,
events,
gas_unit_price,
gas_used,
max_gas_amount,
expiration_timestamp_secs,
vm_status,
state_change_hash,
accumulator_root_hash,
event_root_hash,
state_checkpoint_hash,
DATA,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
@ -103,6 +146,6 @@ SELECT
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
combo qualify(ROW_NUMBER() over (PARTITION BY tx_hash
transformed qualify(ROW_NUMBER() over (PARTITION BY tx_hash
ORDER BY
file_last_updated DESC)) = 1

68
models/silver/core/silver__transactions.yml
View File

@ -1,37 +1,47 @@
version: 2
models:
- name: silver__transactions
tests:
- dbt_constraints.primary_key:
column_name: TRANSACTIONS_ID
- dbt_utils.expression_is_true:
expression: "b.value:timestamp IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "b.value:hash IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "b.value:version IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "b.value:type IS NOT NULL"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
columns:
- name: BLOCK_TIMESTAMP
- name: block_timestamp
- name: tx_hash
tests:
- not_null
- name: TX_HASH
tests:
- not_null
- name: VERSION
tests:
- not_null
- name: TX_TYPE
tests:
- not_null
- name: TRANSACTIONS_ID
tests:
- not_null
- name: INSERTED_TIMESTAMP
tests:
- not_null
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- name: _INVOCATION_ID
tests:
- name: not_null_silver__transactions_INVOCATION_ID
test_name: not_null
- unique:
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: version
- name: tx_type
- name: success
- name: sender
- name: signature
- name: payload
- name: payload_function
- name: changes
- name: events
- name: gas_unit_price
- name: gas_used
- name: max_gas_amount
- name: expiration_timestamp_secs
- name: vm_status
- name: state_change_hash
- name: accumulator_root_hash
- name: event_root_hash
- name: state_checkpoint_hash
- name: data
- name: transactions_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id

43
models/silver/stats/silver__core_metrics_hourly.yml
View File

@ -5,64 +5,47 @@ models:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- BLOCK_TIMESTAMP_HOUR
- dbt_utils.expression_is_true:
expression: "block_number_max >= block_number_min"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "transaction_count = transaction_count_success + transaction_count_failed"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "block_count <= transaction_count"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_utils.expression_is_true:
expression: "total_fees >= 0"
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
columns:
- name: BLOCK_TIMESTAMP_HOUR
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: block_number_MIN
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: block_number_MAX
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_COUNT
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TRANSACTION_COUNT
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TRANSACTION_COUNT_SUCCESS
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TRANSACTION_COUNT_FAILED
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TOTAL_FEES
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- DECIMAL
- FLOAT
- NUMBER
- name: _INSERTED_TIMESTAMP
tests:
- dbt_expectations.expect_row_values_to_have_recent_data: