flip/fsc-ibc
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/fsc-ibc.git synced 2026-02-06 10:58:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

udpated sources with freshness tests, added minimum S/G tests and docs, removed transfers (TODO), modified msgs schema

Browse Source
This commit is contained in:
Mike Stepanovic 2025-05-20 10:51:08 -06:00
parent 70e395f113
commit fbc56fd423
6 changed files with 381 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
models/main_package/core/gold/core/_gold_core_tests.yml Normal file
View File

@ -0,0 +1,172 @@
version: 2
models:
- name: core__fact_blocks
description: '{{ doc("core__fact_blocks") }}'
data_tests:
- dbt_utils.recency:
datepart: hour
field: modified_timestamp
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: blockchain
description: '{{ doc("blockchain") }}'
- name: block_id
description: '{{ doc("block_id") }}'
- name: block_timestamp
description: '{{ doc("block_timestamp") }}'
- name: chain_id
description: '{{ doc("chain_id") }}'
- name: tx_count
description: '{{ doc("tx_count") }}'
- name: proposer_address
description: '{{ doc("proposer_address") }}'
- name: validator_hash
description: '{{ doc("validator_hash") }}'
- name: fact_blocks_id
description: '{{ doc("pk") }}'
- name: inserted_timestamp
description: '{{ doc("inserted_timestamp") }}'
- name: modified_timestamp
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_msg_attributes
description: '{{ doc("core__fact_msg_attributes") }}'
data_tests:
- dbt_utils.recency:
datepart: hour
field: modified_timestamp
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_id
description: '{{ doc("block_id") }}'
- name: block_timestamp
description: '{{ doc("block_timestamp") }}'
- name: tx_id
description: '{{ doc("tx_id") }}'
- name: tx_succeeded
description: '{{ doc("tx_succeeded") }}'
- name: msg_group
description: '{{ doc("msg_group") }}'
- name: msg_index
description: '{{ doc("msg_index") }}'
- name: msg_type
description: '{{ doc("msg_type") }}'
- name: attribute_index
description: '{{ doc("attribute_index") }}'
- name: attribute_key
description: '{{ doc("attribute_key") }}'
- name: attribute_value
description: '{{ doc("attribute_value") }}'
- name: fact_msg_attributes_id
description: '{{ doc("pk") }}'
- name: inserted_timestamp
description: '{{ doc("inserted_timestamp") }}'
- name: modified_timestamp
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_msgs
description: '{{ doc("core__fact_msgs") }}'
data_tests:
- dbt_utils.recency:
datepart: hour
field: modified_timestamp
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_id
description: '{{ doc("block_id") }}'
- name: block_timestamp
description: '{{ doc("block_timestamp") }}'
- name: tx_id
description: '{{ doc("tx_id") }}'
- name: tx_succeeded
description: '{{ doc("tx_succeeded") }}'
- name: msg_group
description: '{{ doc("msg_group") }}'
- name: msg_type
description: '{{ doc("msg_type") }}'
- name: msg_index
description: '{{ doc("msg_index") }}'
- name: msg
description: '{{ doc("msg") }}'
- name: fact_msgs_id
description: '{{ doc("pk") }}'
- name: inserted_timestamp
description: '{{ doc("inserted_timestamp") }}'
- name: modified_timestamp
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_transactions
description: '{{ doc("core__fact_transactions") }}'
data_tests:
- dbt_utils.recency:
datepart: hour
field: modified_timestamp
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_id
description: '{{ doc("block_id") }}'
- name: block_timestamp
description: '{{ doc("block_timestamp") }}'
- name: codespace
description: '{{ doc("codespace") }}'
- name: tx_id
description: '{{ doc("tx_id") }}'
- name: tx_succeeded
description: '{{ doc("tx_succeeded") }}'
- name: tx_code
description: '{{ doc("tx_code") }}'
- name: tx_log
description: '{{ doc("tx_log") }}'
- name: gas_used
description: '{{ doc("gas_used") }}'
- name: gas_wanted
description: '{{ doc("gas_wanted") }}'
- name: fact_transactions_id
description: '{{ doc("pk") }}'
- name: inserted_timestamp
description: '{{ doc("inserted_timestamp") }}'
- name: modified_timestamp
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_transactions_logs
description: '{{ doc("core__fact_transactions_logs") }}'
data_tests:
- dbt_utils.recency:
datepart: hour
field: modified_timestamp
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_id
description: '{{ doc("block_id") }}'
- name: block_timestamp
description: '{{ doc("block_timestamp") }}'
- name: tx_id
description: '{{ doc("tx_id") }}'
- name: tx_succeeded
description: '{{ doc("tx_succeeded") }}'
- name: tx_code
description: '{{ doc("tx_code") }}'
- name: codespace
description: '{{ doc("codespace") }}'
- name: tx_log
description: '{{ doc("tx_log") }}'
- name: transactions_logs_id
description: '{{ doc("pk") }}'
- name: inserted_timestamp
description: '{{ doc("inserted_timestamp") }}'
- name: modified_timestamp
description: '{{ doc("modified_timestamp") }}'

0
models/main_package/core/gold/core/core__fact_transfers.sql
View File

200
models/main_package/core/silver/core/_silver_core_tests.yml Normal file
View File

@ -0,0 +1,200 @@
version: 2
models:
- name: silver__blocks
config:
contract:
enforced: true
data_tests:
- dbt_utils.sequential_values:
column_name: block_id
interval: 1
config:
error_if: ">100"
tags: ['test_recency']
columns:
- name: block_id
data_type: NUMBER
data_tests:
- not_null:
tags: ['test_quality']
- name: block_timestamp
data_type: TIMESTAMP_NTZ
data_tests:
- not_null:
tags: ['test_quality']
- name: chain_id
data_type: VARCHAR
- name: tx_count
data_type: NUMBER
data_tests:
- dbt_utils.expression_is_true:
expression: ">=0"
tags: ['test_quality']
- name: proposer_address
data_type: VARCHAR
data_tests:
- not_null:
tags: ['test_quality']
- name: validator_hash
data_type: VARCHAR
data_tests:
- not_null:
tags: ['test_quality']
- name: header
data_type: VARIANT
- name: _inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: blocks_id
data_type: VARCHAR
- name: inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: modified_timestamp
data_type: TIMESTAMP_NTZ
- name: _invocation_id
data_type: VARCHAR
- name: silver__msg_attributes
columns:
- name: block_id
data_type: NUMBER
- name: block_timestamp
data_type: TIMESTAMP_NTZ
- name: tx_id
data_type: VARCHAR
- name: tx_succeeded
data_type: BOOLEAN
- name: msg_group
data_type: NUMBER
- name: msg_sub_group
data_type: NUMBER
- name: msg_index
data_type: NUMBER
- name: msg_type
data_type: VARCHAR
- name: attribute_index
data_type: NUMBER
- name: attribute_key
data_type: VARCHAR
- name: attribute_value
data_type: VARCHAR
- name: msg_attributes_id
data_type: VARCHAR
- name: inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: modified_timestamp
data_type: TIMESTAMP_NTZ
- name: _inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: _invocation_id
data_type: VARCHAR
- name: silver__msgs
columns:
- name: block_id
data_type: NUMBER
- name: block_timestamp
data_type: TIMESTAMP_NTZ
- name: tx_id
data_type: VARCHAR
- name: tx_succeeded
data_type: BOOLEAN
- name: msg_group
data_type: NUMBER
- name: msg_sub_group
data_type: NUMBER
- name: msg_index
data_type: NUMBER
- name: msg_type
data_type: VARCHAR
- name: msg
data_type: VARIANT
- name: msgs_id
data_type: VARCHAR
- name: inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: modified_timestamp
data_type: TIMESTAMP_NTZ
- name: _inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: _invocation_id
data_type: VARCHAR
- name: silver__transactions_logs
columns:
- name: block_id
data_type: NUMBER
- name: block_timestamp
data_type: TIMESTAMP_NTZ
- name: tx_id
data_type: VARCHAR
- name: tx_succeeded
data_type: BOOLEAN
- name: tx_code
data_type: NUMBER
- name: codespace
data_type: VARIANT
- name: tx_log
data_type: VARIANT
- name: transactions_logs_id
data_type: VARCHAR
- name: _inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: modified_timestamp
data_type: TIMESTAMP_NTZ
- name: _invocation_id
data_type: VARCHAR
- name: silver__transactions
config:
contract:
enforced: true
columns:
- name: block_id
data_type: NUMBER
data_tests:
- not_null:
tags: ['test_quality']
- name: block_timestamp
data_type: TIMESTAMP_NTZ
data_tests:
- not_null:
tags: ['test_quality']
- name: codespace
data_type: VARIANT
- name: tx_id
data_type: VARCHAR
data_tests:
- not_null:
tags: ['test_quality']
- name: tx_index
data_type: NUMBER
- name: tx_log
data_type: VARCHAR
- name: tx_succeeded
data_type: BOOLEAN
- name: gas_used
data_type: NUMBER
- name: gas_wanted
data_type: NUMBER
- name: tx_code
data_type: NUMBER
- name: DATA
data_type: VARIANT
- name: partition_key
data_type: VARCHAR
- name: block_id_requested
data_type: NUMBER
- name: _inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: transactions_id
data_type: VARCHAR
- name: inserted_timestamp
data_type: TIMESTAMP_NTZ
- name: modified_timestamp
data_type: TIMESTAMP_NTZ
- name: _invocation_id
data_type: VARCHAR
- name: silver__transfers

3
models/main_package/core/silver/core/silver__msgs.sql
View File

@ -127,8 +127,6 @@ msgs AS (
concat_ws(
'-',
bronze_msgs.tx_id,
bronze_msgs.msg_index
) AS unique_key,
_inserted_timestamp
FROM
bronze_msgs
@ -146,7 +144,6 @@ SELECT
msg_index,
msg_type,
msg :: OBJECT AS msg,
unique_key,
{{ dbt_utils.generate_surrogate_key(
['tx_id','msg_index']
) }} AS msgs_id,

0
models/main_package/core/silver/core/silver__transfers.sql
View File

9
models/sources.yml
View File

@ -11,8 +11,17 @@ sources:
schema: streamline
tables:
- name: blocks
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 4, period: hour}
- name: tx_counts
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 4, period: hour}
- name: transactions
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 4, period: hour}
- name: crosschain_silver
{{ 'CROSSCHAIN_DEV' if '_DEV' in target.database.upper() else 'CROSSCHAIN' }}
schema: silver