flip/sei-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/sei-models.git synced 2026-02-06 15:06:44 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

tests

Browse Source
This commit is contained in:
sam 2025-01-24 23:51:20 +08:00
parent d07d1bea48
commit 72bda78d23
9 changed files with 338 additions and 492 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
models/evm/gold/core/tests/traces/test_gold__fact_traces_full.sql Normal file
View File

@ -0,0 +1,9 @@
{{ config (
materialized = "view",
tags = ['full_test']
) }}
SELECT
*
FROM
{{ ref('core_evm__fact_traces') }}

120
models/evm/gold/core/tests/traces/test_gold__fact_traces_full.yml Normal file
View File

@ -0,0 +1,120 @@
version: 2
models:
- name: test_gold__fact_traces_full
description: "This is a view used to test all of the gold fact traces model."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_HASH
- TRACE_INDEX
- fsc_utils.sequence_gaps:
partition_by:
- TX_HASH
column_name: TRACE_INDEX
where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
tests:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TX_POSITION
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TRACE_INDEX
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: FROM_ADDRESS
tests:
- not_null:
where: TYPE <> 'SELFDESTRUCT'
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TO_ADDRESS
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
where: TO_ADDRESS IS NOT NULL
- name: INPUT
tests:
- not_null
- name: TYPE
tests:
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: SUB_TRACES
tests:
- not_null
- name: VALUE
tests:
- not_null
- name: VALUE_PRECISE_RAW
tests:
- not_null
- name: VALUE_PRECISE
tests:
- not_null
- name: VALUE_HEX
tests:
- not_null
- name: GAS
tests:
- not_null
- name: GAS_USED
tests:
- not_null
- name: ORIGIN_FROM_ADDRESS
tests:
- not_null
- name: ORIGIN_FUNCTION_SIGNATURE
tests:
- not_null
- name: TRACE_SUCCEEDED
tests:
- not_null
- name: TX_SUCCEEDED
tests:
- not_null
- name: FACT_TRACES_ID
tests:
- not_null
- name: INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2

16
models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.sql Normal file
View File

@ -0,0 +1,16 @@
{{ config (
materialized = "view",
tags = ['recent_test']
) }}
SELECT
*
FROM
{{ ref('core_evm__fact_traces') }}
WHERE
block_number > (
SELECT
block_number
FROM
{{ ref('_block_lookback') }}
)

120
models/evm/gold/core/tests/traces/test_gold__fact_traces_recent.yml Normal file
View File

@ -0,0 +1,120 @@
version: 2
models:
- name: test_gold__fact_traces_recent
description: "This is a view used to test the last three days of fact traces."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_HASH
- TRACE_INDEX
- fsc_utils.sequence_gaps:
partition_by:
- TX_HASH
column_name: TRACE_INDEX
where: TX_HASH IS NOT NULL
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
tests:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TX_POSITION
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TRACE_INDEX
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: FROM_ADDRESS
tests:
- not_null:
where: TYPE <> 'SELFDESTRUCT'
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TO_ADDRESS
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
where: TO_ADDRESS IS NOT NULL
- name: INPUT
tests:
- not_null
- name: TYPE
tests:
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: SUB_TRACES
tests:
- not_null
- name: VALUE
tests:
- not_null
- name: VALUE_PRECISE_RAW
tests:
- not_null
- name: VALUE_PRECISE
tests:
- not_null
- name: VALUE_HEX
tests:
- not_null
- name: GAS
tests:
- not_null
- name: GAS_USED
tests:
- not_null
- name: ORIGIN_FROM_ADDRESS
tests:
- not_null
- name: ORIGIN_FUNCTION_SIGNATURE
tests:
- not_null
- name: TRACE_SUCCEEDED
tests:
- not_null
- name: TX_SUCCEEDED
tests:
- not_null
- name: FACT_TRACES_ID
tests:
- not_null
- name: INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2

429
models/evm/silver/core/silver_evm__traces_old.sql
View File

@ -1,429 +0,0 @@
-- depends_on: {{ ref('bronze_evm__streamline_traces') }}
{{ config (
materialized = "incremental",
incremental_strategy = 'delete+insert',
unique_key = "block_number",
cluster_by = "block_timestamp::date, _inserted_timestamp::date",
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
full_refresh = false,
tags = ['core']
) }}
{% if execute %}
{% set query_traces %}
CREATE
OR REPLACE temporary TABLE silver_evm.traces__traces_intermediate_tmp AS
SELECT
VALUE :BLOCK_NUMBER :: INT AS block_number,
VALUE :array_index :: INT AS tx_position,
DATA :result AS full_traces,
DATA :txHash :: STRING AS tx_hash,
_inserted_timestamp
FROM
{% if is_incremental() %}
{{ ref('bronze_evm__streamline_traces') }}
WHERE
_inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp) _inserted_timestamp
FROM
{{ this }}
)
AND DATA :result IS NOT NULL
{% else %}
{{ ref('bronze_evm__streamline_fr_traces') }}
WHERE
DATA :result IS NOT NULL
{% endif %}
qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position
ORDER BY
_inserted_timestamp DESC)) = 1 {% endset %}
{% do run_query(
query_traces
) %}
{% endif %}
WITH flatten_traces AS (
SELECT
block_number,
tx_hash,
IFF(
path IN (
'result',
'result.value',
'result.type',
'result.to',
'result.input',
'result.gasUsed',
'result.gas',
'result.from',
'result.output',
'result.error',
'result.revertReason',
'gasUsed',
'gas',
'type',
'to',
'from',
'value',
'input',
'error',
'output',
'revertReason'
),
'ORIGIN',
REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '')
) AS trace_address,
_inserted_timestamp,
OBJECT_AGG(
key,
VALUE
) AS trace_json,
CASE
WHEN trace_address = 'ORIGIN' THEN NULL
WHEN POSITION(
'_' IN trace_address
) = 0 THEN 'ORIGIN'
ELSE REGEXP_REPLACE(
trace_address,
'_[0-9]+$',
'',
1,
1
)
END AS parent_trace_address,
SPLIT(
trace_address,
'_'
) AS str_array
FROM
silver_evm.traces__traces_intermediate_tmp txs,
TABLE(
FLATTEN(
input => PARSE_JSON(
txs.full_traces
),
recursive => TRUE
)
) f
WHERE
f.index IS NULL
AND f.key != 'calls'
AND f.path != 'result'
GROUP BY
block_number,
tx_hash,
trace_address,
_inserted_timestamp
),
sub_traces AS (
SELECT
block_number,
tx_hash,
parent_trace_address,
COUNT(*) AS sub_traces
FROM
flatten_traces
GROUP BY
block_number,
tx_hash,
parent_trace_address
),
num_array AS (
SELECT
block_number,
tx_hash,
trace_address,
ARRAY_AGG(flat_value) AS num_array
FROM
(
SELECT
block_number,
tx_hash,
trace_address,
IFF(
VALUE :: STRING = 'ORIGIN',
-1,
VALUE :: INT
) AS flat_value
FROM
flatten_traces,
LATERAL FLATTEN (
input => str_array
)
)
GROUP BY
block_number,
tx_hash,
trace_address
),
cleaned_traces AS (
SELECT
b.block_number,
b.tx_hash,
b.trace_address,
IFNULL(
sub_traces,
0
) AS sub_traces,
num_array,
ROW_NUMBER() over (
PARTITION BY b.block_number,
b.tx_hash
ORDER BY
num_array ASC
) - 1 AS trace_index,
trace_json,
b._inserted_timestamp
FROM
flatten_traces b
LEFT JOIN sub_traces s
ON b.block_number = s.block_number
AND b.tx_hash = s.tx_hash
AND b.trace_address = s.parent_trace_address
JOIN num_array n
ON b.block_number = n.block_number
AND b.tx_hash = n.tx_hash
AND b.trace_address = n.trace_address
),
final_traces AS (
SELECT
tx_hash,
trace_index,
block_number,
trace_address,
trace_json :error :: STRING AS error_reason,
trace_json :from :: STRING AS from_address,
trace_json :to :: STRING AS to_address,
IFNULL(
utils.udf_hex_to_int(
trace_json :value :: STRING
),
'0'
) AS value_precise_raw,
utils.udf_decimal_adjust(
value_precise_raw,
18
) AS value_precise,
value_precise :: FLOAT AS VALUE,
utils.udf_hex_to_int(
trace_json :gas :: STRING
) :: INT AS gas,
utils.udf_hex_to_int(
trace_json :gasUsed :: STRING
) :: INT AS gas_used,
trace_json :input :: STRING AS input,
trace_json :output :: STRING AS output,
trace_json :type :: STRING AS TYPE,
concat_ws(
'_',
TYPE,
trace_address
) AS identifier,
_inserted_timestamp,
trace_json AS DATA,
sub_traces
FROM
cleaned_traces
),
new_records AS (
SELECT
f.block_number,
f.tx_hash,
t.block_timestamp,
t.tx_status,
t.position AS tx_position,
f.trace_index,
f.from_address,
f.to_address,
f.value_precise_raw,
f.value_precise,
f.value,
f.gas,
f.gas_used,
f.input,
f.output,
f.type,
f.identifier,
f.sub_traces,
f.error_reason,
IFF(
f.error_reason IS NULL,
'SUCCESS',
'FAIL'
) AS trace_status,
f.data,
IFF(
t.tx_hash IS NULL
OR t.block_timestamp IS NULL
OR t.tx_status IS NULL,
TRUE,
FALSE
) AS is_pending,
concat_ws(
'-',
f.block_number,
t.position,
f.identifier
) AS _call_id,
f._inserted_timestamp
FROM
final_traces f
LEFT OUTER JOIN {{ ref('silver_evm__transactions') }}
t
ON f.tx_hash = t.tx_hash
AND f.block_number = t.block_number
{% if is_incremental() %}
AND t._INSERTED_TIMESTAMP >= (
SELECT
DATEADD('hour', -24, MAX(_inserted_timestamp))
FROM
{{ this }})
{% endif %}
)
{% if is_incremental() %},
missing_data AS (
SELECT
t.block_number,
t.tx_hash,
txs.block_timestamp,
txs.tx_status,
txs.position AS tx_position,
t.trace_index,
t.from_address,
t.to_address,
t.value_precise_raw,
t.value_precise,
t.value,
t.gas,
t.gas_used,
t.input,
t.output,
t.type,
t.identifier,
t.sub_traces,
t.error_reason,
t.trace_status,
t.data,
FALSE AS is_pending,
concat_ws(
'-',
t.block_number,
txs.position,
t.identifier
) AS _call_id,
GREATEST(
t._inserted_timestamp,
txs._inserted_timestamp
) AS _inserted_timestamp
FROM
{{ this }}
t
INNER JOIN {{ ref('silver_evm__transactions') }}
txs
ON t.tx_hash = txs.tx_hash
AND t.block_number = txs.block_number
WHERE
t.is_pending
)
{% endif %},
FINAL AS (
SELECT
block_number,
tx_hash,
block_timestamp,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
trace_status,
DATA,
is_pending,
_call_id,
_inserted_timestamp
FROM
new_records
{% if is_incremental() %}
UNION
SELECT
block_number,
tx_hash,
block_timestamp,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
trace_status,
DATA,
is_pending,
_call_id,
_inserted_timestamp
FROM
missing_data
{% endif %}
)
SELECT
block_number,
tx_hash,
block_timestamp,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
trace_status,
DATA,
is_pending,
_call_id,
_inserted_timestamp,
value_precise_raw,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }} AS traces_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_hash, trace_index
ORDER BY
_inserted_timestamp DESC, is_pending ASC)) = 1

2
models/evm/silver/core/tests/traces/test_silver_evm__traces_full.sql
View File

@ -6,6 +6,6 @@
SELECT
*
FROM
{{ ref('core_evm__fact_traces') }}
{{ ref('silver_evm__traces') }}
WHERE
block_timestamp < DATEADD('hour', -1, SYSDATE())

75
models/evm/silver/core/tests/traces/test_silver_evm__traces_full.yml
View File

@ -1,59 +1,52 @@
version: 2
models:
- name: test_silver_evm__traces_full
- name: test_silver__traces_full
description: "This is a view used to test all of the silver traces model."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- BLOCK_NUMBER
- TX_POSITION
- TRACE_INDEX
- TRACES_ID
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
- not_null
- name: TX_HASH
tests:
- not_null:
where: NOT IS_PENDING
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: TRACE_JSON
tests:
- not_null
- name: _INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: day
interval: 1
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
tests:
- not_null:
where: NOT IS_PENDING
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: FROM_ADDRESS
tests:
- not_null:
where: TYPE <> 'SELFDESTRUCT'
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TO_ADDRESS
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
where: TO_ADDRESS IS NOT NULL
- name: IDENTIFIER
- name: INSERTED_TIMESTAMP
tests:
- not_null
- name: VALUE
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- name: GAS
tests:
- not_null
- name: GAS_USED
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ

2
models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.sql
View File

@ -13,7 +13,7 @@ WITH last_3_days AS (
SELECT
*
FROM
{{ ref('core_evm__fact_traces') }}
{{ ref('silver_evm__traces') }}
WHERE
block_number >= (
SELECT

57
models/evm/silver/core/tests/traces/test_silver_evm__traces_recent.yml
View File

@ -1,35 +1,52 @@
version: 2
models:
- name: test_silver_evm__traces_recent
- name: test_silver__traces_recent
description: "This is a view used to test the last three days of traces."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- BLOCK_NUMBER
- TX_POSITION
- TRACE_INDEX
- TRACES_ID
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
- not_null
- name: TX_HASH
tests:
- not_null:
where: NOT IS_PENDING
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: TRACE_JSON
tests:
- not_null
- name: _INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: day
interval: 1
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
- name: INSERTED_TIMESTAMP
tests:
- not_null:
where: NOT IS_PENDING
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ