flip/arbitrum-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/arbitrum-models.git synced 2026-02-06 13:56:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

An 5714/traces2 hotswap arb (#298)

Browse Source 
* change to gold

* docs

* add tests

* tests

---------

Co-authored-by: Austin <austin@flipsidecrypto.com>
This commit is contained in:
Sam 2025-01-30 02:22:44 +08:00 committed by GitHub
parent 64171acb06
commit 173a876b27
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 1145 additions and 1306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
.github/workflows/dbt_run_traces_fix.yml vendored
View File

@ -1,50 +0,0 @@
name: dbt_run_traces_fix
run-name: dbt_run_traces_fix
on:
workflow_dispatch:
inputs:
use_xl_env:
description: "Use the 2xl environment"
type: boolean
schedule:
# every 15 minutes (see https://crontab.guru)
- cron: "*/15 * * * *"
env:
DBT_PROFILES_DIR: ./
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: ${{ github.event_name == 'workflow_dispatch' && inputs.use_xl_env && 'workflow_prod_2xl' || 'workflow_prod_backfill' }}
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: run traces fix model
run: |
dbt run -m "arbitrum_models,tag:traces_fix"

640
models/gold/core/core__fact_traces.sql
View File

@ -1,45 +1,625 @@
{{ config(
materialized = 'view',
persist_docs ={ "relation": true,
"columns": true }
{{ config (
materialized = "incremental",
incremental_strategy = 'delete+insert',
unique_key = "block_number",
cluster_by = "block_timestamp::date",
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
tags = ['core','non_realtime'],
incremental_predicates = [fsc_evm.standard_predicate()],
full_refresh = false
) }}
{# {{ fsc_evm.gold_traces_v1(
full_reload_start_block = 30000000,
full_reload_blocks = 10000000,
arb_traces_mode = TRUE
) }}
#}
WITH silver_traces AS (
SELECT
block_number,
tx_position,
trace_address,
parent_trace_address,
trace_address_array,
trace_json,
traces_id,
'regular' AS source
FROM
{{ ref('silver__traces') }}
WHERE
1 = 1
{% if is_incremental() and not full_reload_mode %}
AND modified_timestamp > (
SELECT
MAX(modified_timestamp)
FROM
{{ this }}
) {% elif is_incremental() and full_reload_mode %}
AND block_number BETWEEN (
SELECT
MAX(
block_number
)
FROM
{{ this }}
)
AND (
SELECT
MAX(
block_number
) + 10000000
FROM
{{ this }}
)
{% else %}
AND block_number <= 30000000
{% endif %}
UNION ALL
SELECT
block_number,
tx_position,
trace_address,
parent_trace_address,
IFF(
trace_address = 'ORIGIN',
ARRAY_CONSTRUCT('ORIGIN'),
trace_address_array
) AS trace_address_array,
trace_json,
traces_id,
'arb_traces' AS source
FROM
{{ ref('silver__arb_traces') }}
WHERE
1 = 1
{% if is_incremental() and not full_reload_mode %}
AND modified_timestamp > (
SELECT
DATEADD('hour', -2, MAX(modified_timestamp))
FROM
{{ this }}) {% elif is_incremental() and full_reload_mode %}
AND block_number BETWEEN (
SELECT
MAX(
block_number
)
FROM
{{ this }}
)
AND (
SELECT
MAX(
block_number
) + 10000000
FROM
{{ this }}
)
{% else %}
AND block_number <= 30000000
{% endif %}
),
sub_traces AS (
SELECT
block_number,
tx_position,
parent_trace_address,
COUNT(*) AS sub_traces
FROM
silver_traces
GROUP BY
block_number,
tx_position,
parent_trace_address
),
trace_index_array AS (
SELECT
block_number,
tx_position,
trace_address,
ARRAY_AGG(flat_value) AS number_array
FROM
(
SELECT
block_number,
tx_position,
trace_address,
IFF(
VALUE :: STRING = 'ORIGIN',
-1,
VALUE :: INT
) AS flat_value
FROM
silver_traces,
LATERAL FLATTEN (
input => trace_address_array
)
)
GROUP BY
block_number,
tx_position,
trace_address
),
trace_index_sub_traces AS (
SELECT
b.block_number,
b.tx_position,
b.trace_address,
IFNULL(
sub_traces,
0
) AS sub_traces,
number_array,
ROW_NUMBER() over (
PARTITION BY b.block_number,
b.tx_position
ORDER BY
number_array ASC
) - 1 AS trace_index,
b.trace_json,
b.traces_id,
b.source
FROM
silver_traces b
LEFT JOIN sub_traces s
ON b.block_number = s.block_number
AND b.tx_position = s.tx_position
AND b.trace_address = s.parent_trace_address
JOIN trace_index_array n
ON b.block_number = n.block_number
AND b.tx_position = n.tx_position
AND b.trace_address = n.trace_address
),
errored_traces AS (
SELECT
block_number,
tx_position,
trace_address,
trace_json
FROM
trace_index_sub_traces
WHERE
trace_json :error :: STRING IS NOT NULL
),
error_logic AS (
SELECT
b0.block_number,
b0.tx_position,
b0.trace_address,
b0.trace_json :error :: STRING AS error,
b1.trace_json :error :: STRING AS any_error,
b2.trace_json :error :: STRING AS origin_error
FROM
trace_index_sub_traces b0
LEFT JOIN errored_traces b1
ON b0.block_number = b1.block_number
AND b0.tx_position = b1.tx_position
AND b0.trace_address RLIKE CONCAT(
'^',
b1.trace_address,
'(_[0-9]+)*$'
)
LEFT JOIN errored_traces b2
ON b0.block_number = b2.block_number
AND b0.tx_position = b2.tx_position
AND b2.trace_address = 'ORIGIN'
),
aggregated_errors AS (
SELECT
block_number,
tx_position,
trace_address,
error,
IFF(MAX(any_error) IS NULL
AND error IS NULL
AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded
FROM
error_logic
GROUP BY
block_number,
tx_position,
trace_address,
error,
origin_error),
json_traces AS (
SELECT
block_number,
tx_position,
trace_address,
sub_traces,
number_array,
trace_index,
trace_json AS DATA,
trace_succeeded,
trace_json :error :: STRING AS error_reason,
trace_json :revertReason :: STRING AS revert_reason,
trace_json :from :: STRING AS from_address,
trace_json :to :: STRING AS to_address,
IFNULL(
trace_json :value :: STRING,
'0x0'
) AS value_hex,
IFNULL(
utils.udf_hex_to_int(
trace_json :value :: STRING
),
'0'
) AS value_precise_raw,
utils.udf_decimal_adjust(
value_precise_raw,
18
) AS value_precise,
value_precise :: FLOAT AS VALUE,
utils.udf_hex_to_int(
trace_json :gas :: STRING
) :: INT AS gas,
utils.udf_hex_to_int(
trace_json :gasUsed :: STRING
) :: INT AS gas_used,
trace_json :input :: STRING AS input,
trace_json :output :: STRING AS output,
trace_json :type :: STRING AS TYPE,
concat_ws(
'_',
TYPE,
trace_address
) AS identifier,
IFF(
trace_succeeded,
'SUCCESS',
'FAIL'
) AS trace_status,
traces_id,
trace_json :afterEVMTransfers AS after_evm_transfers,
trace_json :beforeEVMTransfers AS before_evm_transfers
FROM
trace_index_sub_traces t0
JOIN aggregated_errors USING (
block_number,
tx_position,
trace_address
)
WHERE
t0.source <> 'arb_traces'
UNION ALL
SELECT
block_number,
tx_position,
trace_address,
sub_traces,
number_array,
trace_index,
trace_json AS DATA,
trace_succeeded,
trace_json :error :: STRING AS error_reason,
NULL AS revert_reason,
trace_json :action :from :: STRING AS from_address,
COALESCE(
trace_json :action :to :: STRING,
trace_json :result :address :: STRING
) AS to_address,
IFNULL(
trace_json :action :value :: STRING,
'0x0'
) AS value_hex,
IFNULL(
utils.udf_hex_to_int(
trace_json :action :value :: STRING
),
'0'
) AS value_precise_raw,
utils.udf_decimal_adjust(
value_precise_raw,
18
) AS value_precise,
value_precise :: FLOAT AS VALUE,
utils.udf_hex_to_int(
trace_json :action :gas :: STRING
) :: INT AS gas,
IFNULL(
utils.udf_hex_to_int(
trace_json :result :gasUsed :: STRING
),
0
) :: INT AS gas_used,
COALESCE(
trace_json :action :input :: STRING,
trace_json :action :init :: STRING
) AS input,
COALESCE(
trace_json :result :output :: STRING,
trace_json :result :code :: STRING
) AS output,
UPPER(
COALESCE(
trace_json :action :callType :: STRING,
trace_json :type :: STRING
)
) AS TYPE,
concat_ws(
'_',
TYPE,
trace_address
) AS identifier,
IFF(
trace_succeeded,
'SUCCESS',
'FAIL'
) AS trace_status,
traces_id,
NULL AS after_evm_transfers,
NULL AS before_evm_transfers
FROM
trace_index_sub_traces t0
JOIN aggregated_errors USING (
block_number,
tx_position,
trace_address
)
WHERE
t0.source = 'arb_traces'
),
incremental_traces AS (
SELECT
f.block_number,
t.tx_hash,
t.block_timestamp,
t.origin_function_signature,
t.from_address AS origin_from_address,
t.to_address AS origin_to_address,
t.tx_status,
f.tx_position,
f.trace_index,
f.from_address AS from_address,
f.to_address AS to_address,
f.value_hex,
f.value_precise_raw,
f.value_precise,
f.value,
f.gas,
f.gas_used,
f.input,
f.output,
f.type,
f.identifier,
f.sub_traces,
f.error_reason,
f.revert_reason,
f.trace_status,
f.data,
f.traces_id,
f.trace_succeeded,
f.trace_address,
IFF(
t.tx_status = 'SUCCESS',
TRUE,
FALSE
) AS tx_succeeded,
f.before_evm_transfers,
f.after_evm_transfers
FROM
json_traces f
LEFT OUTER JOIN {{ ref('silver__transactions') }}
t
ON f.tx_position = t.position
AND f.block_number = t.block_number
{% if is_incremental() and not full_reload_mode %}
AND t.modified_timestamp >= (
SELECT
DATEADD('hour', -24, MAX(modified_timestamp))
FROM
{{ this }})
{% endif %}
)
{% if is_incremental() %},
overflow_blocks AS (
SELECT
DISTINCT block_number
FROM
silver_traces
WHERE
source = 'overflow'
),
heal_missing_data AS (
SELECT
t.block_number,
txs.tx_hash,
txs.block_timestamp,
txs.origin_function_signature,
txs.from_address AS origin_from_address,
txs.to_address AS origin_to_address,
txs.tx_status,
t.tx_position,
t.trace_index,
t.from_address,
t.to_address,
t.value_hex,
t.value_precise_raw,
t.value_precise,
t.value,
t.gas,
t.gas_used,
t.input,
t.output,
t.type,
t.identifier,
t.sub_traces,
t.error_reason,
t.revert_reason,
t.trace_status,
t.data,
t.fact_traces_id AS traces_id,
t.trace_succeeded,
t.trace_address,
IFF(
txs.tx_status = 'SUCCESS',
TRUE,
FALSE
) AS tx_succeeded,
t.before_evm_transfers,
t.after_evm_transfers
FROM
{{ this }}
t
JOIN {{ ref('silver__transactions') }}
txs
ON t.tx_position = txs.position
AND t.block_number = txs.block_number
WHERE
t.tx_hash IS NULL
OR t.block_timestamp IS NULL
OR t.tx_status IS NULL
)
{% endif %},
all_traces AS (
SELECT
block_number,
tx_hash,
block_timestamp,
origin_function_signature,
origin_from_address,
origin_to_address,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_hex,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
revert_reason,
trace_status,
DATA,
trace_succeeded,
trace_address,
tx_succeeded,
before_evm_transfers,
after_evm_transfers
FROM
incremental_traces
UNION ALL
SELECT
block_number,
tx_hash,
block_timestamp,
origin_function_signature,
origin_from_address,
origin_to_address,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_hex,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
revert_reason,
trace_status,
DATA,
trace_succeeded,
trace_address,
tx_succeeded,
before_evm_transfers,
after_evm_transfers
FROM
heal_missing_data
UNION ALL
SELECT
block_number,
tx_hash,
block_timestamp,
origin_function_signature,
origin_from_address,
origin_to_address,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_hex,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
revert_reason,
trace_status,
DATA,
trace_succeeded,
trace_address,
tx_succeeded,
before_evm_transfers,
after_evm_transfers
FROM
{{ this }}
JOIN overflow_blocks USING (block_number)
)
SELECT
tx_hash,
block_number,
block_timestamp,
tx_hash,
tx_position,
trace_index,
from_address,
to_address,
eth_value AS VALUE,
eth_value_precise_raw AS value_precise_raw,
eth_value_precise AS value_precise,
gas,
gas_used,
input,
output,
TYPE,
trace_address,
sub_traces,
VALUE,
value_precise_raw,
value_precise,
value_hex,
gas,
gas_used,
origin_from_address,
origin_to_address,
origin_function_signature,
before_evm_transfers,
after_evm_transfers,
trace_succeeded,
error_reason,
revert_reason,
tx_succeeded,
identifier,
DATA,
tx_status,
sub_traces,
trace_status,
error_reason,
trace_index,
before_evm_transfers,
after_evm_transfers,
COALESCE (
traces_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }}
) AS fact_traces_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }} AS fact_traces_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp
FROM
{{ ref('silver__traces') }}
all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index
ORDER BY
modified_timestamp DESC, block_timestamp DESC nulls last)) = 1

70
models/gold/core/core__fact_traces.yml
View File

@ -1,56 +1,74 @@
version: 2
models:
- name: core__fact_traces
description: '{{ doc("arb_traces_table_doc") }}'
description: '{{ doc("arb_traces_table_doc") }}'
columns:
- name: BLOCK_NUMBER
description: '{{ doc("arb_traces_block_no") }}'
description: '{{ doc("arb_traces_block_no") }}'
- name: BLOCK_TIMESTAMP
description: '{{ doc("arb_traces_blocktime") }}'
- name: TX_HASH
description: '{{ doc("arb_traces_tx_hash") }}'
description: '{{ doc("arb_traces_tx_hash") }}'
- name: TX_POSITION
description: The position of the transaction within the block.
- name: TRACE_INDEX
description: The index of the trace within the transaction.
- name: FROM_ADDRESS
description: '{{ doc("arb_traces_from") }}'
- name: TO_ADDRESS
description: '{{ doc("arb_traces_to") }}'
- name: VALUE
description: '{{ doc("arb_traces_value") }}'
- name: VALUE_PRECISE_RAW
description: '{{ doc("precise_amount_unadjusted") }}'
- name: VALUE_PRECISE
description: '{{ doc("precise_amount_adjusted") }}'
- name: GAS
description: '{{ doc("arb_traces_gas") }}'
- name: GAS_USED
description: '{{ doc("arb_traces_gas_used") }}'
- name: INPUT
description: '{{ doc("arb_traces_input") }}'
- name: OUTPUT
description: '{{ doc("arb_traces_output") }}'
- name: TYPE
description: '{{ doc("arb_traces_type") }}'
- name: TRACE_ADDRESS
description: This field represents the position of the trace within the transaction.
- name: SUB_TRACES
description: '{{ doc("arb_traces_sub") }}'
- name: IDENTIFIER
description: '{{ doc("arb_traces_identifier") }}'
- name: DATA
description: '{{ doc("arb_traces_call_data") }}'
- name: TX_STATUS
description: '{{ doc("arb_tx_status") }}'
- name: SUB_TRACES
description: '{{ doc("arb_traces_sub") }}'
- name: TRACE_STATUS
description: The status of the trace, either `SUCCESS` or `FAIL`
- name: ERROR_REASON
description: The reason for the trace failure, if any.
- name: TRACE_INDEX
description: The index of the trace within the transaction.
- name: VALUE
description: '{{ doc("arb_traces_value") }}'
- name: VALUE_PRECISE_RAW
description: '{{ doc("precise_amount_unadjusted") }}'
- name: VALUE_PRECISE
description: '{{ doc("precise_amount_adjusted") }}'
- name: VALUE_HEX
description: The value transacted in ETH in hexadecimal format.
- name: GAS
description: '{{ doc("arb_traces_gas") }}'
- name: GAS_USED
description: '{{ doc("arb_traces_gas_used") }}'
- name: ORIGIN_FROM_ADDRESS
description: The from address at the transaction level.
- name: ORIGIN_TO_ADDRESS
description: The to address at the transaction level.
- name: ORIGIN_FUNCTION_SIGNATURE
description: The function signature at the transaction level.
- name: BEFORE_EVM_TRANSFERS
description: The state of all ETH transfers to be executed for a given transaction.
- name: AFTER_EVM_TRANSFERS
description: The state of all ETH transfers executed for a given transaction.
- name: TRACE_STATUS
description: The status of the trace, either `SUCCESS` or `FAIL`
- name: TRACE_SUCCEEDED
description: Whether the trace succeeded or failed
- name: ERROR_REASON
description: The reason for the trace failure, if any.
- name: REVERT_REASON
description: The reason for the trace revert, if any.
- name: TX_STATUS
description: '{{ doc("arb_tx_status") }}'
- name: TX_SUCCEEDED
description: Whether the transaction succeeded or failed
- name: FACT_TRACES_ID
description: '{{ doc("pk") }}'
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'
description: '{{ doc("modified_timestamp") }}'

9
models/gold/core/tests/traces/test_gold__fact_traces_full.sql Normal file
View File

@ -0,0 +1,9 @@
{{ config (
materialized = "view",
tags = ['full_test']
) }}
SELECT
*
FROM
{{ ref('core__fact_traces') }}

120
models/gold/core/tests/traces/test_gold__fact_traces_full.yml Normal file
View File

@ -0,0 +1,120 @@
version: 2
models:
- name: test_gold__fact_traces_full
description: "This is a view used to test all of the gold fact traces model."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_HASH
- TRACE_INDEX
- fsc_utils.sequence_gaps:
partition_by:
- TX_HASH
column_name: TRACE_INDEX
where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
tests:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TX_POSITION
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TRACE_INDEX
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: FROM_ADDRESS
tests:
- not_null:
where: TYPE <> 'SELFDESTRUCT'
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TO_ADDRESS
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
where: TO_ADDRESS IS NOT NULL
- name: INPUT
tests:
- not_null
- name: TYPE
tests:
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: SUB_TRACES
tests:
- not_null
- name: VALUE
tests:
- not_null
- name: VALUE_PRECISE_RAW
tests:
- not_null
- name: VALUE_PRECISE
tests:
- not_null
- name: VALUE_HEX
tests:
- not_null
- name: GAS
tests:
- not_null
- name: GAS_USED
tests:
- not_null
- name: ORIGIN_FROM_ADDRESS
tests:
- not_null
- name: ORIGIN_FUNCTION_SIGNATURE
tests:
- not_null
- name: TRACE_SUCCEEDED
tests:
- not_null
- name: TX_SUCCEEDED
tests:
- not_null
- name: FACT_TRACES_ID
tests:
- not_null
- name: INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2

16
models/gold/core/tests/traces/test_gold__fact_traces_recent.sql Normal file
View File

@ -0,0 +1,16 @@
{{ config (
materialized = "view",
tags = ['recent_test']
) }}
SELECT
*
FROM
{{ ref('core__fact_traces') }}
WHERE
block_number > (
SELECT
block_number
FROM
{{ ref('_block_lookback') }}
)

120
models/gold/core/tests/traces/test_gold__fact_traces_recent.yml Normal file
View File

@ -0,0 +1,120 @@
version: 2
models:
- name: test_gold__fact_traces_recent
description: "This is a view used to test the last three days of fact traces."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_HASH
- TRACE_INDEX
- fsc_utils.sequence_gaps:
partition_by:
- TX_HASH
column_name: TRACE_INDEX
where: TX_HASH IS NOT NULL
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
tests:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TX_POSITION
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: TRACE_INDEX
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: FROM_ADDRESS
tests:
- not_null:
where: TYPE <> 'SELFDESTRUCT'
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TO_ADDRESS
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
where: TO_ADDRESS IS NOT NULL
- name: INPUT
tests:
- not_null
- name: TYPE
tests:
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: SUB_TRACES
tests:
- not_null
- name: VALUE
tests:
- not_null
- name: VALUE_PRECISE_RAW
tests:
- not_null
- name: VALUE_PRECISE
tests:
- not_null
- name: VALUE_HEX
tests:
- not_null
- name: GAS
tests:
- not_null
- name: GAS_USED
tests:
- not_null
- name: ORIGIN_FROM_ADDRESS
tests:
- not_null
- name: ORIGIN_FUNCTION_SIGNATURE
tests:
- not_null
- name: TRACE_SUCCEEDED
tests:
- not_null
- name: TX_SUCCEEDED
tests:
- not_null
- name: FACT_TRACES_ID
tests:
- not_null
- name: INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2

5
models/silver/_observability/silver_observability__traces_completeness.sql
View File

@ -81,7 +81,10 @@ broken_blocks AS (
FROM
{{ ref("silver__transactions") }}
tx
LEFT JOIN {{ ref("silver__traces") }}
LEFT JOIN {{ source(
'arbitrum_silver',
'traces'
) }}
tr USING (
block_number,
tx_hash

11
models/silver/abis/silver__proxies.sql
View File

@ -11,9 +11,9 @@ WITH base AS (
from_address,
to_address,
MIN(block_number) AS start_block,
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(modified_timestamp) AS _inserted_timestamp
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
TYPE = 'DELEGATECALL'
AND trace_status = 'SUCCESS'
@ -21,7 +21,7 @@ WITH base AS (
AND from_address != to_address -- exclude self-calls
{% if is_incremental() %}
AND _inserted_timestamp >= (
AND modified_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '24 hours'
FROM
@ -99,5 +99,6 @@ FROM
FINAL f
JOIN {{ ref('silver__created_contracts') }} C
ON f.contract_address = C.created_contract_address
JOIN {{ ref('silver__created_contracts') }} p
ON f.proxy_address = p.created_contract_address
JOIN {{ ref('silver__created_contracts') }}
p
ON f.proxy_address = p.created_contract_address

2
models/silver/abis/silver__user_verified_abis.sql
View File

@ -473,7 +473,7 @@ valid_traces AS (
SELECT
base_address
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
JOIN function_mapping
ON function_sig = LEFT(
input,

4
models/silver/core/silver__created_contracts.sql
View File

@ -13,7 +13,7 @@ SELECT
to_address AS created_contract_address,
from_address AS creator_address,
input AS created_contract_input,
_inserted_timestamp,
modified_timestamp AS _inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['to_address']
) }} AS created_contracts_id,
@ -21,7 +21,7 @@ SELECT
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
TYPE ILIKE 'create%'
AND to_address IS NOT NULL

625
models/silver/core/silver__fact_traces2.sql
View File

@ -1,625 +0,0 @@
{{ config (
materialized = "incremental",
incremental_strategy = 'delete+insert',
unique_key = "block_number",
cluster_by = "block_timestamp::date",
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
tags = ['core','non_realtime'],
incremental_predicates = [fsc_evm.standard_predicate()],
full_refresh = false
) }}
{# {{ fsc_evm.gold_traces_v1(
full_reload_start_block = 30000000,
full_reload_blocks = 10000000,
arb_traces_mode = TRUE
) }}
#}
WITH silver_traces AS (
SELECT
block_number,
tx_position,
trace_address,
parent_trace_address,
trace_address_array,
trace_json,
traces_id,
'regular' AS source
FROM
{{ ref('silver__traces2') }}
WHERE
1 = 1
{% if is_incremental() and not full_reload_mode %}
AND modified_timestamp > (
SELECT
MAX(modified_timestamp)
FROM
{{ this }}
) {% elif is_incremental() and full_reload_mode %}
AND block_number BETWEEN (
SELECT
MAX(
block_number
)
FROM
{{ this }}
)
AND (
SELECT
MAX(
block_number
) + 10000000
FROM
{{ this }}
)
{% else %}
AND block_number <= 30000000
{% endif %}
UNION ALL
SELECT
block_number,
tx_position,
trace_address,
parent_trace_address,
IFF(
trace_address = 'ORIGIN',
ARRAY_CONSTRUCT('ORIGIN'),
trace_address_array
) AS trace_address_array,
trace_json,
traces_id,
'arb_traces' AS source
FROM
{{ ref('silver__arb_traces') }}
WHERE
1 = 1
{% if is_incremental() and not full_reload_mode %}
AND modified_timestamp > (
SELECT
DATEADD('hour', -2, MAX(modified_timestamp))
FROM
{{ this }}) {% elif is_incremental() and full_reload_mode %}
AND block_number BETWEEN (
SELECT
MAX(
block_number
)
FROM
{{ this }}
)
AND (
SELECT
MAX(
block_number
) + 10000000
FROM
{{ this }}
)
{% else %}
AND block_number <= 30000000
{% endif %}
),
sub_traces AS (
SELECT
block_number,
tx_position,
parent_trace_address,
COUNT(*) AS sub_traces
FROM
silver_traces
GROUP BY
block_number,
tx_position,
parent_trace_address
),
trace_index_array AS (
SELECT
block_number,
tx_position,
trace_address,
ARRAY_AGG(flat_value) AS number_array
FROM
(
SELECT
block_number,
tx_position,
trace_address,
IFF(
VALUE :: STRING = 'ORIGIN',
-1,
VALUE :: INT
) AS flat_value
FROM
silver_traces,
LATERAL FLATTEN (
input => trace_address_array
)
)
GROUP BY
block_number,
tx_position,
trace_address
),
trace_index_sub_traces AS (
SELECT
b.block_number,
b.tx_position,
b.trace_address,
IFNULL(
sub_traces,
0
) AS sub_traces,
number_array,
ROW_NUMBER() over (
PARTITION BY b.block_number,
b.tx_position
ORDER BY
number_array ASC
) - 1 AS trace_index,
b.trace_json,
b.traces_id,
b.source
FROM
silver_traces b
LEFT JOIN sub_traces s
ON b.block_number = s.block_number
AND b.tx_position = s.tx_position
AND b.trace_address = s.parent_trace_address
JOIN trace_index_array n
ON b.block_number = n.block_number
AND b.tx_position = n.tx_position
AND b.trace_address = n.trace_address
),
errored_traces AS (
SELECT
block_number,
tx_position,
trace_address,
trace_json
FROM
trace_index_sub_traces
WHERE
trace_json :error :: STRING IS NOT NULL
),
error_logic AS (
SELECT
b0.block_number,
b0.tx_position,
b0.trace_address,
b0.trace_json :error :: STRING AS error,
b1.trace_json :error :: STRING AS any_error,
b2.trace_json :error :: STRING AS origin_error
FROM
trace_index_sub_traces b0
LEFT JOIN errored_traces b1
ON b0.block_number = b1.block_number
AND b0.tx_position = b1.tx_position
AND b0.trace_address RLIKE CONCAT(
'^',
b1.trace_address,
'(_[0-9]+)*$'
)
LEFT JOIN errored_traces b2
ON b0.block_number = b2.block_number
AND b0.tx_position = b2.tx_position
AND b2.trace_address = 'ORIGIN'
),
aggregated_errors AS (
SELECT
block_number,
tx_position,
trace_address,
error,
IFF(MAX(any_error) IS NULL
AND error IS NULL
AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded
FROM
error_logic
GROUP BY
block_number,
tx_position,
trace_address,
error,
origin_error),
json_traces AS (
SELECT
block_number,
tx_position,
trace_address,
sub_traces,
number_array,
trace_index,
trace_json AS DATA,
trace_succeeded,
trace_json :error :: STRING AS error_reason,
trace_json :revertReason :: STRING AS revert_reason,
trace_json :from :: STRING AS from_address,
trace_json :to :: STRING AS to_address,
IFNULL(
trace_json :value :: STRING,
'0x0'
) AS value_hex,
IFNULL(
utils.udf_hex_to_int(
trace_json :value :: STRING
),
'0'
) AS value_precise_raw,
utils.udf_decimal_adjust(
value_precise_raw,
18
) AS value_precise,
value_precise :: FLOAT AS VALUE,
utils.udf_hex_to_int(
trace_json :gas :: STRING
) :: INT AS gas,
utils.udf_hex_to_int(
trace_json :gasUsed :: STRING
) :: INT AS gas_used,
trace_json :input :: STRING AS input,
trace_json :output :: STRING AS output,
trace_json :type :: STRING AS TYPE,
concat_ws(
'_',
TYPE,
trace_address
) AS identifier,
IFF(
trace_succeeded,
'SUCCESS',
'FAIL'
) AS trace_status,
traces_id,
trace_json :afterEVMTransfers AS after_evm_transfers,
trace_json :beforeEVMTransfers AS before_evm_transfers
FROM
trace_index_sub_traces t0
JOIN aggregated_errors USING (
block_number,
tx_position,
trace_address
)
WHERE
t0.source <> 'arb_traces'
UNION ALL
SELECT
block_number,
tx_position,
trace_address,
sub_traces,
number_array,
trace_index,
trace_json AS DATA,
trace_succeeded,
trace_json :error :: STRING AS error_reason,
NULL AS revert_reason,
trace_json :action :from :: STRING AS from_address,
COALESCE(
trace_json :action :to :: STRING,
trace_json :result :address :: STRING
) AS to_address,
IFNULL(
trace_json :action :value :: STRING,
'0x0'
) AS value_hex,
IFNULL(
utils.udf_hex_to_int(
trace_json :action :value :: STRING
),
'0'
) AS value_precise_raw,
utils.udf_decimal_adjust(
value_precise_raw,
18
) AS value_precise,
value_precise :: FLOAT AS VALUE,
utils.udf_hex_to_int(
trace_json :action :gas :: STRING
) :: INT AS gas,
IFNULL(
utils.udf_hex_to_int(
trace_json :result :gasUsed :: STRING
),
0
) :: INT AS gas_used,
COALESCE(
trace_json :action :input :: STRING,
trace_json :action :init :: STRING
) AS input,
COALESCE(
trace_json :result :output :: STRING,
trace_json :result :code :: STRING
) AS output,
UPPER(
COALESCE(
trace_json :action :callType :: STRING,
trace_json :type :: STRING
)
) AS TYPE,
concat_ws(
'_',
TYPE,
trace_address
) AS identifier,
IFF(
trace_succeeded,
'SUCCESS',
'FAIL'
) AS trace_status,
traces_id,
NULL AS after_evm_transfers,
NULL AS before_evm_transfers
FROM
trace_index_sub_traces t0
JOIN aggregated_errors USING (
block_number,
tx_position,
trace_address
)
WHERE
t0.source = 'arb_traces'
),
incremental_traces AS (
SELECT
f.block_number,
t.tx_hash,
t.block_timestamp,
t.origin_function_signature,
t.from_address AS origin_from_address,
t.to_address AS origin_to_address,
t.tx_status,
f.tx_position,
f.trace_index,
f.from_address AS from_address,
f.to_address AS to_address,
f.value_hex,
f.value_precise_raw,
f.value_precise,
f.value,
f.gas,
f.gas_used,
f.input,
f.output,
f.type,
f.identifier,
f.sub_traces,
f.error_reason,
f.revert_reason,
f.trace_status,
f.data,
f.traces_id,
f.trace_succeeded,
f.trace_address,
IFF(
t.tx_status = 'SUCCESS',
TRUE,
FALSE
) AS tx_succeeded,
f.before_evm_transfers,
f.after_evm_transfers
FROM
json_traces f
LEFT OUTER JOIN {{ ref('silver__transactions') }}
t
ON f.tx_position = t.position
AND f.block_number = t.block_number
{% if is_incremental() and not full_reload_mode %}
AND t.modified_timestamp >= (
SELECT
DATEADD('hour', -24, MAX(modified_timestamp))
FROM
{{ this }})
{% endif %}
)
{% if is_incremental() %},
overflow_blocks AS (
SELECT
DISTINCT block_number
FROM
silver_traces
WHERE
source = 'overflow'
),
heal_missing_data AS (
SELECT
t.block_number,
txs.tx_hash,
txs.block_timestamp,
txs.origin_function_signature,
txs.from_address AS origin_from_address,
txs.to_address AS origin_to_address,
txs.tx_status,
t.tx_position,
t.trace_index,
t.from_address,
t.to_address,
t.value_hex,
t.value_precise_raw,
t.value_precise,
t.value,
t.gas,
t.gas_used,
t.input,
t.output,
t.type,
t.identifier,
t.sub_traces,
t.error_reason,
t.revert_reason,
t.trace_status,
t.data,
t.fact_traces_id AS traces_id,
t.trace_succeeded,
t.trace_address,
IFF(
txs.tx_status = 'SUCCESS',
TRUE,
FALSE
) AS tx_succeeded,
t.before_evm_transfers,
t.after_evm_transfers
FROM
{{ this }}
t
JOIN {{ ref('silver__transactions') }}
txs
ON t.tx_position = txs.position
AND t.block_number = txs.block_number
WHERE
t.tx_hash IS NULL
OR t.block_timestamp IS NULL
OR t.tx_status IS NULL
)
{% endif %},
all_traces AS (
SELECT
block_number,
tx_hash,
block_timestamp,
origin_function_signature,
origin_from_address,
origin_to_address,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_hex,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
revert_reason,
trace_status,
DATA,
trace_succeeded,
trace_address,
tx_succeeded,
before_evm_transfers,
after_evm_transfers
FROM
incremental_traces
UNION ALL
SELECT
block_number,
tx_hash,
block_timestamp,
origin_function_signature,
origin_from_address,
origin_to_address,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_hex,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
revert_reason,
trace_status,
DATA,
trace_succeeded,
trace_address,
tx_succeeded,
before_evm_transfers,
after_evm_transfers
FROM
heal_missing_data
UNION ALL
SELECT
block_number,
tx_hash,
block_timestamp,
origin_function_signature,
origin_from_address,
origin_to_address,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
value_hex,
value_precise_raw,
value_precise,
VALUE,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
revert_reason,
trace_status,
DATA,
trace_succeeded,
trace_address,
tx_succeeded,
before_evm_transfers,
after_evm_transfers
FROM
{{ this }}
JOIN overflow_blocks USING (block_number)
)
SELECT
block_number,
block_timestamp,
tx_hash,
tx_position,
trace_index,
from_address,
to_address,
input,
output,
TYPE,
trace_address,
sub_traces,
VALUE,
value_precise_raw,
value_precise,
value_hex,
gas,
gas_used,
origin_from_address,
origin_to_address,
origin_function_signature,
before_evm_transfers,
after_evm_transfers,
trace_succeeded,
error_reason,
revert_reason,
tx_succeeded,
identifier,
DATA,
tx_status,
trace_status,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }} AS fact_traces_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp
FROM
all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index
ORDER BY
modified_timestamp DESC, block_timestamp DESC nulls last)) = 1

25
models/silver/core/silver__native_transfers.sql
View File

@ -16,17 +16,26 @@ WITH eth_base AS (
identifier,
from_address,
to_address,
eth_value,
_call_id,
_inserted_timestamp,
eth_value_precise_raw,
eth_value_precise,
VALUE AS eth_value,
concat_ws(
'-',
block_number,
tx_position,
CONCAT(
TYPE,
'_',
trace_address
)
) AS _call_id,
modified_timestamp AS _inserted_timestamp,
value_precise_raw AS eth_value_precise_raw,
value_precise AS eth_value_precise,
tx_position,
trace_index
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
eth_value > 0
VALUE > 0
AND tx_status = 'SUCCESS'
AND trace_status = 'SUCCESS'
AND TYPE NOT IN (
@ -35,7 +44,7 @@ WITH eth_base AS (
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
AND modified_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '72 hours'
FROM

8
models/silver/core/silver__relevant_contracts.sql
View File

@ -31,10 +31,10 @@ function_calls AS (
SELECT
to_address AS contract_address,
COUNT(*) AS function_call_count,
MAX(_inserted_timestamp) AS max_inserted_timestamp_traces,
MAX(modified_timestamp) AS max_inserted_timestamp_traces,
MAX(block_number) AS latest_call_block
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
tx_status = 'SUCCESS'
AND trace_status = 'SUCCESS'
@ -43,7 +43,7 @@ function_calls AS (
AND input <> '0x'
{% if is_incremental() %}
AND _inserted_timestamp > (
AND modified_timestamp > (
SELECT
MAX(max_inserted_timestamp_traces)
FROM
@ -131,4 +131,4 @@ FROM
LEFT JOIN function_calls f
ON C.contract_address = f.contract_address
LEFT JOIN previous_totals p
ON C.contract_address = p.contract_address
ON C.contract_address = p.contract_address

515
models/silver/core/silver__traces.sql
View File

@ -3,24 +3,28 @@
materialized = "incremental",
incremental_strategy = 'delete+insert',
unique_key = "block_number",
cluster_by = "block_timestamp::date, _inserted_timestamp::date",
cluster_by = ['modified_timestamp::DATE','partition_key'],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
full_refresh = false,
tags = ['core','non_realtime']
) }}
{# {{ fsc_evm.silver_traces_v1(
full_reload_start_block = 30000000,
full_reload_blocks = 10000000,
arb_traces_mode = TRUE
) }}
#}
WITH bronze_traces AS (
SELECT
block_number,
DATA,
VALUE,
_partition_by_block_id,
id,
_partition_by_block_id AS partition_key,
VALUE :array_index :: INT AS tx_position,
DATA :result AS full_traces,
_inserted_timestamp
FROM
{% if is_incremental() %}
{% if is_incremental() and not full_reload_mode %}
{{ ref('bronze__streamline_traces') }}
WHERE
_inserted_timestamp >= (
@ -29,30 +33,54 @@ WHERE
FROM
{{ this }}
)
AND DATA :result IS NOT NULL
AND block_number > 22207817 {% elif is_incremental() and full_reload_mode %}
{{ ref('bronze__streamline_fr_traces') }}
WHERE
{% if use_partition_key %}
partition_key BETWEEN (
SELECT
MAX(partition_key) - 100000
FROM
{{ this }}
)
AND (
SELECT
MAX(partition_key) + 10000000
FROM
{{ this }}
)
{% else %}
_partition_by_block_id BETWEEN (
SELECT
MAX(_partition_by_block_id) - 100000
FROM
{{ this }}
)
AND (
SELECT
MAX(_partition_by_block_id) + 10000000
FROM
{{ this }}
)
{% endif %}
AND block_number > 22207817
{% else %}
{{ ref('bronze__streamline_fr_traces') }}
WHERE
_partition_by_block_id <= 5000000
_partition_by_block_id <= 30000000
AND block_number > 22207817
{% endif %}
),
debug_traces AS (
SELECT
block_number,
VALUE :array_index :: INT AS tx_position,
DATA :result AS full_traces,
_inserted_timestamp
FROM
bronze_traces
WHERE
block_number > 22207817
AND DATA :result IS NOT NULL qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position
ORDER BY
_inserted_timestamp DESC)) = 1
qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position
ORDER BY
_inserted_timestamp DESC)) = 1
),
flatten_traces AS (
SELECT
block_number,
tx_position,
partition_key,
IFF(
path IN (
'result',
@ -66,6 +94,7 @@ flatten_traces AS (
'result.output',
'result.error',
'result.revertReason',
'result.time',
'gasUsed',
'gas',
'type',
@ -75,6 +104,7 @@ flatten_traces AS (
'input',
'error',
'output',
'time',
'revertReason',
'afterEVMTransfers',
'beforeEVMTransfers',
@ -105,9 +135,9 @@ flatten_traces AS (
SPLIT(
trace_address,
'_'
) AS str_array
) AS trace_address_array
FROM
debug_traces txs,
bronze_traces txs,
TABLE(
FLATTEN(
input => PARSE_JSON(
@ -125,443 +155,26 @@ flatten_traces AS (
GROUP BY
block_number,
tx_position,
partition_key,
trace_address,
_inserted_timestamp
),
sub_traces AS (
SELECT
block_number,
tx_position,
parent_trace_address,
COUNT(*) AS sub_traces
FROM
flatten_traces
GROUP BY
block_number,
tx_position,
parent_trace_address
),
num_array AS (
SELECT
block_number,
tx_position,
trace_address,
ARRAY_AGG(flat_value) AS num_array
FROM
(
SELECT
block_number,
tx_position,
trace_address,
IFF(
VALUE :: STRING = 'ORIGIN',
-1,
VALUE :: INT
) AS flat_value
FROM
flatten_traces,
LATERAL FLATTEN (
input => str_array
)
)
GROUP BY
block_number,
tx_position,
trace_address
),
cleaned_traces AS (
SELECT
b.block_number,
b.tx_position,
b.trace_address,
IFNULL(
sub_traces,
0
) AS sub_traces,
num_array,
ROW_NUMBER() over (
PARTITION BY b.block_number,
b.tx_position
ORDER BY
num_array ASC
) - 1 AS trace_index,
trace_json,
b._inserted_timestamp
FROM
flatten_traces b
LEFT JOIN sub_traces s
ON b.block_number = s.block_number
AND b.tx_position = s.tx_position
AND b.trace_address = s.parent_trace_address
JOIN num_array n
ON b.block_number = n.block_number
AND b.tx_position = n.tx_position
AND b.trace_address = n.trace_address
),
final_debug_traces AS (
SELECT
tx_position,
trace_index,
block_number,
trace_address,
trace_json :error :: STRING AS error_reason,
trace_json :from :: STRING AS from_address,
trace_json :to :: STRING AS to_address,
IFNULL(
utils.udf_hex_to_int(
trace_json :value :: STRING
),
'0'
) AS eth_value_precise_raw,
utils.udf_decimal_adjust(
eth_value_precise_raw,
18
) AS eth_value_precise,
eth_value_precise :: FLOAT AS eth_value,
trace_json :afterEVMTransfers AS after_evm_transfers,
trace_json :beforeEVMTransfers AS before_evm_transfers,
utils.udf_hex_to_int(
trace_json :gas :: STRING
) :: INT AS gas,
utils.udf_hex_to_int(
trace_json :gasUsed :: STRING
) :: INT AS gas_used,
trace_json :input :: STRING AS input,
trace_json :output :: STRING AS output,
trace_json :type :: STRING AS TYPE,
concat_ws(
'_',
TYPE,
trace_address
) AS identifier,
concat_ws(
'-',
block_number,
tx_position,
identifier
) AS _call_id,
_inserted_timestamp,
trace_json AS DATA,
sub_traces
FROM
cleaned_traces
),
dedupe_arb_traces AS (
SELECT
*
FROM
bronze_traces
WHERE
block_number <= 22207817 qualify(ROW_NUMBER() over(PARTITION BY block_number, VALUE :array_index :: INT, DATA :transactionPosition :: INT
ORDER BY
_inserted_timestamp DESC)) = 1
),
arb_traces AS (
SELECT
DATA :transactionPosition :: INT AS tx_position,
block_number,
DATA :error :: STRING AS error_reason,
DATA :action :from :: STRING AS from_address,
COALESCE(
DATA :action :to :: STRING,
DATA :result :address :: STRING
) AS to_address,
IFNULL(
utils.udf_hex_to_int(
DATA :action :value :: STRING
),
'0'
) AS eth_value_precise_raw,
utils.udf_decimal_adjust(
eth_value_precise_raw,
18
) AS eth_value_precise,
eth_value_precise :: FLOAT AS eth_value,
utils.udf_hex_to_int(
DATA :action :gas :: STRING
) :: INT AS gas,
ROW_NUMBER() over (
PARTITION BY block_number,
tx_position
ORDER BY
VALUE :array_index :: INT ASC
) AS trace_index,
IFNULL(
utils.udf_hex_to_int(
DATA :result :gasUsed :: STRING
),
0
) :: INT AS gas_used,
COALESCE(
DATA :action :input :: STRING,
DATA :action :init :: STRING
) AS input,
COALESCE(
DATA :result :output :: STRING,
DATA :result :code :: STRING
) AS output,
UPPER(
COALESCE(
DATA :action :callType :: STRING,
DATA :type :: STRING
)
) AS TYPE,
CASE
WHEN trace_index = 1 THEN CONCAT(
TYPE,
'_',
'ORIGIN'
)
WHEN DATA :traceAddress :: STRING <> '[]' THEN CONCAT(
TYPE,
'_',
REPLACE(
REPLACE(REPLACE(DATA :traceAddress :: STRING, '['), ']'),
',',
'_'
)
)
ELSE TYPE
END AS identifier,
concat_ws(
'-',
block_number,
tx_position,
identifier
) AS _call_id,
_inserted_timestamp,
DATA,
DATA :subtraces :: INT AS sub_traces,
NULL AS after_evm_transfers,
NULL AS before_evm_transfers
FROM
dedupe_arb_traces
),
all_traces AS (
SELECT
tx_position,
trace_index,
block_number,
error_reason,
from_address,
to_address,
eth_value_precise_raw,
eth_value_precise,
eth_value,
gas,
gas_used,
input,
output,
TYPE,
identifier,
_call_id,
_inserted_timestamp,
DATA,
sub_traces,
after_evm_transfers,
before_evm_transfers
FROM
final_debug_traces
WHERE
identifier IS NOT NULL
UNION ALL
SELECT
tx_position,
trace_index,
block_number,
error_reason,
from_address,
to_address,
eth_value_precise_raw,
eth_value_precise,
eth_value,
gas,
gas_used,
input,
output,
TYPE,
identifier,
_call_id,
_inserted_timestamp,
DATA,
sub_traces,
after_evm_transfers,
before_evm_transfers
FROM
arb_traces
WHERE
identifier IS NOT NULL
),
new_records AS (
SELECT
f.block_number,
t.tx_hash,
t.block_timestamp,
t.tx_status,
f.tx_position,
f.trace_index,
f.from_address,
f.to_address,
f.eth_value_precise_raw,
f.eth_value_precise,
f.eth_value,
f.gas,
f.gas_used,
f.input,
f.output,
f.type,
f.identifier,
f.sub_traces,
f.after_evm_transfers,
f.before_evm_transfers,
f.error_reason,
IFF(
f.error_reason IS NULL,
'SUCCESS',
'FAIL'
) AS trace_status,
f.data,
IFF(
t.tx_hash IS NULL
OR t.block_timestamp IS NULL
OR t.tx_status IS NULL,
TRUE,
FALSE
) AS is_pending,
f._call_id,
f._inserted_timestamp
FROM
all_traces f
LEFT OUTER JOIN {{ ref('silver__transactions') }}
t
ON f.tx_position = t.position
AND f.block_number = t.block_number
{% if is_incremental() %}
AND t._INSERTED_TIMESTAMP >= (
SELECT
DATEADD('hour', -24, MAX(_inserted_timestamp))
FROM
{{ this }})
{% endif %}
)
{% if is_incremental() %},
missing_data AS (
SELECT
t.block_number,
txs.tx_hash,
txs.block_timestamp,
txs.tx_status,
t.tx_position,
t.trace_index,
t.from_address,
t.to_address,
t.eth_value_precise_raw,
t.eth_value_precise,
t.eth_value,
t.gas,
t.gas_used,
t.input,
t.output,
t.type,
t.identifier,
t.sub_traces,
t.after_evm_transfers,
t.before_evm_transfers,
t.error_reason,
t.trace_status,
t.data,
FALSE AS is_pending,
t._call_id,
GREATEST(
t._inserted_timestamp,
txs._inserted_timestamp
) AS _inserted_timestamp
FROM
{{ this }}
t
INNER JOIN {{ ref('silver__transactions') }}
txs
ON t.tx_position = txs.position
AND t.block_number = txs.block_number
WHERE
t.is_pending
)
{% endif %},
FINAL AS (
SELECT
block_number,
tx_hash,
block_timestamp,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
eth_value_precise_raw,
eth_value_precise,
eth_value,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
trace_status,
after_evm_transfers,
before_evm_transfers,
DATA,
is_pending,
_call_id,
_inserted_timestamp
FROM
new_records
{% if is_incremental() %}
UNION
SELECT
block_number,
tx_hash,
block_timestamp,
tx_status,
tx_position,
trace_index,
from_address,
to_address,
eth_value_precise_raw,
eth_value_precise,
eth_value,
gas,
gas_used,
input,
output,
TYPE,
identifier,
sub_traces,
error_reason,
trace_status,
after_evm_transfers,
before_evm_transfers,
DATA,
is_pending,
_call_id,
_inserted_timestamp
FROM
missing_data
{% endif %}
)
SELECT
*,
trace_address,
parent_trace_address,
trace_address_array,
trace_json,
partition_key,
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
['block_number', 'tx_position', 'trace_address']
) }} AS traces_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index
flatten_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id
ORDER BY
_inserted_timestamp DESC, is_pending ASC)) = 1
_inserted_timestamp DESC)) = 1

15
models/silver/core/silver__traces2.sql
View File

@ -1,15 +0,0 @@
-- depends_on: {{ ref('bronze__streamline_traces') }}
{{ config (
materialized = "incremental",
incremental_strategy = 'delete+insert',
unique_key = "block_number",
cluster_by = ['modified_timestamp::DATE','partition_key'],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
full_refresh = false,
tags = ['core','non_realtime']
) }}
{{ fsc_evm.silver_traces_v1(
full_reload_start_block = 30000000,
full_reload_blocks = 10000000,
arb_traces_mode = true
) }}

73
models/silver/core/tests/traces/test_silver__traces_full.yml
View File

@ -1,59 +1,52 @@
version: 2
models:
- name: test_silver__traces_full
description: "This is a view used to test all of the silver traces model."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- BLOCK_NUMBER
- TX_POSITION
- TRACE_INDEX
- TRACES_ID
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
- not_null
- name: TX_POSITION
tests:
- not_null:
where: NOT IS_PENDING
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: TRACE_JSON
tests:
- not_null
- name: _INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: day
interval: 1
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
tests:
- not_null:
where: NOT IS_PENDING
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: FROM_ADDRESS
tests:
- not_null:
where: TYPE <> 'SELFDESTRUCT'
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: TO_ADDRESS
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
where: TO_ADDRESS IS NOT NULL
- name: IDENTIFIER
- name: INSERTED_TIMESTAMP
tests:
- not_null
- name: ETH_VALUE
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- name: GAS
tests:
- not_null
- name: GAS_USED
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ

2
models/silver/core/tests/traces/test_silver__traces_recent.sql
View File

@ -20,4 +20,4 @@ WHERE
block_number
FROM
last_3_days
)
)

55
models/silver/core/tests/traces/test_silver__traces_recent.yml
View File

@ -1,35 +1,52 @@
version: 2
models:
- name: test_silver__traces_recent
description: "This is a view used to test the last three days of traces."
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- BLOCK_NUMBER
- TX_POSITION
- TRACE_INDEX
- TRACES_ID
columns:
- name: BLOCK_NUMBER
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
- not_null
- name: TX_POSITION
tests:
- not_null:
where: NOT IS_PENDING
- not_null
- name: TRACE_ADDRESS
tests:
- not_null
- name: TRACE_JSON
tests:
- not_null
- name: _INSERTED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: day
interval: 1
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: TX_HASH
- name: INSERTED_TIMESTAMP
tests:
- not_null:
where: NOT IS_PENDING
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ
- name: MODIFIED_TIMESTAMP
tests:
- not_null
- dbt_expectations.expect_row_values_to_have_recent_data:
datepart: hour
interval: 2
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_LTZ
- TIMESTAMP_NTZ

13
models/silver/core/traces2_fix/silver__fact_traces2_fix.sql
View File

@ -1,12 +1,11 @@
{{ config (
materialized = "incremental",
incremental_strategy = 'delete+insert',
unique_key = ["block_number", "tx_position", "trace_address"],
tags = ['traces_fix']
{# {{ config (
materialized = "incremental",
incremental_strategy = 'delete+insert',
unique_key = ["block_number", "tx_position", "trace_address"],
tags = ['traces_fix']
) }}
{% set batch_query %}
SELECT
MAX(next_batch_id) AS next_batch_id
FROM
@ -161,4 +160,4 @@ aggregated_errors AS (
prod_trace_succeeded
FROM
batch
CROSS JOIN final_errors
CROSS JOIN final_errors #}

15
models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql
View File

@ -13,10 +13,19 @@ WITH base_contracts AS (
block_timestamp,
from_address,
to_address AS contract_address,
_call_id,
_inserted_timestamp
concat_ws(
'-',
block_number,
tx_position,
CONCAT(
TYPE,
'_',
trace_address
)
) AS _call_id,
modified_timestamp AS _inserted_timestamp
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
from_address = '0x55bdb4164d28fbaf0898e0ef14a589ac09ac9970'
AND TYPE ILIKE 'create%'

15
models/silver/defi/dex/curve/silver_dex__curve_pools.sql
View File

@ -14,15 +14,24 @@ WITH contract_deployments AS (
block_timestamp,
from_address AS deployer_address,
to_address AS contract_address,
_call_id,
_inserted_timestamp,
concat_ws(
'-',
block_number,
tx_position,
CONCAT(
TYPE,
'_',
trace_address
)
) AS _call_id,
modified_timestamp AS _inserted_timestamp,
ROW_NUMBER() over (
ORDER BY
contract_address
) AS row_num
FROM
{{ ref(
'silver__traces'
'core__fact_traces'
) }}
WHERE
-- curve contract deployers

15
models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql
View File

@ -13,10 +13,19 @@ WITH contract_deployments AS (
block_timestamp,
from_address AS deployer_address,
to_address AS contract_address,
_call_id,
_inserted_timestamp
concat_ws(
'-',
block_number,
tx_position,
CONCAT(
TYPE,
'_',
trace_address
)
) AS _call_id,
modified_timestamp AS _inserted_timestamp
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
from_address IN (
'0xe43632337d3f9a52ffd098fe71a57cc5961c041f',

3
models/silver/defi/lending/dforce/silver__dforce_asset_details.sql
View File

@ -24,6 +24,7 @@ WITH log_pull AS (
LOWER('0x655284bebcc6e1dffd098ec538750d43b57bc743'),
LOWER('0xde6d6f23aabbdc9469c8907ece7c379f98e4cb75')
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
@ -39,7 +40,7 @@ traces_pull AS (
from_address AS token_address,
to_address AS underlying_asset
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
tx_hash IN (
SELECT

8
models/silver/defi/lending/lodestar/silver__lodestar_asset_details.sql
View File

@ -21,6 +21,7 @@ WITH log_pull AS (
WHERE
topics [0] = '0x7ac369dbd14fa5ea3f473ed67cc9d598964a77501540ba6751eb0b3decf5870d'
AND origin_from_address = '0x0f01756bc6183994d90773c8f22e3f44355ffa0e'
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
@ -36,7 +37,7 @@ traces_pull AS (
from_address AS token_address,
to_address AS underlying_asset
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
tx_hash IN (
SELECT
@ -45,14 +46,15 @@ traces_pull AS (
log_pull
)
AND identifier = 'STATICCALL_0_2'
{% if is_incremental() %}
AND _inserted_timestamp >= (
AND modified_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '12 hours'
FROM
{{ this }}
)
AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day'
AND modified_timestamp >= SYSDATE() - INTERVAL '7 day'
{% endif %}
),
contracts AS (

10
models/silver/nft/sales/silver__treasure_sales.sql
View File

@ -78,7 +78,7 @@ eth_payment AS (
tx_hash,
from_address,
to_address,
eth_value,
VALUE AS eth_value,
trace_index,
IFF(
to_address = '0xdb6ab450178babcf0e467c1f3b436050d907e233',
@ -99,11 +99,11 @@ eth_payment AS (
ELSE NULL
END AS intra_tx_grouping_raw
FROM
{{ ref('silver__traces') }}
{{ ref('core__fact_traces') }}
WHERE
block_timestamp :: DATE >= '2021-11-01'
AND from_address = '0x09986b4e255b3c548041a30a2ee312fe176731c2'
AND eth_value > 0
AND VALUE > 0
AND identifier != 'CALL_ORIGIN'
AND TYPE != 'DELEGATECALL'
AND trace_status = 'SUCCESS'
@ -118,13 +118,13 @@ eth_payment AS (
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
AND modified_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '12 hours'
FROM
{{ this }}
)
AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day'
AND modified_timestamp >= SYSDATE() - INTERVAL '7 day'
{% endif %}
),
eth_payment_intra_fill AS (

1
models/sources.yml
View File

@ -40,6 +40,7 @@ sources:
schema: silver
tables:
- name: verified_abis
- name: traces
- name: arbitrum_bronze_api
database: arbitrum
schema: bronze_api

2
models/streamline/silver/core/retry/_missing_traces.sql
View File

@ -14,7 +14,7 @@ SELECT
FROM
{{ ref("silver__transactions") }}
tx
LEFT JOIN {{ ref("silver__traces") }}
LEFT JOIN {{ ref("core__fact_traces") }}
tr
ON tx.block_number = tr.block_number
AND tx.tx_hash = tr.tx_hash

2
tests/arbitrum/test_silver__missing_traces.sql → tests/arbitrum/test_gold__missing_traces.sql
View File

@ -1 +1 @@
{{ missing_traces(ref("test_silver__transactions_full"), ref("test_silver__traces_full")) }}
{{ missing_traces(ref("test_silver__transactions_full"), ref("test_gold__fact_traces_full")) }}

2
tests/arbitrum/test_silver__recent_missing_traces.sql → tests/arbitrum/test_gold__recent_missing_traces.sql
View File

@ -1 +1 @@
{{ missing_traces(ref("test_silver__transactions_recent"), ref("test_silver__traces_recent")) }}
{{ missing_traces(ref("test_silver__transactions_recent"), ref("test_gold__fact_traces_recent")) }}