Merge pull request #65 from FlipsideCrypto/traces

Traces

.github/workflows/dbt_run_adhoc.yml

@@ -7,7 +7,7 @@ on:
       - "main"
     inputs:
       environment:
-        type: choice 
+        type: choice
         description: DBT Run Environment
         required: true
         options:
@@ -15,9 +15,9 @@ on:
           - prod
         default: dev
       warehouse:
-        type: choice 
+        type: choice
         description: Snowflake warehouse
-        required: true 
+        required: true
         options:
           - DBT
           - DBT_CLOUD
@@ -25,9 +25,9 @@ on:
         default: DBT
       dbt_command:
         type: string
-        description: 'DBT Run Command'
+        description: "DBT Run Command"
         required: true
-    
+
 env:
   USE_VARS: "${{ vars.USE_VARS }}"
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
@@ -47,7 +47,7 @@ concurrency:
 jobs:
   run_dbt_jobs:
     runs-on: ubuntu-latest
-    environment: 
+    environment:
       name: workflow_${{ inputs.environment }}
 
     steps:

dbt_project.yml

@@ -68,5 +68,5 @@ dispatch:
       - dbt
 
 query-comment:
-  comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}'
+  comment: "{{ dbt_snowflake_query_tags.get_query_comment(node) }}"
   append: true # Snowflake removes prefixed comments.

models/descriptions/precise_amounts.md

@@ -0,0 +1,17 @@
+{% docs precise_amount_unadjusted %}
+
+The precise, unadjusted amount of the transaction. This is returned as a string to avoid precision loss.
+
+{% enddocs %}
+
+{% docs precise_amount_adjusted %}
+
+The precise, adjusted amount of the transaction. This is returned as a string to avoid precision loss.
+
+{% enddocs %}
+
+{% docs tx_fee_precise %}
+
+The precise amount of the transaction fee. This is returned as a string to avoid precision loss.
+
+{% enddocs %}

models/descriptions/traces_block_no.md

@@ -0,0 +1,5 @@
+{% docs traces_block_no %}
+
+The block number of this transaction.
+
+{% enddocs %}

models/descriptions/traces_blocktime.md

@@ -0,0 +1,5 @@
+{% docs traces_blocktime %}
+
+The block timestamp of this transaction.
+
+{% enddocs %}

models/descriptions/traces_call_data.md

@@ -0,0 +1,5 @@
+{% docs traces_call_data %}
+
+The raw JSON data for this trace.
+
+{% enddocs %}

models/descriptions/traces_from.md

@@ -0,0 +1,5 @@
+{% docs traces_from %}
+
+The sending address of this trace. This is not necessarily the from address of the transaction.
+
+{% enddocs %}

models/descriptions/traces_gas.md

@@ -0,0 +1,5 @@
+{% docs traces_gas %}
+
+The gas supplied for this trace.
+
+{% enddocs %}

models/descriptions/traces_gas_used.md

@@ -0,0 +1,5 @@
+{% docs traces_gas_used %}
+
+The gas used for this trace.
+
+{% enddocs %}

models/descriptions/traces_identifier.md

@@ -0,0 +1,5 @@
+{% docs traces_identifier %}
+
+This field represents the position and type of the trace within the transaction.
+
+{% enddocs %}

models/descriptions/traces_index.md

@@ -0,0 +1,5 @@
+{% docs trace_index %}
+
+The index of the trace within the transaction.
+
+{% enddocs %}

models/descriptions/traces_input.md

@@ -0,0 +1,5 @@
+{% docs traces_input %}
+
+The input data for this trace.
+
+{% enddocs %}

models/descriptions/traces_output.md

@@ -0,0 +1,5 @@
+{% docs traces_output %}
+
+The output data for this trace.
+
+{% enddocs %}

models/descriptions/traces_sub.md

@@ -0,0 +1,5 @@
+{% docs traces_sub %}
+
+The amount of nested sub traces for this trace.
+
+{% enddocs %}

models/descriptions/traces_table_doc.md

@@ -0,0 +1,5 @@
+{% docs traces_table_doc %}
+
+This table contains flattened trace data for internal contract calls on the Ethereum blockchain. Hex encoded fields can be decoded to integers by using `TO_NUMBER(<FIELD>, 'XXXXXXXXXXXX')`, with the number of Xs being the max length of the encoded field. You must also remove the `0x` from your field to use the `TO_NUMBER()` function, if applicable.
+
+{% enddocs %}

models/descriptions/traces_to.md

@@ -0,0 +1,5 @@
+{% docs traces_to %}
+
+The receiving address of this trace. This is not necessarily the to address of the transaction.
+
+{% enddocs %}

models/descriptions/traces_tx_hash.md

@@ -0,0 +1,5 @@
+{% docs traces_tx_hash %}
+
+The transaction hash for the trace. Please note, this is not necessarily unique in this table as transactions frequently have multiple traces.
+
+{% enddocs %}

models/descriptions/traces_type.md

@@ -0,0 +1,5 @@
+{% docs traces_type %}
+
+The type of internal transaction. Common trace types are `CALL`, `DELEGATECALL`, and `STATICCALL`.
+
+{% enddocs %}

models/descriptions/traces_value.md

@@ -0,0 +1,5 @@
+{% docs traces_value %}
+
+The amount of ETH transferred in this trace.
+
+{% enddocs %}

models/gold/core/core__fact_receipts.sql

@@ -36,4 +36,4 @@ SELECT
         _inserted_timestamp
     ) AS modified_timestamp
 FROM
-    {{ ref('silver__receipts') }}
+    {{ ref('silver__receipts') }}

models/gold/core/core__fact_traces.sql

@@ -0,0 +1,44 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    tags = ['core']
+) }}
+
+SELECT
+    tx_hash,
+    block_number,
+    block_timestamp,
+    from_address,
+    to_address,
+    value,
+    value_precise_raw,
+    value_precise,
+    gas,
+    gas_used,
+    input,
+    output,
+    TYPE,
+    identifier,
+    DATA,
+    tx_status,
+    sub_traces,
+    trace_status,
+    error_reason,
+    trace_index,
+    COALESCE (
+        traces_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_hash', 'trace_index']
+        ) }}
+    ) AS fact_traces_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
+FROM
+    {{ ref('silver__traces') }}

models/gold/core/core__fact_traces.yml

@@ -0,0 +1,52 @@
+version: 2
+models:
+  - name: core__fact_traces
+    description: '{{ doc("traces_table_doc") }}'
+
+    columns:
+      - name: BLOCK_NUMBER
+        description: '{{ doc("traces_block_no") }}'
+      - name: BLOCK_TIMESTAMP
+        description: '{{ doc("traces_blocktime") }}'
+      - name: TX_HASH
+        description: '{{ doc("traces_tx_hash") }}'
+      - name: FROM_ADDRESS
+        description: '{{ doc("traces_from") }}'
+      - name: TO_ADDRESS
+        description: '{{ doc("traces_to") }}'
+      - name: VALUE
+        description: '{{ doc("traces_value") }}'
+      - name: VALUE_PRECISE_RAW
+        description: '{{ doc("precise_amount_unadjusted") }}'
+      - name: VALUE_PRECISE
+        description: '{{ doc("precise_amount_adjusted") }}'
+      - name: GAS
+        description: '{{ doc("traces_gas") }}'
+      - name: GAS_USED
+        description: '{{ doc("traces_gas_used") }}'
+      - name: INPUT
+        description: '{{ doc("traces_input") }}'
+      - name: OUTPUT
+        description: '{{ doc("traces_output") }}'
+      - name: TYPE
+        description: '{{ doc("traces_type") }}'
+      - name: IDENTIFIER
+        description: '{{ doc("traces_identifier") }}'
+      - name: DATA
+        description: '{{ doc("traces_call_data") }}'
+      - name: TX_STATUS
+        description: '{{ doc("tx_status") }}'
+      - name: SUB_TRACES
+        description: '{{ doc("traces_sub") }}'
+      - name: TRACE_STATUS
+        description: The status of the trace, either `SUCCESS` or `FAIL`
+      - name: ERROR_REASON
+        description: The reason for the trace failure, if any.
+      - name: TRACE_INDEX
+        description: The index of the trace within the transaction.
+      - name: FACT_TRACES_ID
+        description: '{{ doc("pk_id") }}'
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}'

models/silver/silver__traces.sql

@@ -0,0 +1,413 @@
+-- depends_on: {{ ref('bronze__streamline_traces') }}
+{{ config (
+    materialized = "incremental",
+    incremental_strategy = 'delete+insert',
+    unique_key = "block_number",
+    cluster_by = "block_timestamp::date, _inserted_timestamp::date",
+    tags = ['core']
+) }}
+
+WITH bronze_traces AS (
+
+    SELECT
+        block_number,
+        VALUE :metadata :request :params [0] :: STRING AS tx_hash,
+        DATA :result AS full_traces,
+        _inserted_timestamp
+    FROM
+
+{% if is_incremental() %}
+{{ ref('bronze__streamline_traces') }}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp) _inserted_timestamp
+        FROM
+            {{ this }}
+    )
+{% else %}
+    {{ ref('bronze__streamline_FR_traces') }}
+{% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash
+ORDER BY
+    _inserted_timestamp DESC)) = 1
+),
+flatten_traces AS (
+    SELECT
+        block_number,
+        tx_hash,
+        IFF(
+            path IN (
+                'result',
+                'result.value',
+                'result.type',
+                'result.to',
+                'result.input',
+                'result.gasUsed',
+                'result.gas',
+                'result.from',
+                'result.output',
+                'result.error',
+                'result.revertReason',
+                'gasUsed',
+                'gas',
+                'type',
+                'to',
+                'from',
+                'value',
+                'input',
+                'error',
+                'output',
+                'revertReason'
+            ),
+            'ORIGIN',
+            REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '')
+        ) AS trace_address,
+        _inserted_timestamp,
+        OBJECT_AGG(
+            key,
+            VALUE
+        ) AS trace_json,
+        CASE
+            WHEN trace_address = 'ORIGIN' THEN NULL
+            WHEN POSITION(
+                '_' IN trace_address
+            ) = 0 THEN 'ORIGIN'
+            ELSE REGEXP_REPLACE(
+                trace_address,
+                '_[0-9]+$',
+                '',
+                1,
+                1
+            )
+        END AS parent_trace_address,
+        SPLIT(
+            trace_address,
+            '_'
+        ) AS str_array
+    FROM
+        bronze_traces txs,
+        TABLE(
+            FLATTEN(
+                input => PARSE_JSON(
+                    txs.full_traces
+                ),
+                recursive => TRUE
+            )
+        ) f
+    WHERE
+        f.index IS NULL
+        AND f.key != 'calls'
+        AND f.path != 'result'
+    GROUP BY
+        block_number,
+        tx_hash,
+        trace_address,
+        _inserted_timestamp
+),
+sub_traces AS (
+    SELECT
+        block_number,
+        tx_hash,
+        parent_trace_address,
+        COUNT(*) AS sub_traces
+    FROM
+        flatten_traces
+    GROUP BY
+        block_number,
+        tx_hash,
+        parent_trace_address
+),
+num_array AS (
+    SELECT
+        block_number,
+        tx_hash,
+        trace_address,
+        ARRAY_AGG(flat_value) AS num_array
+    FROM
+        (
+            SELECT
+                block_number,
+                tx_hash,
+                trace_address,
+                IFF(
+                    VALUE :: STRING = 'ORIGIN',
+                    -1,
+                    VALUE :: INT
+                ) AS flat_value
+            FROM
+                flatten_traces,
+                LATERAL FLATTEN (
+                    input => str_array
+                )
+        )
+    GROUP BY
+        block_number,
+        tx_hash,
+        trace_address
+),
+cleaned_traces AS (
+    SELECT
+        b.block_number,
+        b.tx_hash,
+        b.trace_address,
+        IFNULL(
+            sub_traces,
+            0
+        ) AS sub_traces,
+        num_array,
+        ROW_NUMBER() over (
+            PARTITION BY b.block_number,
+            b.tx_hash
+            ORDER BY
+                num_array ASC
+        ) - 1 AS trace_index,
+        trace_json,
+        b._inserted_timestamp
+    FROM
+        flatten_traces b
+        LEFT JOIN sub_traces s
+        ON b.block_number = s.block_number
+        AND b.tx_hash = s.tx_hash
+        AND b.trace_address = s.parent_trace_address
+        JOIN num_array n
+        ON b.block_number = n.block_number
+        AND b.tx_hash = n.tx_hash
+        AND b.trace_address = n.trace_address
+),
+final_traces AS (
+    SELECT
+        tx_hash,
+        trace_index,
+        block_number,
+        trace_address,
+        trace_json :error :: STRING AS error_reason,
+        trace_json :from :: STRING AS from_address,
+        trace_json :to :: STRING AS to_address,
+        IFNULL(
+            utils.udf_hex_to_int(
+                trace_json :value :: STRING
+            ),
+            '0'
+        ) AS value_precise_raw,
+        utils.udf_decimal_adjust(
+            value_precise_raw,
+            18
+        ) AS value_precise,
+        value_precise :: FLOAT AS value,
+        utils.udf_hex_to_int(
+            trace_json :gas :: STRING
+        ) :: INT AS gas,
+        utils.udf_hex_to_int(
+            trace_json :gasUsed :: STRING
+        ) :: INT AS gas_used,
+        trace_json :input :: STRING AS input,
+        trace_json :output :: STRING AS output,
+        trace_json :type :: STRING AS TYPE,
+        concat_ws(
+            '_',
+            TYPE,
+            trace_address
+        ) AS identifier,
+        concat_ws(
+            '-',
+            block_number,
+            tx_hash,
+            identifier
+        ) AS _call_id,
+        _inserted_timestamp,
+        trace_json AS DATA,
+        sub_traces
+    FROM
+        cleaned_traces
+),
+new_records AS (
+    SELECT
+        f.block_number,
+        t.tx_hash,
+        t.block_timestamp,
+        t.tx_status,
+        t.position,
+        f.trace_index,
+        f.from_address,
+        f.to_address,
+        f.value_precise_raw,
+        f.value_precise,
+        f.value,
+        f.gas,
+        f.gas_used,
+        f.input,
+        f.output,
+        f.type,
+        f.identifier,
+        f.sub_traces,
+        f.error_reason,
+        IFF(
+            f.error_reason IS NULL,
+            'SUCCESS',
+            'FAIL'
+        ) AS trace_status,
+        f.data,
+        IFF(
+            t.tx_hash IS NULL
+            OR t.block_timestamp IS NULL
+            OR t.tx_status IS NULL,
+            TRUE,
+            FALSE
+        ) AS is_pending,
+        f._call_id,
+        f._inserted_timestamp
+    FROM
+        final_traces f
+        LEFT OUTER JOIN {{ ref('silver__transactions') }}
+        t
+        ON f.tx_hash = t.tx_hash
+        AND f.block_number = t.block_number
+
+{% if is_incremental() %}
+AND t._INSERTED_TIMESTAMP >= (
+    SELECT
+        DATEADD('hour', -24, MAX(_inserted_timestamp))
+    FROM
+        {{ this }})
+    {% endif %}
+)
+
+{% if is_incremental() %},
+missing_data AS (
+    SELECT
+        t.block_number,
+        txs.tx_hash,
+        txs.block_timestamp,
+        txs.tx_status,
+        txs.position,
+        t.trace_index,
+        t.from_address,
+        t.to_address,
+        t.value_precise_raw,
+        t.value_precise,
+        t.value,
+        t.gas,
+        t.gas_used,
+        t.input,
+        t.output,
+        t.type,
+        t.identifier,
+        t.sub_traces,
+        t.error_reason,
+        t.trace_status,
+        t.data,
+        FALSE AS is_pending,
+        t._call_id,
+        GREATEST(
+            t._inserted_timestamp,
+            txs._inserted_timestamp
+        ) AS _inserted_timestamp
+    FROM
+        {{ this }}
+        t
+        INNER JOIN {{ ref('silver__transactions') }}
+        txs
+        ON t.tx_hash = txs.tx_hash
+        AND t.block_number = txs.block_number
+    WHERE
+        t.is_pending
+)
+{% endif %},
+FINAL AS (
+    SELECT
+        block_number,
+        tx_hash,
+        block_timestamp,
+        tx_status,
+        position,
+        trace_index,
+        from_address,
+        to_address,
+        value_precise_raw,
+        value_precise,
+        value,
+        gas,
+        gas_used,
+        input,
+        output,
+        TYPE,
+        identifier,
+        sub_traces,
+        error_reason,
+        trace_status,
+        DATA,
+        is_pending,
+        _call_id,
+        _inserted_timestamp
+    FROM
+        new_records
+
+{% if is_incremental() %}
+UNION
+SELECT
+    block_number,
+    tx_hash,
+    block_timestamp,
+    tx_status,
+    position,
+    tx_hash,
+    trace_index,
+    from_address,
+    to_address,
+    value_precise_raw,
+    value_precise,
+    value,
+    gas,
+    gas_used,
+    input,
+    output,
+    TYPE,
+    identifier,
+    sub_traces,
+    error_reason,
+    trace_status,
+    DATA,
+    is_pending,
+    _call_id,
+    _inserted_timestamp
+FROM
+    missing_data
+{% endif %}
+)
+SELECT
+    block_number,
+    tx_hash,
+    block_timestamp,
+    tx_status,
+    position AS tx_position,
+    trace_index,
+    from_address,
+    to_address,
+    value_precise,
+    value,
+    gas,
+    gas_used,
+    input,
+    output,
+    TYPE,
+    identifier,
+    sub_traces,
+    error_reason,
+    trace_status,
+    DATA,
+    is_pending,
+    _call_id,
+    _inserted_timestamp,
+    value_precise_raw,
+    {{ dbt_utils.generate_surrogate_key(
+        ['tx_hash', 'trace_index']
+    ) }} AS traces_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_hash, trace_index
+ORDER BY
+    _inserted_timestamp DESC, is_pending ASC)) = 1

models/silver/test/traces/test_silver__traces_full.sql

@@ -0,0 +1,9 @@
+{{ config (
+    materialized = 'view',
+    tags = ['full_test']
+) }}
+
+SELECT
+    *
+FROM
+    {{ ref('silver__traces') }}

models/silver/test/traces/test_silver__traces_full.yml

@@ -0,0 +1,57 @@
+version: 2
+models:
+  - name: test_silver__traces_full
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_NUMBER
+            - TX_POSITION
+            - TRACE_INDEX
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: BLOCK_TIMESTAMP
+        tests:
+          - not_null:
+              where: NOT IS_PENDING
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ
+      - name: TX_HASH
+        tests:
+          - not_null:
+              where: NOT IS_PENDING
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: FROM_ADDRESS
+        tests:
+          - not_null:
+              where: TYPE <> 'SELFDESTRUCT'
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: TO_ADDRESS
+        tests:
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+              where: TO_ADDRESS IS NOT NULL
+      - name: IDENTIFIER
+        tests:
+          - not_null
+      - name: ETH_VALUE
+        tests:
+          - not_null
+      - name: GAS
+        tests:
+          - not_null
+      - name: GAS_USED
+        tests:
+          - not_null

models/silver/test/traces/test_silver__traces_recent.sql

@@ -0,0 +1,27 @@
+{{ config (
+    materialized = 'view',
+    tags = ['recent_test']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_number
+    FROM
+        {{ ref("_max_block_by_date") }}
+        qualify ROW_NUMBER() over (
+            ORDER BY
+                block_number DESC
+        ) = 3
+)
+SELECT
+    *
+FROM
+    {{ ref('silver__traces') }}
+WHERE
+    block_number >= (
+        SELECT
+            block_number
+        FROM
+            last_3_days
+    )

models/silver/test/traces/test_silver__traces_recent.yml

@@ -0,0 +1,34 @@
+version: 2
+models:
+  - name: test_silver__traces_recent
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_NUMBER
+            - TX_POSITION
+            - TRACE_INDEX
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: BLOCK_TIMESTAMP
+        tests:
+          - not_null:
+              where: NOT IS_PENDING
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ
+      - name: TX_HASH
+        tests:
+          - not_null:
+              where: NOT IS_PENDING
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+