An 1643/labels (#2)

* docs, labels, db comments

* traces fix

* add docs

dbt_project.yml

@@ -27,6 +27,8 @@ clean-targets: # directories to be removed by `dbt clean`
 on-run-start:
   - "{{ create_sps() }}"
   - "{{ create_udfs() }}"
+  # - "{{ db_comment() }}"
+  
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models

macros/db_comment.sql

@@ -0,0 +1,13 @@
+-- {% macro db_comment() %}
+--     {% set query %}
+-- SELECT
+--     TO_DATE(MIN(block_timestamp))
+-- FROM
+--     silver.blocks {% endset %}
+--     {% set results = run_query(query) %}
+--     {% set results_list = results.columns [0].values() [0].strftime('%Y-%m-%d') %}
+--     {% set sql %}
+--     COMMENT
+--     ON database arbitrum_dev IS 'Lite Mode dataset with recent data only. Min block_timestamp: {{ results_list }} 🌱 ' {% endset %}
+--     {% do run_query(sql) %}
+-- {% endmacro %}

models/doc_descriptions/general/__overview__.md

@@ -2,6 +2,8 @@
 
 # Welcome to the Flipside Crypto Arbitrum Models Documentation!
 
+# NOTE: Data is in 'lite mode' - meaning, histrorical data has not yet been backfilled. Please see `min(block_timestamp)`
+
 ## **What is Flipside?**
 
 [Flipside Crypto](https://flipsidecrypto.xyz/earn) provides Community Enabled Crypto Analytics, allowing our users to create and share data insights on the crypto projects they care most about.
@@ -23,10 +25,11 @@ The documentation included here details the design of the Arbitrum tables and vi
 
 ### **Quick Links to Table Documentation**
 
-- [fact_blocks]()
-- [fact_event_logs]()
-- [fact_traces]()
-- [fact_transactions]()
+- [fact_blocks](https://flipsidecrypto.github.io/arbitrum-models/#!/model/model.arbitrum_models.core__fact_blocks)
+- [fact_event_logs](https://flipsidecrypto.github.io/arbitrum-models/#!/model/model.arbitrum_models.core__fact_event_logs)
+- [fact_traces](https://flipsidecrypto.github.io/arbitrum-models/#!/model/model.arbitrum_models.core__fact_traces)
+- [fact_transactions](https://flipsidecrypto.github.io/arbitrum-models/#!/model/model.arbitrum_models.core__fact_transactions)
+- [dim_labels](https://flipsidecrypto.github.io/arbitrum-models/#!/model/model.arbitrum_models.core__dim_labels)
 
 ## **Data Model Overview**
 

models/doc_descriptions/labels/arb_label.md

@@ -0,0 +1,5 @@
+{% docs arb_project_name %}
+
+The name of the project for this address. 
+
+{% enddocs %}

models/doc_descriptions/labels/arb_label_address.md

@@ -0,0 +1,5 @@
+{% docs arb_label_address %}
+
+Address that the label is for. This is the field that should be used to join other tables with labels. 
+
+{% enddocs %}

models/doc_descriptions/labels/arb_label_address_name.md

@@ -0,0 +1,5 @@
+{% docs arb_label_address_name %}
+
+The most granular label for this address.  
+
+{% enddocs %}

models/doc_descriptions/labels/arb_label_blockchain.md

@@ -0,0 +1,5 @@
+{% docs arb_label_blockchain %}
+
+The name of the blockchain.
+
+{% enddocs %}

models/doc_descriptions/labels/arb_label_creator.md

@@ -0,0 +1,5 @@
+{% docs arb_label_creator %}
+
+The name of the creator of the label.
+
+{% enddocs %}

models/doc_descriptions/labels/arb_label_subtype.md

@@ -0,0 +1,5 @@
+{% docs arb_label_subtype %}
+
+A sub-category nested within label type providing further detail.
+
+{% enddocs %}

models/doc_descriptions/labels/arb_label_type.md

@@ -0,0 +1,5 @@
+{% docs arb_label_type %}
+
+A high-level category describing the addresses main function or ownership.
+
+{% enddocs %}

models/doc_descriptions/labels/arb_labels_table_doc.md

@@ -0,0 +1,5 @@
+{% docs arb_labels_table %}
+
+This table contains labels for addresses on the Optimism Blockchain. 
+
+{% enddocs %}

models/gold/core__dim_labels.sql

@@ -0,0 +1,22 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    blockchain,
+    creator,
+    address,
+    address_name,
+    label_type,
+    label_subtype,
+    project_name
+FROM
+    {{ source(
+        'crosschain',
+        'address_labels'
+    ) }}
+WHERE
+    blockchain = 'arbitrum'
+    AND address LIKE '0x%'

models/gold/core__dim_labels.yml

@@ -0,0 +1,57 @@
+version: 2
+models:
+  - name: core__dim_labels
+    description: '{{ doc("arb_labels_table") }}'
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCKCHAIN
+            - CREATOR
+            - ADDRESS
+    columns:
+      - name: BLOCKCHAIN
+        description: '{{ doc("arb_label_blockchain") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set: ['arbitrum']
+      - name: CREATOR
+        description: '{{ doc("arb_label_creator") }}'
+        tests:
+          - not_null
+      - name: ADDRESS
+        description: '{{ doc("arb_label_address") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: ADDRESS_NAME
+        description: '{{ doc("arb_labels_table") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: LABEL_TYPE
+        description: '{{ doc("arb_label_type") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_set:
+              value_set: ['flotsam', 'nft', 'defi', 'dex', 'cex', 'dapp', 'token', 'operator', 'layer2', 'chadmin', 'project']
+      - name: LABEL_SUBTYPE
+        description: '{{ doc("arb_label_subtype") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: PROJECT_NAME
+        description: '{{ doc("arb_project_name") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+

models/gold/core__fact_traces.sql

@@ -18,7 +18,6 @@ SELECT
     TYPE,
     identifier,
     DATA,
-    tx_status,
-    sub_traces
+    tx_status
 FROM
     {{ ref('silver__traces') }}

models/gold/core__fact_traces.yml

@@ -32,8 +32,6 @@ models:
         description: '{{ doc("arb_traces_call_data") }}'
       - name: TX_STATUS
         description: '{{ doc("arb_tx_status") }}' 
-      - name: SUB_TRACES
-        description: '{{ doc("arb_traces_sub") }}' 
 
 
 

models/silver/silver__traces.sql

@@ -40,182 +40,79 @@ LIMIT
     ORDER BY
         _inserted_timestamp DESC)) = 1
 ),
-base_table AS (
+traces_raw AS (
     SELECT
-        CASE
-            WHEN POSITION(
-                '.',
-                path :: STRING
-            ) > 0 THEN REPLACE(
-                REPLACE(
-                    path :: STRING,
-                    SUBSTR(path :: STRING, len(path :: STRING) - POSITION('.', REVERSE(path :: STRING)) + 1, POSITION('.', REVERSE(path :: STRING))),
-                    ''
-                ),
-                '.',
-                '__'
-            )
-            ELSE '__'
-        END AS id,
-        OBJECT_AGG(
-            DISTINCT key,
-            VALUE
-        ) AS DATA,
-        txs.tx_id AS tx_hash,
-        txs.block_id AS block_number,
-        txs.block_timestamp AS block_timestamp,
-        CASE
-            WHEN txs.tx :receipt :status :: STRING = '0x1' THEN 'SUCCESS'
-            ELSE 'FAIL'
-        END AS tx_status,
-        txs.ingested_at AS ingested_at,
-        txs._inserted_timestamp AS _inserted_timestamp
-    FROM
-        traces_txs txs,
-        TABLE(
-            FLATTEN(
-                input => PARSE_JSON(
-                    txs.tx :traces
-                ),
-                recursive => TRUE
-            )
-        ) f
-    WHERE
-        f.index IS NULL
-        AND f.key != 'calls'
-    GROUP BY
-        tx_hash,
-        id,
-        block_number,
+        block_id,
         block_timestamp,
-        _inserted_timestamp,
-        ingested_at,
-        tx_status
+        tx_id AS tx_hash,
+        tx: traces AS full_traces,
+        ingested_at :: TIMESTAMP AS ingested_at,
+        _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp,
+        CASE
+            WHEN tx :receipt :status :: STRING = '0x1' THEN 'SUCCESS'
+            ELSE 'FAIL'
+        END AS tx_status
+    FROM
+        traces_txs
 ),
-flattened_traces AS (
+traces_flat AS (
     SELECT
-        DATA :from :: STRING AS from_address,
+        VALUE :from :: STRING AS from_address,
         udf_hex_to_int(
-            DATA :gas :: STRING
+            VALUE :gas :: STRING
         ) AS gas,
         udf_hex_to_int(
-            DATA :gasUsed :: STRING
+            VALUE :gasUsed :: STRING
         ) AS gas_used,
-        DATA :input :: STRING AS input,
-        DATA :output :: STRING AS output,
-        DATA :time :: STRING AS TIME,
-        DATA :to :: STRING AS to_address,
-        DATA :type :: STRING AS TYPE,
+        VALUE :input :: STRING AS input,
+        VALUE :output :: STRING AS output,
+        VALUE :time :: STRING AS TIME,
+        VALUE :to :: STRING AS to_address,
+        VALUE :type :: STRING AS TYPE,
         CASE
-            WHEN DATA :type :: STRING = 'CALL' THEN udf_hex_to_int(
-                DATA :value :: STRING
+            WHEN VALUE :type :: STRING = 'call' THEN udf_hex_to_int(
+                VALUE :value :: STRING
             ) / pow(
                 10,
                 18
             )
             ELSE 0
-        END AS eth_value,
-        CASE
-            WHEN id = '__' THEN CONCAT(
-                DATA :type :: STRING,
-                '_ORIGIN'
-            )
-            ELSE CONCAT(
-                DATA :type :: STRING,
-                '_',
-                REPLACE(
-                    REPLACE(REPLACE(REPLACE(id, 'calls', ''), '[', ''), ']', ''),
-                    '__',
-                    '_'
-                )
-            )
-        END AS identifier,
-        concat_ws(
-            '-',
-            tx_hash,
-            identifier
-        ) AS _call_id,
-        SPLIT(
-            identifier,
-            '_'
-        ) AS id_split,
-        ARRAY_SLICE(id_split, 1, ARRAY_SIZE(id_split)) AS levels,
-        ARRAY_TO_STRING(
-            levels,
-            '_'
-        ) AS LEVEL,
-        CASE
-            WHEN ARRAY_SIZE(levels) = 1
-            AND levels [0] :: STRING = 'ORIGIN' THEN NULL
-            WHEN ARRAY_SIZE(levels) = 1 THEN 'ORIGIN'
-            ELSE ARRAY_TO_STRING(ARRAY_SLICE(levels, 0, ARRAY_SIZE(levels) -1), '_')END AS parent_level,
-            COUNT(parent_level) over (
-                PARTITION BY tx_hash,
-                parent_level
-            ) AS sub_traces,*
-            FROM
-                base_table
-        ),
-        group_sub_traces AS (
-            SELECT
-                tx_hash,
-                parent_level,
-                sub_traces
-            FROM
-                flattened_traces
-            GROUP BY
-                tx_hash,
-                parent_level,
-                sub_traces
-        ),
-        FINAL AS (
-            SELECT
-                flattened_traces.tx_hash AS tx_hash,
-                flattened_traces.block_number AS block_number,
-                flattened_traces.block_timestamp AS block_timestamp,
-                flattened_traces.from_address AS from_address,
-                flattened_traces.to_address AS to_address,
-                flattened_traces.eth_value AS eth_value,
-                flattened_traces.gas AS gas,
-                flattened_traces.gas_used AS gas_used,
-                flattened_traces.input AS input,
-                flattened_traces.output AS output,
-                flattened_traces.type AS TYPE,
-                flattened_traces.identifier AS identifier,
-                flattened_traces._call_id AS _call_id,
-                flattened_traces.ingested_at AS ingested_at,
-                flattened_traces.data AS DATA,
-                flattened_traces.tx_status AS tx_status,
-                group_sub_traces.sub_traces AS sub_traces,
-                flattened_traces._inserted_timestamp AS _inserted_timestamp
-            FROM
-                flattened_traces
-                LEFT OUTER JOIN group_sub_traces
-                ON flattened_traces.tx_hash = group_sub_traces.tx_hash
-                AND flattened_traces.level = group_sub_traces.parent_level
-        )
-    SELECT
-        tx_hash,
-        block_number,
-        block_timestamp,
-        from_address,
-        to_address,
-        eth_value,
-        gas,
-        gas_used,
-        input,
-        output,
-        TYPE,
-        identifier,
-        _call_id,
-        ingested_at,
-        DATA,
-        tx_status,
-        sub_traces,
-        _inserted_timestamp
+        END AS eth_value,*
     FROM
-        FINAL
-    WHERE
-        identifier IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY _call_id
-    ORDER BY
-        _inserted_timestamp DESC)) = 1
+        traces_raw,
+        LATERAL FLATTEN (
+            input => full_traces
+        )
+)
+SELECT
+    tx_hash,
+    block_id AS block_number,
+    block_timestamp,
+    from_address,
+    to_address,
+    eth_value,
+    gas,
+    gas_used,
+    input,
+    output,
+    TYPE,
+    CONCAT(
+        TYPE,
+        '_',
+        INDEX
+    ) AS identifier,
+    concat_ws(
+        '-',
+        tx_hash,
+        identifier
+    ) AS _call_id,
+    ingested_at,
+    VALUE AS DATA,
+    tx_status,
+    _inserted_timestamp
+FROM
+    traces_flat
+WHERE
+    identifier IS NOT NULL qualify (ROW_NUMBER() over (PARTITION BY _call_id
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/sources.yml

@@ -7,4 +7,8 @@ sources:
     tables:
       - name: arbitrum_blocks
       - name: arbitrum_txs
-  
+  - name: crosschain
+    database: flipside_prod_db
+    schema: crosschain
+    tables:
+      - name: address_labels