An 2013/op l1 info (#28)

* add l1 info

* incremental

Co-authored-by: Austin <austin@flipsidecrypto.com>

models/bronze/bronze__state_hashes.sql

@@ -0,0 +1,33 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = "state_tx_hash"
+) }}
+
+SELECT
+    state_tx_hash,
+    state_block_number,
+    state_block_timestamp,
+    state_batch_index,
+    state_batch_root,
+    state_batch_size,
+    state_prev_total_elements,
+    state_min_block,
+    state_max_block,
+    _inserted_timestamp
+FROM
+    {{ source(
+        'ethereum_silver',
+        'optimism_state_hashes'
+    ) }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/bronze/bronze__submission_hashes.sql

@@ -0,0 +1,33 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = "l1_submission_tx_hash"
+) }}
+
+SELECT
+    l1_submission_tx_hash,
+    l1_submission_block_number,
+    l1_submission_block_timestamp,
+    l1_submission_batch_index,
+    l1_submission_batch_root,
+    l1_submission_size,
+    l1_submission_prev_total_elements,
+    sub_min_block,
+    sub_max_block,
+    _inserted_timestamp
+FROM
+    {{ source(
+        'ethereum_silver',
+        'optimism_submission_hashes'
+    ) }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/doc_descriptions/L1_fields/batch_size.md

@@ -0,0 +1,5 @@
+{% docs op_batch_size %}
+
+Total Optimism Txs included within batch.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_batch_root.md

@@ -0,0 +1,5 @@
+{% docs op_batch_root %}
+
+Root of batch, either for sumbission or state.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_block_number.md

@@ -0,0 +1,5 @@
+{% docs op_l1_block_no %}
+
+The Ethereum block number that contained the batch.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_block_timestamp.md

@@ -0,0 +1,5 @@
+{% docs op_l1_block_time %}
+
+The timestamp of the Ethereum block that contained this batch.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_fee_scalar.md

@@ -0,0 +1,5 @@
+{% docs op_l1_fee_scalar %}
+
+This value covers the change in L1 gas price between the time the transaction is submitted and when it is published.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_gas_price.md

@@ -0,0 +1,5 @@
+{% docs op_l1_gas_price %}
+
+The gas price for L1 transactions when the transaction was processed.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_gas_used.md

@@ -0,0 +1,5 @@
+{% docs op_l1_gas_used %}
+
+The gas used on L1 to publish the transaction.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_state_root_batch_index.md

@@ -0,0 +1,5 @@
+{% docs op_l1_state_batch %}
+
+The batch index of when this block was included in the Ethereum state root.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_state_root_tx_hash.md

@@ -0,0 +1,5 @@
+{% docs op_l1_state_tx_hash %}
+
+The L1 tx hash of when this block was included in the Ethereum state root.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_submission_batch_index.md

@@ -0,0 +1,5 @@
+{% docs op_l1_sub_batch %}
+
+The batch index of when this block was submitted to L1.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/l1_submission_tx_hash.md

@@ -0,0 +1,5 @@
+{% docs op_l1_sub_tx_hash %}
+
+The L1 tx hash of when this block was submitted to L1.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/op_max_block.md

@@ -0,0 +1,5 @@
+{% docs op_max_block %}
+
+The max block on Optimism this batch relates to.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/op_min_block.md

@@ -0,0 +1,5 @@
+{% docs op_min_block %}
+
+The min block on Optimism this batch relates to.
+
+{% enddocs %}

models/doc_descriptions/L1_fields/prev_total_elements.md

@@ -0,0 +1,5 @@
+{% docs op_prev_total_elements %}
+
+Confirmed blocks prior to this batch.
+
+{% enddocs %}

models/gold/core__fact_blocks.sql

@@ -21,6 +21,10 @@ SELECT
     sha3_uncles,
     SIZE,
     uncle_blocks,
-    block_header_json
+    block_header_json,
+    l1_state_root_tx_hash,
+    l1_state_root_batch_index,
+    l1_submission_tx_hash,
+    l1_submission_batch_index
 FROM
     {{ ref('silver__blocks') }}

models/gold/core__fact_blocks.yml

@@ -37,4 +37,12 @@ models:
       - name: UNCLE_BLOCKS
         description: '{{ doc("opt_uncle_blocks") }}' 
       - name: BLOCK_HEADER_JSON
-        description: '{{ doc("opt_block_header_json") }}'
+        description: '{{ doc("opt_block_header_json") }}'
+      - name: L1_STATE_ROOT_TX_HASH
+        description: '{{ doc("op_l1_state_tx_hash") }}'
+      - name: L1_STATE_ROOT_BATCH_INDEX
+        description: '{{ doc("op_l1_state_batch") }}'
+      - name: L1_SUBMISSION_TX_HASH
+        description: '{{ doc("op_l1_sub_tx_hash") }}'
+      - name: L1_SUBMISSION_BATCH_INDEX
+        description: '{{ doc("op_l1_sub_batch") }}'

models/gold/core__fact_l1_state_root_submissions.sql

@@ -0,0 +1,18 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    state_tx_hash AS l1_state_root_tx_hash,
+    state_block_number AS l1_block_number,
+    state_block_timestamp AS l1_block_timestamp,
+    state_batch_index AS l1_state_root_batch_index,
+    state_batch_root AS batch_root,
+    state_batch_size AS batch_size,
+    state_prev_total_elements AS prev_total_elements,
+    state_min_block AS op_min_block,
+    state_max_block AS op_max_block
+FROM
+    {{ ref('bronze__state_hashes') }}

models/gold/core__fact_l1_state_root_submissions.yml

@@ -0,0 +1,24 @@
+version: 2
+models:
+  - name: core__fact_l1_state_root_submissions
+    description: 'Contains details on the state root batch submission to L1.'  
+
+    columns:
+      - name: L1_STATE_ROOT_TX_HASH 
+        description: '{{ doc("op_l1_state_tx_hash") }}'   
+      - name: L1_BLOCK_NUMBER 
+        description: '{{ doc("op_l1_block_no") }}'   
+      - name: L1_BLOCK_TIMESTAMP
+        description: '{{ doc("op_l1_block_time") }}'   
+      - name: L1_STATE_ROOT_BATCH_INDEX 
+        description: '{{ doc("op_l1_state_batch") }}'   
+      - name: BATCH_ROOT 
+        description: '{{ doc("op_batch_root") }}'   
+      - name: BATCH_SIZE
+        description: '{{ doc("op_batch_size") }}'    
+      - name: PREV_TOTAL_ELEMENTS
+        description: '{{ doc("op_prev_total_elements") }}'   
+      - name: OP_MIN_BLOCK 
+        description: '{{ doc("op_min_block") }}'   
+      - name: OP_MAX_BLOCK 
+        description: '{{ doc("op_max_block") }}'   

models/gold/core__fact_l1_submissions.sql

@@ -0,0 +1,18 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    l1_submission_tx_hash,
+    l1_submission_block_number AS l1_block_number,
+    l1_submission_block_timestamp AS l1_block_timestamp,
+    l1_submission_batch_index AS l1_submission_batch_index,
+    l1_submission_batch_root AS batch_root,
+    l1_submission_size AS batch_size,
+    l1_submission_prev_total_elements AS prev_total_elements,
+    sub_min_block AS op_min_block,
+    sub_max_block AS op_max_block
+FROM
+    {{ ref('bronze__submission_hashes') }}

models/gold/core__fact_l1_submissions.yml

@@ -0,0 +1,24 @@
+version: 2
+models:
+  - name: core__fact_l1_submissions
+    description: 'Contains details on the batch submission to L1.'  
+
+    columns:
+      - name: L1_SUBMISSION_ROOT_TX_HASH 
+        description: '{{ doc("op_l1_sub_tx_hash") }}'   
+      - name: L1_BLOCK_NUMBER 
+        description: '{{ doc("op_l1_block_no") }}'   
+      - name: L1_BLOCK_TIMESTAMP
+        description: '{{ doc("op_l1_block_time") }}'   
+      - name: L1_SUBMISSION_BATCH_INDEX 
+        description: '{{ doc("op_l1_sub_batch") }}'   
+      - name: BATCH_ROOT 
+        description: '{{ doc("op_batch_root") }}'   
+      - name: BATCH_SIZE
+        description: '{{ doc("op_batch_size") }}'    
+      - name: PREV_TOTAL_ELEMENTS
+        description: '{{ doc("op_prev_total_elements") }}'   
+      - name: OP_MIN_BLOCK 
+        description: '{{ doc("op_min_block") }}'   
+      - name: OP_MAX_BLOCK 
+        description: '{{ doc("op_max_block") }}'   

models/gold/core__fact_transactions.sql

@@ -19,6 +19,13 @@ SELECT
     gas_price,
     gas_limit,
     gas_used,
+    l1_gas_price,
+    l1_gas_used,
+    l1_fee_scalar,
+    l1_submission_batch_index,
+    l1_submission_tx_hash,
+    l1_state_root_batch_index,
+    l1_state_root_tx_hash,
     cumulative_Gas_Used,
     input_data,
     status,

models/gold/core__fact_transactions.yml

@@ -30,6 +30,20 @@ models:
         description: '{{ doc("opt_tx_gas_limit") }}' 
       - name: GAS_USED
         description: '{{ doc("opt_tx_gas_used") }}'
+      - name: L1_GAS_PRICE
+        description: '{{ doc("op_l1_gas_price") }}'
+      - name: L1_GAS_USED
+        description: '{{ doc("op_l1_gas_used") }}'
+      - name: L1_FEE_SCALAR
+        description: '{{ doc("op_l1_fee_scalar") }}'
+      - name: L1_STATE_ROOT_TX_HASH
+        description: '{{ doc("op_l1_state_tx_hash") }}'
+      - name: L1_STATE_ROOT_BATCH_INDEX
+        description: '{{ doc("op_l1_state_batch") }}'
+      - name: L1_SUBMISSION_TX_HASH
+        description: '{{ doc("op_l1_sub_tx_hash") }}'
+      - name: L1_SUBMISSION_BATCH_INDEX
+        description: '{{ doc("op_l1_sub_batch") }}'        
       - name: CUMULATIVE_GAS_USED
         description: '{{ doc("opt_cumulative_gas_used") }}' 
       - name: STATUS

models/silver/silver__blocks.sql

@@ -22,14 +22,22 @@ WITH base_tables AS (
 
 {% if is_incremental() %}
 WHERE
-    ingested_at >= (
+    _inserted_timestamp >= (
         SELECT
             MAX(
-                ingested_at
+                _inserted_timestamp
             )
         FROM
             {{ this }}
     )
+    OR block_id IN (
+        SELECT
+            block_number
+        FROM
+            {{ this }}
+        WHERE
+            l1_state_root_tx_hash IS NULL
+    )
 {% endif %}
 )
 SELECT
@@ -68,8 +76,18 @@ SELECT
     END AS uncle_blocks,
     ingested_at :: TIMESTAMP AS ingested_at,
     header :: OBJECT AS block_header_json,
-    _inserted_timestamp :: TIMESTAMP as _inserted_timestamp
+    base_tables._inserted_timestamp :: TIMESTAMP AS _inserted_timestamp,
+    state_tx_hash AS l1_state_root_tx_hash,
+    state_batch_index AS l1_state_root_batch_index,
+    l1_submission_tx_hash,
+    l1_submission_batch_index AS l1_submission_batch_index
 FROM
-    base_tables qualify(ROW_NUMBER() over(PARTITION BY block_id
+    base_tables
+    LEFT JOIN {{ ref('bronze__state_hashes') }}
+    ON block_id BETWEEN state_min_block
+    AND state_max_block
+    LEFT JOIN {{ ref('bronze__submission_hashes') }}
+    ON block_id BETWEEN sub_min_block
+    AND sub_max_block qualify(ROW_NUMBER() over(PARTITION BY block_number
 ORDER BY
-    _inserted_timestamp DESC)) = 1
+    base_tables._inserted_timestamp DESC)) = 1

models/silver/silver__blocks.yml

@@ -109,5 +109,20 @@ models:
           - dbt_expectations.expect_column_values_to_be_in_type_list:
               column_type_list:
                 - OBJECT
-
+      - name: L1_STATE_ROOT_TX_HASH
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
+      - name: L1_STATE_ROOT_BATCH_INDEX
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
+      - name: L1_SUBMISSION_BATCH_INDEX
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
+      - name: L1_SUBMISSION_TX_HASH
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
 

models/silver/silver__transactions.sql

@@ -28,9 +28,6 @@ WITH base_table AS (
         (
             udf_hex_to_int(
                 tx :gasPrice :: STRING
-            ) / pow(
-                10,
-                9
             )
         ) :: FLOAT AS gas_price,
         udf_hex_to_int(
@@ -50,26 +47,59 @@ WITH base_table AS (
         udf_hex_to_int(
             tx :receipt :effectiveGasPrice :: STRING
         ) :: INTEGER AS effective_Gas_Price,
-        (gas_price * gas_used) / pow(10,9) As tx_fee,
         ingested_at :: TIMESTAMP AS ingested_at,
-        _inserted_timestamp :: TIMESTAMP as _inserted_timestamp,
+        _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp,
         OBJECT_DELETE(
             tx,
             'traces'
-        ) AS tx_json
+        ) AS tx_json,
+        COALESCE(
+            udf_hex_to_int(
+                tx :receipt :l1Fee :: STRING
+            ) :: FLOAT,
+            0
+        ) AS l1_fee,
+        COALESCE(
+            tx :receipt :l1FeeScalar :: INTEGER,
+            0
+        ) :: FLOAT AS l1_fee_scalar,
+        COALESCE(
+            udf_hex_to_int(
+                tx :receipt :l1GasPrice :: STRING
+            ) :: FLOAT,
+            0
+        ) AS l1_gas_price,
+        COALESCE(
+            udf_hex_to_int(
+                tx :receipt :l1GasUsed :: STRING
+            ) :: FLOAT,
+            0
+        ) AS l1_gas_used,
+        ((gas_used * gas_price) + (l1_gas_price * l1_gas_used * l1_fee_scalar)) / pow(
+            10,
+            18
+        ) AS tx_fee
     FROM
         {{ ref('bronze__transactions') }}
 
 {% if is_incremental() %}
 WHERE
-    ingested_at >= (
+    _inserted_timestamp >= (
         SELECT
             MAX(
-                ingested_at
+                _inserted_timestamp
             )
         FROM
             {{ this }}
     )
+    OR block_id IN (
+        SELECT
+            block_number
+        FROM
+            {{ this }}
+        WHERE
+            l1_state_root_tx_hash IS NULL
+    )
 {% endif %}
 )
 SELECT
@@ -87,18 +117,37 @@ SELECT
     to_address,
     eth_value,
     block_hash,
-    gas_price,
+    gas_price / pow(
+        10,
+        9
+    ) AS gas_price,
     gas_limit,
     DATA AS input_data,
     status,
     gas_used,
     cumulative_Gas_Used,
     effective_Gas_Price,
+    l1_gas_price / pow(
+        10,
+        9
+    ) AS l1_gas_price,
+    l1_gas_used,
+    l1_fee_scalar,
     tx_fee,
     ingested_at,
-    _inserted_timestamp,
-    tx_json
+    base_table._inserted_timestamp AS _inserted_timestamp,
+    tx_json,
+    state_tx_hash AS l1_state_root_tx_hash,
+    state_batch_index AS l1_state_root_batch_index,
+    l1_submission_tx_hash,
+    l1_submission_batch_index AS l1_submission_batch_index
 FROM
-    base_table qualify(ROW_NUMBER() over(PARTITION BY tx_hash
+    base_table
+    LEFT JOIN {{ ref('bronze__state_hashes') }}
+    ON block_number BETWEEN state_min_block
+    AND state_max_block
+    LEFT JOIN {{ ref('bronze__submission_hashes') }}
+    ON block_number BETWEEN sub_min_block
+    AND sub_max_block qualify(ROW_NUMBER() over(PARTITION BY tx_hash
 ORDER BY
-    _inserted_timestamp DESC)) = 1
+    base_table._inserted_timestamp DESC)) = 1

models/silver/silver__transactions.yml

@@ -117,5 +117,21 @@ models:
       - name: ORIGIN_FUNCTION_SIGNATURE
         tests:
           - not_null
+      - name: L1_STATE_ROOT_TX_HASH
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
+      - name: L1_STATE_ROOT_BATCH_INDEX
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
+      - name: L1_SUBMISSION_BATCH_INDEX
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
+      - name: L1_SUBMISSION_TX_HASH
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP < CURRENT_DATE
 
 

models/sources.yml

@@ -39,6 +39,12 @@ sources:
     schema: SILVER
     tables:
       - name: prices_v2
+  - name: ethereum_silver
+    database: ethereum
+    schema: silver
+    tables:
+      - name: optimism_state_hashes
+      - name: optimism_submission_hashes