wip gold and silver updates

2026-02-06 11:06:43 +00:00 · 2025-03-05 12:43:19 -07:00 · 2025-03-05 12:43:19 -07:00 · 84fd43d288
commit 84fd43d288
parent 6fddc417d2
13 changed files with 77 additions and 38 deletions
--- a/dbt_project.yml
+++ b/dbt_project.yml
@ -57,6 +57,16 @@ models:
    columns: true
  +on_schema_change: "append_new_columns"
  
+  movement_models:
+    gold:  # This will apply to ALL models under the gold directory
+      +tests:
+        - dbt_utils.recency:
+            datepart: hour
+            field: BLOCK_TIMESTAMP
+            interval: 3
+            severity: error
+            tags: ['recency']
+  
 # In this example config, we tell dbt to build all models in the example/ directory
 # as tables. These settings can be overridden in the individual model files
 # using the `{{ config(...) }}` macro.
--- a/models/gold/core/core__fact_blocks.yml
+++ b/models/gold/core/core__fact_blocks.yml
@ -2,7 +2,6 @@ version: 2
 models:
  - name: core__fact_blocks
    description: '{{ doc("core__fact_blocks") }}'   
-
    columns:
      - name: block_number
        description: '{{ doc("block_number") }}'  
--- a/models/gold/core/core__fact_changes.yml
+++ b/models/gold/core/core__fact_changes.yml
@ -2,7 +2,6 @@ version: 2
 models:
  - name: core__fact_changes
    description: '{{ doc("core__fact_changes") }}'   
-
    columns:
      - name: block_number
        description: '{{ doc("block_number") }}'  
--- a/models/gold/core/core__fact_events.yml
+++ b/models/gold/core/core__fact_events.yml
@ -2,7 +2,6 @@ version: 2
 models:
  - name: core__fact_events
    description: '{{ doc("core__fact_events") }}'   
-
    columns:
      - name: block_number
        description: '{{ doc("block_number") }}'  
--- a/models/gold/core/core__fact_transactions.yml
+++ b/models/gold/core/core__fact_transactions.yml
@ -2,12 +2,18 @@ version: 2
 models:
  - name: core__fact_transactions
    description: '{{ doc("core__fact_transactions") }}'   
-
+    tests:
+      - dbt_utils.recency:
+          datepart: hour
+          field: BLOCK_TIMESTAMP
+          interval: 3
+          severity: error
+          tags: ['recency']
    columns:
      - name: block_number
        description: '{{ doc("block_number") }}'  
      - name: BLOCK_TIMESTAMP
-        description: '{{ doc("block_timestamp") }}'   
+        description: '{{ doc("block_timestamp") }}' 
      - name: TX_HASH
        description: '{{ doc("tx_hash") }}'   
      - name: VERSION
--- a/models/gold/core/core__fact_transactions_block_metadata.sql
+++ b/models/gold/core/core__fact_transactions_block_metadata.sql
@ -14,23 +14,23 @@ SELECT
    A.block_timestamp,
    A.version,
    A.tx_hash,
-    A.data :success :: BOOLEAN AS success,
+    A.success,
    A.tx_type,
-    A.data :sender :: STRING AS sender,
-    A.data :signature :: STRING AS signature,
-    A.data :payload AS payload,
-    A.data :payload :function :: STRING AS payload_function,
-    A.data :changes AS changes,
-    A.data :events AS events,
-    A.data :failed_proposer_indices :: STRING AS failed_proposer_indices,
-    A.data :id :: STRING AS id,
-    A.data :previous_block_votes_bitvec :: STRING AS previous_block_votes_bitvec,
-    A.data :proposer :: STRING AS proposer,
-    A.data :ROUND :: INT AS ROUND,
-    A.data :vm_status :: STRING AS vm_status,
-    A.data :state_change_hash :: STRING AS state_change_hash,
-    A.data :accumulator_root_hash :: STRING AS accumulator_root_hash,
-    A.data :event_root_hash :: STRING AS event_root_hash,
+    A.sender,
+    A.signature,
+    A.payload,
+    A.payload_function,
+    A.changes,
+    A.events,
+    A.failed_proposer_indices,
+    A.id,
+    A.previous_block_votes_bitvec,
+    A.proposer,
+    A.ROUND,
+    A.vm_status,
+    A.state_change_hash,
+    A.accumulator_root_hash,
+    A.event_root_hash,
    {{ dbt_utils.generate_surrogate_key(
        ['tx_hash']
    ) }} AS fact_transactions_block_metadata_id,
--- a/models/gold/core/core__fact_transactions_block_metadata.yml
+++ b/models/gold/core/core__fact_transactions_block_metadata.yml
@ -2,7 +2,6 @@ version: 2
 models:
  - name: core__fact_transactions_block_metadata
    description: '{{ doc("core__fact_transactions_block_metadata") }}'   
-
    columns:
      - name: block_number
        description: '{{ doc("block_number") }}'  
--- a/models/gold/core/core__fact_transactions_state_checkpoint.yml
+++ b/models/gold/core/core__fact_transactions_state_checkpoint.yml
@ -2,7 +2,6 @@ version: 2
 models:
  - name: core__fact_transactions_state_checkpoint
    description: '{{ doc("core__fact_transactions_state_checkpoint") }}'   
-
    columns:
      - name: block_number
        description: '{{ doc("block_number") }}'  
--- a/models/silver/core/silver__blocks.yml
+++ b/models/silver/core/silver__blocks.yml
@ -1,25 +1,44 @@
 version: 2
+
 models:
  - name: silver__blocks
+    config:
+      where: "modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}"
    tests:
-      - sequence_gaps:
-          column: block_number
+      - dbt_utils.sequential_values:
+          column_name: block_number
+          interval: 1
+          config:
+            severity: error
+            error_if: ">100"
+          tags: ['test_recent']
    columns:
      - name: block_number
        tests:
          - not_null:
-              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+              tags: ['test_integrity']
      - name: block_hash
-      - name: block_timestamp_num
+        tests:
+          - not_null:
+              tags: ['test_integrity']
+          - unique:
+              tags: ['test_integrity']
      - name: block_timestamp
        tests:
          - not_null:
-              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+              tags: ['test_integrity']
+      - name: tx_count_from_transactions_array
+        tests:
+          - dbt_utils.expression_is_true:
+              expression: "tx_count_from_transactions_array >= 0"
+      - name: tx_count_from_versions
+        tests:
+          - dbt_utils.expression_is_true:
+              expression: "tx_count_from_versions >= 0"
+      - name: block_timestamp_num
      - name: first_version
      - name: last_version
-      - name: tx_count_from_transactions_array
-      - name: tx_count_from_versions
      - name: blocks_id
      - name: inserted_timestamp
      - name: modified_timestamp
-      - name: _invocation_id
+      - name: _invocation_id
--- a/models/silver/core/silver__transactions.sql
+++ b/models/silver/core/silver__transactions.sql
@ -112,6 +112,11 @@ transformed AS (
    DATA :accumulator_root_hash :: STRING AS accumulator_root_hash,
    DATA :event_root_hash :: STRING AS event_root_hash,
    DATA :state_checkpoint_hash :: STRING AS state_checkpoint_hash,
+    DATA :failed_proposer_indices :: STRING AS failed_proposer_indices,
+    DATA :id :: STRING AS id,
+    DATA :previous_block_votes_bitvec :: STRING AS previous_block_votes_bitvec,
+    DATA :proposer :: STRING AS proposer,
+    DATA :ROUND :: INT AS ROUND,
    DATA,
    file_last_updated
  FROM
@ -138,6 +143,11 @@ SELECT
  accumulator_root_hash,
  event_root_hash,
  state_checkpoint_hash,
+  failed_proposer_indices,
+  id,
+  previous_block_votes_bitvec,
+  proposer,
+  ROUND,
  DATA,
  {{ dbt_utils.generate_surrogate_key(
    ['tx_hash']
--- a/models/silver/core/silver__transactions.yml
+++ b/models/silver/core/silver__transactions.yml
@ -39,6 +39,11 @@ models:
      - name: accumulator_root_hash
      - name: event_root_hash
      - name: state_checkpoint_hash
+      - name: failed_proposer_indices
+      - name: id
+      - name: previous_block_votes_bitvec
+      - name: proposer
+      - name: ROUND
      - name: data
      - name: transactions_id
      - name: inserted_timestamp
--- a/models/streamline/silver/realtime/streamline__transactions_realtime.sql
+++ b/models/streamline/silver/realtime/streamline__transactions_realtime.sql
@ -16,17 +16,11 @@ WITH blocks AS (
    SELECT
        A.block_number,
        tx_count_from_versions AS tx_count,
-        --tx_count_from_versions -100 AS tx_count, --REMOVE FOR PUBLIC MAINNET
-        --first_version + 100 version_start --REMOVE FOR PUBLIC MAINNET
        first_version AS version_start
    FROM
        {{ ref('silver__blocks') }} A
-    WHERE
-        -- tx_count_from_versions > 100 --REMOVE FOR PUBLIC MAINNET
-        block_number != 0 --REMOVE FOR PUBLIC MAINNET
 ),
 numbers AS (
-    -- Recursive CTE to generate numbers. We'll use the maximum txcount value to limit our recursion.
    SELECT
        1 AS n
    UNION ALL
--- a/packages.yml
+++ b/packages.yml
@ -4,7 +4,7 @@ packages:
  - package: dbt-labs/dbt_utils
    version: 1.0.0
  - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: v1.29.0
+    revision: v1.32.0
  - package: get-select/dbt_snowflake_query_tags
    version: [">=2.0.0", "<3.0.0"]
  - package: Snowflake-Labs/dbt_constraints