silver blocks and transactions

macros/incremental_utils.sql

@@ -2,7 +2,7 @@
   -- dbt makes it easy to query your target table by using the "{{ this }}" variable.
 
 {% if is_incremental() %}
-{{ time_col }} > (
+{{ time_col }} >= (
   SELECT
     MAX(
       {{ time_col }}

models/bronze/bronze__transactions.sql

@@ -4,7 +4,7 @@
 
 SELECT
     record_id,
-    tx_id,
+    tx_id AS txn_hash,
     tx_block_index,
     offset_id,
     block_id,

models/core/actions_events.sql

@@ -2,7 +2,8 @@
   materialized = 'incremental',
   cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'],
   unique_key = 'action_id',
-  tags = ['actions']
+  tags = ['actions'],
+  enabled = false
 ) }}
 
 WITH txs AS (

models/core/actions_events_addkey.sql

@@ -2,7 +2,8 @@
   materialized = 'incremental',
   unique_key = 'action_id',
   cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'],
-  tags = ['actions_events']
+  tags = ['actions_events'],
+  enabled = false
 ) }}
 
 WITH action_events AS (

models/core/actions_events_function_call.sql

@@ -3,7 +3,8 @@
   incremental_strategy = 'delete+insert',
   unique_key = 'action_id',
   cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'],
-  tags = ['near','actions','events','functioncall']
+  tags = ['near','actions','events','functioncall'],
+  enabled = false
 ) }}
 
 WITH action_events AS (

models/core/receipts.sql

@@ -3,7 +3,8 @@
     unique_key = 'receipt_object_id',
     incremental_strategy = 'delete+insert',
     tags = ['core', 'transactions'],
-    cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE']
+    cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'],
+    enabled = false
 ) }}
 
 WITH txs AS (

models/core/transfers.sql

@@ -2,7 +2,8 @@
   materialized = 'incremental',
   cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'],
   unique_key = 'action_id',
-  tags = ['core', 'transfers']
+  tags = ['core', 'transfers'],
+  enabled = false
 ) }}
 
 WITH action_events AS(

models/descriptions/_ingested_at.md

@@ -1,4 +1,4 @@
-{% docs ingested_at %}
+{% docs _ingested_at %}
 
 The timestamp for when the block was ingested.
 

models/descriptions/_insterted_timestamp.md

@@ -0,0 +1,5 @@
+{% docs _inserted_timestamp %}
+
+The timestamp for when the block was inserted into the data warehouse by the Chainwalker.
+
+{% enddocs %}

models/metrics/metrics_active_wallets.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     incremental_strategy = 'delete+insert',
     tags = ['metrics'],
-    cluster_by = ['date']
+    cluster_by = ['date'],
+    enabled = false
 ) }}
 
 WITH txs AS (

models/metrics/metrics_daily_gas.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     incremental_strategy = 'delete+insert',
     tags = ['metrics', 'transactions'],
-    cluster_by = ['date']
+    cluster_by = ['date'],
+    enabled = false
 ) }}
 
 WITH txs AS (

models/metrics/metrics_daily_transactions.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     incremental_strategy = 'delete+insert',
     tags = ['metrics', 'transactions'],
-    cluster_by = ['date']
+    cluster_by = ['date'],
+    enabled = false
 ) }}
 
 WITH txs AS (

models/silver/silver__blocks.sql

@@ -2,8 +2,7 @@
     materialized = 'incremental',
     unique_key = 'block_height',
     incremental_strategy = 'delete+insert',
-    tags = ['core'],
-    cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE']
+    cluster_by = ['_inserted_timestamp::DATE']
 ) }}
 
 WITH base_blocks AS (
@@ -11,9 +10,14 @@ WITH base_blocks AS (
     SELECT
         *
     FROM
-        {{ ref("stg_blocks") }}
+        {{ ref('bronze__blocks') }}
     WHERE
-        {{ incremental_load_filter("ingested_at") }}
+        {{ incremental_load_filter('_inserted_timestamp') }}
+        qualify ROW_NUMBER() over (
+            PARTITION BY block_id
+            ORDER BY
+                _inserted_timestamp DESC
+        ) = 1
 ),
 FINAL AS (
     SELECT
@@ -52,7 +56,8 @@ FINAL AS (
         header :total_supply :: FLOAT AS total_supply,
         header :validator_proposals AS validator_proposals,
         header :validator_reward :: FLOAT AS validator_reward,
-        ingested_at
+        _ingested_at,
+        _inserted_timestamp
     FROM
         base_blocks
 )

models/silver/silver__blocks.yml

@@ -1,7 +1,7 @@
 version: 2
 
 models:
-  - name: blocks
+  - name: silver__blocks
     description: |-
       This table records all the blocks of Near blockchain.
     tests:
@@ -50,7 +50,7 @@ models:
               column_type_list:
                 - NUMBER
                 - FLOAT
-                
+
       - name: block_author
         description: "{{ doc('block_author')}}"
         tests:
@@ -161,7 +161,7 @@ models:
                 - ARRAY
                 - VARIANT
                 - OBJECT
-                
+
       - name: gas_price
         description: "{{ doc('gas_price')}}"
         tests:
@@ -170,7 +170,7 @@ models:
               column_type_list:
                 - NUMBER
                 - FLOAT
-                
+
       - name: last_ds_final_block
         description: "{{ doc('last_ds_final_block')}}"
         tests:
@@ -197,7 +197,7 @@ models:
               column_type_list:
                 - NUMBER
                 - FLOAT
-                
+
       - name: next_bp_hash
         description: "{{ doc('next_bp_hash')}}"
         tests:
@@ -241,7 +241,7 @@ models:
               column_type_list:
                 - NUMBER
                 - FLOAT
-                
+
       - name: prev_state_root
         description: "{{ doc('prev_state_root')}}"
         tests:
@@ -267,7 +267,7 @@ models:
               column_type_list:
                 - NUMBER
                 - FLOAT
-                
+
       - name: signature
         description: "{{ doc('signature')}}"
         tests:
@@ -285,7 +285,7 @@ models:
               column_type_list:
                 - NUMBER
                 - FLOAT
-                
+
       - name: validator_proposals
         description: "{{ doc('validator_proposals')}}"
         tests:
@@ -305,8 +305,16 @@ models:
                 - NUMBER
                 - FLOAT
 
-      - name: ingested_at
-        description: "{{ doc('ingested_at')}}"
+      - name: _ingested_at
+        description: "{{ doc('_ingested_at')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+
+      - name: _inserted_timestamp
+        description: "{{ doc('_inserted_timestamp')}}"
         tests:
           - not_null
           - dbt_expectations.expect_column_values_to_be_in_type_list:

models/silver/silver__transactions.sql

@@ -2,16 +2,28 @@
   materialized = 'incremental',
   unique_key = 'txn_hash',
   incremental_strategy = 'delete+insert',
-  tags = ['core', 'transactions'],
-  cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE']
+  cluster_by = ['_inserted_timestamp::DATE']
 ) }}
 
-WITH transactions AS (
+WITH base_transactions AS (
 
+  SELECT
+    *
+  FROM
+    {{ ref('bronze__transactions') }}
+  WHERE
+    {{ incremental_load_filter('_inserted_timestamp') }}
+    qualify ROW_NUMBER() over (
+      PARTITION BY txn_hash
+      ORDER BY
+        _inserted_timestamp DESC
+    ) = 1
+),
+transactions AS (
   SELECT
     block_id AS block_height,
     tx :outcome :block_hash :: STRING AS block_hash,
-    tx_id AS txn_hash,
+    txn_hash,
     block_timestamp,
     tx :nonce :: NUMBER AS nonce,
     tx :signature :: STRING AS signature,
@@ -26,11 +38,10 @@ WITH transactions AS (
       tx :actions,
       0
     ) :FunctionCall :gas :: NUMBER AS attached_gas,
-    ingested_at
+    _ingested_at,
+    _inserted_timestamp
   FROM
-    {{ ref('stg_txs') }}
-  WHERE
-    {{ incremental_load_filter("ingested_at") }}
+    base_transactions
 ),
 receipts AS (
   SELECT
@@ -68,7 +79,8 @@ FINAL AS (
       t.attached_gas,
       gas_used
     ) AS attached_gas,
-    t.ingested_at
+    t._ingested_at,
+    t._inserted_timestamp
   FROM
     transactions AS t
     JOIN receipts AS r

models/silver/silver__transactions.yml

@@ -1,8 +1,7 @@
-
 version: 2
 
 models:
-  - name: transactions
+  - name: silver__transactions
     description: "Near transactions."
     tests:
       - dbt_utils.unique_combination_of_columns:
@@ -143,8 +142,16 @@ models:
                 - NUMBER
                 - FLOAT
 
-      - name: ingested_at
-        description: "{{ doc('ingested_at')}}"
+      - name: _ingested_at
+        description: "{{ doc('_ingested_at')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+
+      - name: _inserted_timestamp
+        description: "{{ doc('_inserted_timestamp')}}"
         tests:
           - not_null
           - dbt_expectations.expect_column_values_to_be_in_type_list:

models/staging/stg_blocks.sql

@@ -3,7 +3,8 @@
     unique_key = 'block_id',
     incremental_strategy = 'delete+insert',
     tags = ['core'],
-    cluster_by = ['block_timestamp']
+    cluster_by = ['block_timestamp'],
+    enabled = false
 ) }}
 
 WITH FINAL AS (

models/staging/stg_txs.sql

@@ -3,7 +3,8 @@
     unique_key = 'tx_id',
     incremental_strategy = 'delete+insert',
     tags = ['core', 'transactions'],
-    cluster_by = ['block_timestamp']
+    cluster_by = ['block_timestamp'],
+    enabled = false
 ) }}
 
 WITH FINAL AS (