updating testing and refactoring transactions

models/gold/core/core__fact_transactions.sql

@@ -14,23 +14,23 @@ SELECT
     A.block_timestamp,
     A.version,
     A.tx_hash,
-    A.data :success :: BOOLEAN AS success,
+    A.success,
     A.tx_type,
-    A.data :sender :: STRING AS sender,
-    A.data :signature :: STRING AS signature,
-    A.data :payload AS payload,
-    A.data :payload :function :: STRING AS payload_function,
-    A.data :changes AS changes,
-    A.data :events AS events,
-    A.data: gas_unit_price :: bigint AS gas_unit_price,
-    A.data :gas_used :: INT AS gas_used,
-    A.data :max_gas_amount :: bigint AS max_gas_amount,
-    A.data :expiration_timestamp_secs :: bigint AS expiration_timestamp_secs,
-    A.data :vm_status :: STRING AS vm_status,
-    A.data :state_change_hash :: STRING AS state_change_hash,
-    A.data :accumulator_root_hash :: STRING AS accumulator_root_hash,
-    A.data :event_root_hash :: STRING AS event_root_hash,
-    A.data :state_checkpoint_hash :: STRING AS state_checkpoint_hash,
+    A.sender,
+    A.signature,
+    A.payload,
+    A.payload_function,
+    A.changes,
+    A.events,
+    A.gas_unit_price,
+    A.gas_used,
+    A.max_gas_amount,
+    A.expiration_timestamp_secs,
+    A.vm_status,
+    A.state_change_hash,
+    A.accumulator_root_hash,
+    A.event_root_hash,
+    A.state_checkpoint_hash,
     {{ dbt_utils.generate_surrogate_key(
         ['tx_hash']
     ) }} AS fact_transactions_id,

models/gold/stats/stats__ez_core_metrics_hourly.yml

@@ -2,33 +2,94 @@ version: 2
 models:
   - name: stats__ez_core_metrics_hourly
     description: '{{ doc("ez_core_metrics_hourly_table_doc") }}'
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_TIMESTAMP_HOUR
+      - dbt_utils.expression_is_true:
+          expression: "block_number_max >= block_number_min"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "transaction_count = transaction_count_success + transaction_count_failed"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "block_count <= transaction_count"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "total_fees_native >= 0"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "total_fees_usd >= 0"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
 
     columns:
       - name: BLOCK_TIMESTAMP_HOUR
         description: '{{ doc("block_timestamp_hour") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ
       - name: BLOCK_NUMBER_MIN
         description: '{{ doc("block_number_min") }}'
+        tests:
+          - not_null
       - name: BLOCK_NUMBER_MAX
         description: '{{ doc("block_number_max") }}'
+        tests:
+          - not_null
       - name: BLOCK_COUNT
         description: '{{ doc("block_count") }}'
+        tests:
+          - not_null
       - name: TRANSACTION_COUNT
         description: '{{ doc("transaction_count") }}'
+        tests:
+          - not_null
       - name: TRANSACTION_COUNT_SUCCESS
         description: '{{ doc("transaction_count_success") }}'
+        tests:
+          - not_null
       - name: TRANSACTION_COUNT_FAILED
         description: '{{ doc("transaction_count_failed") }}'
+        tests:
+          - not_null
       - name: UNIQUE_SENDER_COUNT
         description: '{{ doc("unique_sender_count") }}'
+        tests:
+          - not_null
       - name: UNIQUE_PAYLOAD_FUNCTION_COUNT
         description: '{{ doc("unique_payload_function_count") }}'
+        tests:
+          - not_null
       - name: TOTAL_FEES_NATIVE
         description: '{{ doc("total_fees_native") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DECIMAL(38,8)
       - name: TOTAL_FEES_USD
         description: '{{ doc("total_fees_usd") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DECIMAL(38,8)
       - name: EZ_CORE_METRICS_HOURLY_ID
         description: '{{ doc("pk") }}'   
+        tests:
+          - not_null
+          - unique
       - name: INSERTED_TIMESTAMP
         description: '{{ doc("inserted_timestamp") }}'   
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
       - name: MODIFIED_TIMESTAMP
         description: '{{ doc("modified_timestamp") }}'   
+        tests:
+          - not_null 

models/silver/core/silver__blocks.yml

@@ -1,39 +1,25 @@
 version: 2
 models:
   - name: silver__blocks
-
     tests:
-      - dbt_constraints.primary_key:
-          column_name: BLOCKS_ID
-
+      - sequence_gaps:
+          column: block_number
     columns:
       - name: block_number
         tests:
-          - not_null  
-      - name: BLOCK_TIMESTAMP_NUM
+          - not_null:
+              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - name: block_hash
+      - name: block_timestamp_num
+      - name: block_timestamp
         tests:
-          - not_null
-      - name: BLOCK_TIMESTAMP
-        tests:
-          - not_null
-      - name: BLOCK_HASH
-        tests:
-          - not_null 
-      - name: FIRST_VERSION
-        tests:
-          - not_null
-      - name: LAST_VERSION
-        tests:
-          - not_null
-      - name: TX_COUNT_FROM_TRANSACTIONS_ARRAY
-        tests:
-          - not_null
-      - name: TX_COUNT_FROM_VERSIONS
-        tests:
-          - not_null
-      - name: BLOCKS_ID
-        tests:
-          - not_null
-      - name: INSERTED_TIMESTAMP
-      - name: MODIFIED_TIMESTAMP
-      - name: _INVOCATION_ID
+          - not_null:
+              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - name: first_version
+      - name: last_version
+      - name: tx_count_from_transactions_array
+      - name: tx_count_from_versions
+      - name: blocks_id
+      - name: inserted_timestamp
+      - name: modified_timestamp
+      - name: _invocation_id

models/silver/core/silver__changes.yml

@@ -1,26 +1,34 @@
 version: 2
 models:
   - name: silver__changes
-          
+    tests:
+      - dbt_utils.expression_is_true:
+          expression: "b.value:type IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "b.value:data: 'type' IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
     columns:
       - name: block_number
-      - name: BLOCK_TIMESTAMP
-      - name: TX_HASH
-      - name: VERSION
-      - name: SUCCESS
-      - name: TX_TYPE
-      - name: CHANGE_INDEX
-      - name: CHANGE_DATA
-      - name: CHANGE_TYPE
-      - name: ADDRESS
-      - name: HANDLE
-      - name: INNER_CHANGE_TYPE      
-      - name: CHANGE_ADDRESS
-      - name: CHANGE_MODULE
-      - name: CHANGE_RESOURCE
-      - name: KEY
-      - name: VALUE
-      - name: STATE_KEY_HASH
-      - name: CHANGES_ID
-      - name: INSERTED_TIMESTAMP
-      - name: MODIFIED_TIMESTAMP
+      - name: block_timestamp
+      - name: tx_hash
+      - name: version
+      - name: success
+      - name: tx_type
+      - name: payload_function
+      - name: change_index
+      - name: change_data
+      - name: change_type
+      - name: address
+      - name: handle
+      - name: inner_change_type
+      - name: change_address
+      - name: change_module
+      - name: change_resource
+      - name: key
+      - name: value
+      - name: state_key_hash
+      - name: changes_id
+      - name: inserted_timestamp
+      - name: modified_timestamp
+      - name: _invocation_id

models/silver/core/silver__events.yml

@@ -1,23 +1,31 @@
 version: 2
 models:
   - name: silver__events
-
+    tests:
+      - dbt_utils.expression_is_true:
+          expression: "b.value:type IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "b.value:guid:account_address IS NOT NULL AND b.value:guid:creation_number IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
     columns:
       - name: block_number
-      - name: BLOCK_TIMESTAMP
-      - name: TX_HASH
-      - name: VERSION
-      - name: SUCCESS
-      - name: TX_TYPE
-      - name: EVENT_INDEX
-      - name: EVENT_TYPE
-      - name: EVENT_ADDRESS
-      - name: EVENT_MODULE
-      - name: EVENT_RESOURCE   
-      - name: EVENT_DATA
-      - name: ACCOUNT_ADDRESS
-      - name: CREATION_NUMBER
-      - name: SEQUENCE_NUMBER
-      - name: EVENTS_ID
-      - name: INSERTED_TIMESTAMP
-      - name: MODIFIED_TIMESTAMP
+      - name: block_timestamp
+      - name: tx_hash
+      - name: version
+      - name: success
+      - name: tx_type
+      - name: payload_function
+      - name: event_index
+      - name: event_type
+      - name: event_address
+      - name: event_module
+      - name: event_resource
+      - name: event_data
+      - name: account_address
+      - name: creation_number
+      - name: sequence_number
+      - name: events_id
+      - name: inserted_timestamp
+      - name: modified_timestamp
+      - name: _invocation_id

models/silver/core/silver__transactions.sql

@@ -86,15 +86,58 @@ combo AS (
     file_last_updated
   FROM
     from_transactions A
+),
+transformed AS (
+  SELECT
+    COALESCE(
+      block_timestamp,
+      '1970-01-01 00:00:00.000'
+    ) AS block_timestamp,
+    tx_hash,
+    version,
+    tx_type,
+    DATA :success :: BOOLEAN AS success,
+    DATA :sender :: STRING AS sender,
+    DATA :signature :: STRING AS signature,
+    DATA :payload AS payload,
+    DATA :payload :function :: STRING AS payload_function,
+    DATA :changes AS changes,
+    DATA :events AS events,
+    DATA :gas_unit_price :: bigint AS gas_unit_price,
+    DATA :gas_used :: INT AS gas_used,
+    DATA :max_gas_amount :: bigint AS max_gas_amount,
+    DATA :expiration_timestamp_secs :: bigint AS expiration_timestamp_secs,
+    DATA :vm_status :: STRING AS vm_status,
+    DATA :state_change_hash :: STRING AS state_change_hash,
+    DATA :accumulator_root_hash :: STRING AS accumulator_root_hash,
+    DATA :event_root_hash :: STRING AS event_root_hash,
+    DATA :state_checkpoint_hash :: STRING AS state_checkpoint_hash,
+    DATA,
+    file_last_updated
+  FROM
+    combo
 )
 SELECT
-  COALESCE(
-    block_timestamp,
-    '1970-01-01 00:00:00.000'
-  ) AS block_timestamp,
+  block_timestamp,
   tx_hash,
   version,
   tx_type,
+  success,
+  sender,
+  signature,
+  payload,
+  payload_function,
+  changes,
+  events,
+  gas_unit_price,
+  gas_used,
+  max_gas_amount,
+  expiration_timestamp_secs,
+  vm_status,
+  state_change_hash,
+  accumulator_root_hash,
+  event_root_hash,
+  state_checkpoint_hash,
   DATA,
   {{ dbt_utils.generate_surrogate_key(
     ['tx_hash']
@@ -103,6 +146,6 @@ SELECT
   SYSDATE() AS modified_timestamp,
   '{{ invocation_id }}' AS _invocation_id
 FROM
-  combo qualify(ROW_NUMBER() over (PARTITION BY tx_hash
+  transformed qualify(ROW_NUMBER() over (PARTITION BY tx_hash
 ORDER BY
   file_last_updated DESC)) = 1

models/silver/core/silver__transactions.yml

@@ -1,37 +1,47 @@
 version: 2
 models:
   - name: silver__transactions
-
     tests:
-      - dbt_constraints.primary_key:
-          column_name: TRANSACTIONS_ID
-
+      - dbt_utils.expression_is_true:
+          expression: "b.value:timestamp IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "b.value:hash IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "b.value:version IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "b.value:type IS NOT NULL"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
 
     columns:
-      - name: BLOCK_TIMESTAMP
+      - name: block_timestamp
+      - name: tx_hash
         tests:
-          - not_null
-      - name: TX_HASH
-        tests:
-          - not_null
-      - name: VERSION
-        tests:
-          - not_null
-      - name: TX_TYPE
-        tests:
-          - not_null
-      - name: TRANSACTIONS_ID
-        tests:
-          - not_null
-      - name: INSERTED_TIMESTAMP
-        tests:
-          - not_null
-      - name: MODIFIED_TIMESTAMP
-        tests:
-          - not_null
-
-      - name: _INVOCATION_ID
-        tests:
-          - name: not_null_silver__transactions_INVOCATION_ID
-            test_name: not_null
+          - unique:
+              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - name: version
+      - name: tx_type
+      - name: success
+      - name: sender
+      - name: signature
+      - name: payload
+      - name: payload_function
+      - name: changes
+      - name: events
+      - name: gas_unit_price
+      - name: gas_used
+      - name: max_gas_amount
+      - name: expiration_timestamp_secs
+      - name: vm_status
+      - name: state_change_hash
+      - name: accumulator_root_hash
+      - name: event_root_hash
+      - name: state_checkpoint_hash
+      - name: data
+      - name: transactions_id
+      - name: inserted_timestamp
+      - name: modified_timestamp
+      - name: _invocation_id
       

models/silver/stats/silver__core_metrics_hourly.yml

@@ -5,64 +5,47 @@ models:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
             - BLOCK_TIMESTAMP_HOUR
+      - dbt_utils.expression_is_true:
+          expression: "block_number_max >= block_number_min"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "transaction_count = transaction_count_success + transaction_count_failed"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "block_count <= transaction_count"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - dbt_utils.expression_is_true:
+          expression: "total_fees >= 0"
+          where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+
     columns:
       - name: BLOCK_TIMESTAMP_HOUR
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_LTZ
-                - TIMESTAMP_NTZ
       - name: block_number_MIN
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT
       - name: block_number_MAX
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT
       - name: BLOCK_COUNT
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT
       - name: TRANSACTION_COUNT
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT
       - name: TRANSACTION_COUNT_SUCCESS
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT
       - name: TRANSACTION_COUNT_FAILED
         tests:
           - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT
       - name: TOTAL_FEES
         tests:
           - not_null
           - dbt_expectations.expect_column_values_to_be_in_type_list:
               column_type_list:
                 - DECIMAL
-                - FLOAT
-                - NUMBER
       - name: _INSERTED_TIMESTAMP
         tests:
           - dbt_expectations.expect_row_values_to_have_recent_data: