upd actions dir with IS_MIGRATION and add _modified_ts

models/gold/core/core__fact_blocks.sql

@@ -15,7 +15,7 @@ SELECT
     block_id,
     block_timestamp,
     block_hash,
-    tx_count,
+    tx_count, -- TO DEPRECATE
     block_author,
     header,
     header :challenges_result :: ARRAY AS block_challenges_result,
@@ -28,7 +28,7 @@ SELECT
     header :chunks_included :: NUMBER AS chunks_included,
     epoch_id,
     header :epoch_sync_data_hash :: STRING AS epoch_sync_data_hash,
-    events,
+    events, -- TO DEPRECATE
     gas_price,
     header :last_ds_final_block :: STRING AS last_ds_final_block,
     header :last_final_block :: STRING AS last_final_block,

models/gold/core/core__fact_transactions.sql

@@ -25,7 +25,7 @@ SELECT
     transaction_fee,
     attached_gas,
     tx_succeeded,
-    tx_status,
+    tx_status, -- TO DEPRECATE
     COALESCE(
         streamline_transactions_final_id,
         {{ dbt_utils.generate_surrogate_key(

models/silver/actions/silver__actions_events_addkey_s3.sql

@@ -12,14 +12,26 @@
 WITH action_events AS (
 
   SELECT
-    *
+    action_id,
+    tx_hash,
+    block_id,
+    block_timestamp,
+    action_data,
+    _partition_by_block_number,
+    _inserted_timestamp,
+    modified_timestamp AS _modified_timestamp
   FROM
     {{ ref('silver__actions_events_s3') }}
   WHERE
-    action_name = 'AddKey' {% if var("MANUAL_FIX") %}
+    action_name = 'AddKey' 
+    {% if var("MANUAL_FIX") %}
       AND {{ partition_load_manual('no_buffer') }}
     {% else %}
-      AND {{ incremental_load_filter('_inserted_timestamp') }}
+      {% if var('IS_MIGRATION') %}
+        AND {{ incremental_load_filter('_inserted_timestamp') }}
+      {% else %}
+        AND {{ incremental_load_filter('_modified_timestamp') }}
+      {% endif %}
     {% endif %}
 ),
 addkey_events AS (
@@ -35,8 +47,8 @@ addkey_events AS (
     action_data :access_key :permission :FunctionCall :method_names :: ARRAY AS method_name,
     action_data :access_key :permission :FunctionCall :receiver_id :: STRING AS receiver_id,
     _partition_by_block_number,
-    _load_timestamp,
-    _inserted_timestamp
+    _inserted_timestamp,
+    _modified_timestamp
   FROM
     action_events
 )

models/silver/actions/silver__actions_events_addkey_s3.yml

@@ -39,12 +39,12 @@ models:
       - name: _PARTITION_BY_BLOCK_NUMBER
         description: "{{ doc('_partition_by_block_number')}}"
 
-      - name: _load_timestamp
-        description: "{{ doc('_load_timestamp')}}"
-
       - name: _INSERTED_TIMESTAMP
         description: "{{ doc('_inserted_timestamp')}}"
 
+      - name: _MODIFIED_TIMESTAMP
+        description: "{{ doc('_modified_timestamp')}}"
+
       - name: actions_events_addkey_id
         description: "{{doc('id')}}"
 

models/silver/actions/silver__actions_events_function_call_s3.sql

@@ -10,7 +10,19 @@
 WITH action_events AS (
 
   SELECT
-    *
+    action_id,
+    tx_hash,
+    receiver_id,
+    signer_id,
+    block_id,
+    block_timestamp,
+    action_name,
+    action_data,
+    logs,
+    receipt_succeeded,
+    _partition_by_block_number,
+    _inserted_timestamp,
+    modified_timestamp AS _modified_timestamp
   FROM
     {{ ref('silver__actions_events_s3') }}
   WHERE
@@ -18,7 +30,11 @@ WITH action_events AS (
     {% if var("MANUAL_FIX") %}
       AND {{ partition_load_manual('no_buffer') }}
     {% else %}
-      AND {{ incremental_load_filter('_inserted_timestamp') }}
+      {% if var('IS_MIGRATION') %}
+        AND {{ incremental_load_filter('_inserted_timestamp') }}
+      {% else %}
+        AND {{ incremental_load_filter('_modified_timestamp') }}
+      {% endif %}
     {% endif %}
 ),
 FINAL AS (
@@ -41,17 +57,17 @@ FINAL AS (
     receipt_succeeded,
     _partition_by_block_number,
     _inserted_timestamp,
-    _load_timestamp,
-    {{ dbt_utils.generate_surrogate_key(
-      ['action_id']
-    ) }} AS actions_events_function_call_id,
-    SYSDATE() AS inserted_timestamp,
-    SYSDATE() AS modified_timestamp,
-    '{{ invocation_id }}' AS _invocation_id
+    _modified_timestamp
   FROM
     action_events
 )
 SELECT
-  *
+  *,
+  {{ dbt_utils.generate_surrogate_key(
+    ['action_id']
+  ) }} AS actions_events_function_call_id,
+  SYSDATE() AS inserted_timestamp,
+  SYSDATE() AS modified_timestamp,
+  '{{ invocation_id }}' AS _invocation_id
 FROM
   FINAL

models/silver/actions/silver__actions_events_function_call_s3.yml

@@ -57,6 +57,9 @@ models:
       - name: _INSERTED_TIMESTAMP
         description: "{{ doc('_inserted_timestamp')}}"
 
+      - name: _MODIFIED_TIMESTAMP
+        description: "{{ doc('_modified_timestamp')}}"
+
       - name: ACTIONS_EVENTS_FUNCTION_CALL_ID
         description: "{{doc('id')}}"
 

models/silver/actions/silver__actions_events_s3.sql

@@ -10,7 +10,20 @@
 WITH receipts AS (
 
     SELECT
-        *
+        tx_hash,
+        receipt_object_id,
+        receiver_id,
+        signer_id,
+        block_id,
+        block_timestamp,
+        chunk_hash,
+        logs,
+        receipt_actions,
+        execution_outcome,
+        receipt_succeeded,
+        _partition_by_block_number,
+        _inserted_timestamp,
+        modified_timestamp AS _modified_timestamp
     FROM
         {{ ref('silver__streamline_receipts_final') }}
 
@@ -18,8 +31,11 @@ WITH receipts AS (
         WHERE
             {{ partition_load_manual('no_buffer') }}
         {% else %}
-        WHERE
-            {{ incremental_load_filter('_inserted_timestamp') }}
+            {% if var('IS_MIGRATION') %}
+                WHERE {{ incremental_load_filter('_inserted_timestamp') }}
+            {% else %}
+                WHERE {{ incremental_load_filter('_modified_timestamp') }}
+            {% endif %}
         {% endif %}
 ),
 flatten_actions AS (
@@ -32,17 +48,14 @@ flatten_actions AS (
         block_timestamp,
         chunk_hash,
         logs,
-        _partition_by_block_number,
-        COALESCE(
-            _inserted_timestamp,
-            _load_timestamp
-        ) AS _inserted_timestamp,
-        _load_timestamp,
         receipt_actions,
         execution_outcome,
         VALUE AS action_object,
         INDEX AS action_index,
-        receipt_succeeded
+        receipt_succeeded,
+        _partition_by_block_number,
+        _inserted_timestamp,
+        _modified_timestamp
     FROM
         receipts,
         LATERAL FLATTEN(
@@ -56,13 +69,13 @@ FINAL AS (
             receipt_object_id,
             action_index
         ) AS action_id,
-        receiver_id,
-        signer_id,
-        chunk_hash,
-        tx_hash,
-        receipt_object_id,
         block_id,
         block_timestamp,
+        tx_hash,
+        receipt_object_id,
+        chunk_hash,
+        receiver_id,
+        signer_id,
         action_index,
         key AS action_name,
         TRY_PARSE_JSON(VALUE) AS action_data,
@@ -70,13 +83,7 @@ FINAL AS (
         receipt_succeeded,
         _partition_by_block_number,
         _inserted_timestamp,
-        _load_timestamp,
-        {{ dbt_utils.generate_surrogate_key(
-            ['receipt_object_id', 'action_index']
-        ) }} AS actions_events_id,
-        SYSDATE() AS inserted_timestamp,
-        SYSDATE() AS modified_timestamp,
-        '{{ invocation_id }}' AS _invocation_id
+        _modified_timestamp
     FROM
         flatten_actions,
         LATERAL FLATTEN(
@@ -84,6 +91,12 @@ FINAL AS (
         )
 )
 SELECT
-    *
+    *,
+    {{ dbt_utils.generate_surrogate_key(
+        ['receipt_object_id', 'action_index']
+    ) }} AS actions_events_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     FINAL

models/silver/actions/silver__actions_events_s3.yml

@@ -12,27 +12,6 @@ models:
           - unique:
               where: tx_hash != 'J4CZZQrZK6kYPVLkrdbTEpcqhUNZiRxktbMzHviqeGgf'
           - not_null
-
-      - name: RECEIVER_ID
-        description: "{{ doc('receiver_id')}}"
-
-      - name: SIGNER_ID
-        description: "{{ doc('signer_id')}}"
-
-      - name: CHUNK_HASH
-        description: "{{ doc('chunk_hash')}}"
-        tests:
-          - not_null:
-              where: "block_id not in (34691244, 34691277)"
-
-      - name: TX_HASH
-        description: "{{ doc('tx_hash')}}"
-        tests:
-          - not_null:
-              where: _inserted_timestamp <= current_timestamp - interval '1 hour'
-               
-      - name: RECEIPT_OBJECT_ID
-        description: "{{ doc('receipt_object_id')}}"
           
       - name: BLOCK_ID
         description: "{{ doc('block_id')}}"
@@ -43,6 +22,27 @@ models:
           - not_null:
               where: _inserted_timestamp <= current_timestamp - interval '1 hour'
 
+      - name: TX_HASH
+        description: "{{ doc('tx_hash')}}"
+        tests:
+          - not_null:
+              where: _inserted_timestamp <= current_timestamp - interval '1 hour'
+               
+      - name: RECEIPT_OBJECT_ID
+        description: "{{ doc('receipt_object_id')}}"
+
+      - name: CHUNK_HASH
+        description: "{{ doc('chunk_hash')}}"
+        tests:
+          - not_null:
+              where: "block_id not in (34691244, 34691277)"
+  
+      - name: RECEIVER_ID
+        description: "{{ doc('receiver_id')}}"
+
+      - name: SIGNER_ID
+        description: "{{ doc('signer_id')}}"
+
       - name: ACTION_INDEX
         description: "{{ doc('action_index')}}"
           
@@ -64,6 +64,9 @@ models:
       - name: _INSERTED_TIMESTAMP
         description: "{{ doc('_inserted_timestamp')}}"
 
+      - name: _MODIFIED_TIMESTAMP
+        description: "{{ doc('_modified_timestamp')}}"
+
       - name: ACTIONS_EVENTS_ID
         description: "{{doc('id')}}"