pk invo dates

models/gold/core/core__dim_address_labels.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    secure = true
+    tags = ['core']
 ) }}
 
 SELECT
@@ -12,7 +12,21 @@ SELECT
     label,
     project_name,
     decimals,
-    deployment_tx_id
+    deployment_tx_id,
+    COALESCE (
+        address_labels_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['address']
+        ) }}
+    ) AS dim_address_labels_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__address_labels') }}
 WHERE

models/gold/core/core__ez_messages.sql

@@ -21,6 +21,20 @@ SELECT
     message_index,
     message_type,
     message_value,
-    attributes
+    attributes,
+    COALESCE (
+        messages_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['message_id']
+        ) }}
+    ) AS ez_messages_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     messages

models/gold/core/core__ez_transfers.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    secure = true
+    tags = ['core']
 ) }}
 
 SELECT
@@ -22,7 +22,21 @@ SELECT
     sender,
     receiver,
     'terra' blockchain,
-    transfer_type
+    transfer_type,
+    COALESCE (
+        transfers_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_id', 'msg_index']
+        ) }}
+    ) AS ez_transfers_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__transfers') }}
 UNION ALL
@@ -45,6 +59,20 @@ SELECT
     sender,
     receiver,
     'terra' blockchain,
-    transfer_type
+    transfer_type,
+    COALESCE (
+        transfers_ibc_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_id', 'msg_index']
+        ) }}
+    ) AS ez_transfers_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__transfers_ibc') }}

models/gold/core/core__fact_blocks.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    secure = true
+    tags = ['core']
 ) }}
 
 WITH blocks AS (
@@ -10,8 +10,7 @@ WITH blocks AS (
     FROM
         {{ ref('silver__blocks') }}
 ),
-final AS (
-
+FINAL AS (
     SELECT
         block_id,
         block_timestamp,
@@ -30,9 +29,25 @@ final AS (
         next_validators_hash,
         proposer_address,
         validators_hash,
-        validator_address_array
+        validator_address_array,
+        COALESCE (
+            blocks_id,
+            {{ dbt_utils.generate_surrogate_key(
+                ['block_id']
+            ) }}
+        ) AS fact_blocks_id,
+        COALESCE(
+            inserted_timestamp,
+            '2000-01-01'
+        ) AS inserted_timestamp,
+        COALESCE(
+            modified_timestamp,
+            '2000-01-01'
+        ) AS modified_timestamp
     FROM
         blocks
 )
-
-SELECT * FROM final
+SELECT
+    *
+FROM
+    FINAL

models/gold/core/core__fact_msg_attributes.sql

@@ -1,16 +1,15 @@
 {{ config(
-    materialized = 'view',
-    secure = true
+  materialized = 'view',
+  tags = ['core']
 ) }}
 
 WITH msg_attributes AS (
 
-    SELECT
-        *
-    FROM
-        {{ ref('silver__msg_attributes') }}
+  SELECT
+    *
+  FROM
+    {{ ref('silver__msg_attributes') }}
 )
-
 SELECT
   block_id,
   block_timestamp,
@@ -23,6 +22,20 @@ SELECT
   msg_type,
   attribute_key,
   attribute_value,
-  attribute_index
+  attribute_index,
+  COALESCE (
+    msg_attributes_id,
+    {{ dbt_utils.generate_surrogate_key(
+      ['message_id']
+    ) }}
+  ) AS fact_msg_attributes_id,
+  COALESCE(
+    inserted_timestamp,
+    '2000-01-01'
+  ) AS inserted_timestamp,
+  COALESCE(
+    modified_timestamp,
+    '2000-01-01'
+  ) AS modified_timestamp
 FROM
-  msg_attributes
+  msg_attributes

models/gold/core/core__fact_msg_attributes_standard.sql

@@ -1,6 +1,6 @@
 {{ config(
   materialized = 'view',
-  secure = true
+  tags = ['core']
 ) }}
 
 WITH msg_attributes AS (
@@ -21,6 +21,20 @@ SELECT
   msg_type,
   attribute_index,
   attribute_key,
-  attribute_value
+  attribute_value,
+  COALESCE (
+    msg_attributes_2_id,
+    {{ dbt_utils.generate_surrogate_key(
+      ['_unique_key']
+    ) }}
+  ) AS fact_msg_attributes_standard_id,
+  COALESCE(
+    inserted_timestamp,
+    '2000-01-01'
+  ) AS inserted_timestamp,
+  COALESCE(
+    modified_timestamp,
+    '2000-01-01'
+  ) AS modified_timestamp
 FROM
   msg_attributes

models/gold/core/core__fact_msgs.sql

@@ -1,16 +1,15 @@
 {{ config(
-    materialized = 'view',
-    secure = true
+  materialized = 'view',
+  tags = ['core']
 ) }}
 
 WITH messages AS (
 
-    SELECT
-        *
-    FROM
-        {{ ref('silver__msgs') }}
+  SELECT
+    *
+  FROM
+    {{ ref('silver__msgs') }}
 )
-
 SELECT
   block_id,
   block_timestamp,
@@ -21,6 +20,20 @@ SELECT
   msg_group,
   msg_index,
   msg_type,
-  msg
+  msg,
+  COALESCE (
+    msgs_id,
+    {{ dbt_utils.generate_surrogate_key(
+      ['message_id']
+    ) }}
+  ) AS fact_msgs_id,
+  COALESCE(
+    inserted_timestamp,
+    '2000-01-01'
+  ) AS inserted_timestamp,
+  COALESCE(
+    modified_timestamp,
+    '2000-01-01'
+  ) AS modified_timestamp
 FROM
-  messages
+  messages

models/gold/core/core__fact_transactions.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    secure = true
+    tags = ['core']
 ) }}
 
 SELECT
@@ -21,6 +21,20 @@ SELECT
     codespace,
     tx_code,
     tx_succeeded,
-    tx
+    tx,
+    COALESCE (
+        transactions_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_id']
+        ) }}
+    ) AS fact_transactions_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__transactions') }}

models/gold/defi/defi__ez_staking.sql

@@ -25,7 +25,21 @@ SELECT
    validator_src_address,
    validator_label.label AS validator_label,
    validator_src_label.label AS validator_src_label,
-   staking_id
+   staking_id,
+   COALESCE (
+      stakings_id,
+      {{ dbt_utils.generate_surrogate_key(
+         ['staking_id']
+      ) }}
+   ) AS ez_staking_id,
+   COALESCE(
+      staking.inserted_timestamp,
+      '2000-01-01'
+   ) AS inserted_timestamp,
+   COALESCE(
+      staking.modified_timestamp,
+      '2000-01-01'
+   ) AS modified_timestamp
 FROM
    staking
    LEFT OUTER JOIN {{ ref('core__dim_address_labels') }}

models/gold/defi/defi__ez_swaps.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = "view",
-    tags = ['core']
+    tags = ['core'],
+    enabled = false
 ) }}
 
 WITH swap AS (

models/silver/silver__address_labels.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'address',
+    merge_exclude_columns = ["inserted_timestamp"],
     tags = ['noncore']
 ) }}
 
@@ -80,7 +81,13 @@ FINAL AS (
         GREATEST(
             l._inserted_timestamp,
             t._inserted_timestamp
-        ) AS _inserted_timestamp
+        ) AS _inserted_timestamp,
+        {{ dbt_utils.generate_surrogate_key(
+            ['COALESCE( t.address, l.address )']
+        ) }} AS address_labels_id,
+        SYSDATE() AS inserted_timestamp,
+        SYSDATE() AS modified_timestamp,
+        '{{ invocation_id }}' AS _invocation_id
     FROM
         labels l full
         JOIN tokens t USING (

models/silver/silver__blockchain.sql

@@ -62,7 +62,14 @@ SELECT
     block_header,
     block_id AS block_id_object,
     num_txs,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['block_id']
+    ) }} AS blockchain_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    _inserted_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     fin qualify(ROW_NUMBER() over(PARTITION BY height
 ORDER BY

models/silver/silver__blocks.sql

@@ -157,7 +157,13 @@ FINAL AS (
         base_blocks.header :validators_hash :: STRING AS validators_hash,
         base_blocks._inserted_timestamp AS _ingested_at,
         base_blocks._inserted_timestamp AS _inserted_timestamp,
-        validators_address_array.validator_address_array :: ARRAY AS validator_address_array
+        validators_address_array.validator_address_array :: ARRAY AS validator_address_array,
+        {{ dbt_utils.generate_surrogate_key(
+            ['base_blocks.block_id']
+        ) }} AS blocks_id,
+        SYSDATE() AS inserted_timestamp,
+        SYSDATE() AS modified_timestamp,
+        '{{ invocation_id }}' AS _invocation_id
     FROM
         base_blocks
         LEFT JOIN validators_address_array

models/silver/silver__blocks_val_array.sql

@@ -159,6 +159,12 @@ validators_address_array AS (
         _inserted_timestamp
 )
 SELECT
-    *
+    *,
+    {{ dbt_utils.generate_surrogate_key(
+        ['block_id']
+    ) }} AS blocks_val_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     validators_address_array

models/silver/silver__dex_swaps.sql

@@ -3,7 +3,8 @@
     unique_key = "SWAP_ID",
     incremental_strategy = "delete+insert",
     cluster_by = ["block_timestamp::DATE", "_inserted_timestamp::DATE"],
-    tags = ['noncore']
+    tags = ['noncore'],
+    enabled = false
 ) }}
 
 WITH all_swaps AS(

models/silver/silver__messages.sql

@@ -1,7 +1,8 @@
 {{ config(
     materialized = "incremental",
     cluster_by = ["_inserted_timestamp"],
-    unique_key = "message_id"
+    unique_key = "message_id",
+    merge_exclude_columns = ["inserted_timestamp"]
 ) }}
 
 WITH txs AS (
@@ -218,7 +219,13 @@ final_table AS (
             message_index
         ) AS attributes,
         _ingested_at,
-        _inserted_timestamp
+        _inserted_timestamp,
+        {{ dbt_utils.generate_surrogate_key(
+            ['message_id']
+        ) }} AS messages_id,
+        SYSDATE() AS inserted_timestamp,
+        SYSDATE() AS modified_timestamp,
+        '{{ invocation_id }}' AS _invocation_id
     FROM
         distinct_events_table
 )

models/silver/silver__msg_attributes.sql

@@ -2,6 +2,7 @@
   materialized = "incremental",
   cluster_by = ["_inserted_timestamp"],
   unique_key = "message_id",
+  merge_exclude_columns = ["inserted_timestamp"],
   incremental_strategy = 'delete+insert',
   tags = ['core']
 ) }}
@@ -223,7 +224,13 @@ FINAL AS (
     attribute_key,
     attribute_value,
     _ingested_at,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+      ['message_id']
+    ) }} AS msg_attributes_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
   FROM
     msg_attribute
 )

models/silver/silver__msg_attributes_2.sql

@@ -2,6 +2,7 @@
   materialized = 'incremental',
   unique_key = "_unique_key",
   incremental_strategy = 'merge',
+  merge_exclude_columns = ["inserted_timestamp"],
   cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
   post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
   tags = ['core']
@@ -35,7 +36,13 @@ SELECT
     tx_id,
     msg_index,
     attribute_index
-  ) AS _unique_key
+  ) AS _unique_key,
+  {{ dbt_utils.generate_surrogate_key(
+    ['_unique_key']
+  ) }} AS msg_attributes_2_id,
+  SYSDATE() AS inserted_timestamp,
+  SYSDATE() AS modified_timestamp,
+  '{{ invocation_id }}' AS _invocation_id
 FROM
   {{ ref('silver__msgs_2') }} A,
   LATERAL FLATTEN(

models/silver/silver__msgs.sql

@@ -191,7 +191,13 @@ FINAL AS (
     msg_type,
     msg,
     _ingested_at,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+      ['message_id']
+    ) }} AS msgs_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
   FROM
     prefinal
 )

models/silver/silver__msgs_2.sql

@@ -2,6 +2,7 @@
   materialized = 'incremental',
   unique_key = "_unique_key",
   incremental_strategy = 'merge',
+  merge_exclude_columns = ["inserted_timestamp"],
   cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
   post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
   tags = ['core']
@@ -160,7 +161,13 @@ SELECT
     '-',
     A.tx_id,
     A.msg_index
-  ) AS _unique_key
+  ) AS _unique_key,
+  {{ dbt_utils.generate_surrogate_key(
+    ['_unique_key']
+  ) }} AS msgs_2_id,
+  SYSDATE() AS inserted_timestamp,
+  SYSDATE() AS modified_timestamp,
+  '{{ invocation_id }}' AS _invocation_id
 FROM
   prefinal A
   LEFT JOIN grp b

models/silver/silver__staking.sql

@@ -2,6 +2,7 @@
     materialized = 'incremental',
     unique_key = "staking_id",
     incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE'],
     tags = ['noncore']
 ) }}
@@ -375,6 +376,12 @@ SELECT
     A.validator_address,
     A.redelegate_source_validator_address AS validator_src_address,
     A.completion_time,
-    A._inserted_timestamp
+    A._inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['staking_id']
+    ) }} AS stakings_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     add_dec A

models/silver/silver__token_labels.sql

@@ -2,6 +2,7 @@
     materialized = "incremental",
     cluster_by = ["_inserted_timestamp::DATE"],
     unique_key = "address",
+    merge_exclude_columns = ["inserted_timestamp"],
     tags = ['noncore']
 ) }}
 
@@ -35,6 +36,12 @@ SELECT
     address,
     decimals,
     _ingested_at,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['address']
+    ) }} AS token_labels_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     token_labels

models/silver/silver__transactions.sql

@@ -2,6 +2,7 @@
     materialized = 'incremental',
     cluster_by = ['_inserted_timestamp::DATE'],
     unique_key = 'tx_id',
+    merge_exclude_columns = ["inserted_timestamp"],
     tags = ['core']
 ) }}
 -- depends_on: {{ ref('bronze__streamline_transactions') }}
@@ -249,7 +250,13 @@ SELECT
     A.tx_succeeded,
     A.tx,
     A._ingested_at,
-    A._inserted_timestamp
+    A._inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['a.tx_id']
+    ) }} AS transactions_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     silver_txs A
     JOIN silver_blocks b

models/silver/silver__transfers.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['tx_id','msg_index'],
+    merge_exclude_columns = ["inserted_timestamp"],
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
     tags = ['core']
@@ -132,7 +133,13 @@ SELECT
         amount_int AS INT
     ) AS amount,
     currency,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['tx_id','msg_index']
+    ) }} AS transfers_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     new_fin
 WHERE

models/silver/silver__transfers_ibc.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['tx_id','msg_index'],
+    merge_exclude_columns = ["inserted_timestamp"],
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
     tags = ['core']
@@ -266,7 +267,13 @@ SELECT
     ) AS receiver,
     amount_int :: INT AS amount,
     currency,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['tx_id','msg_index']
+    ) }} AS transfers_ibc_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     new_fin
 WHERE

models/silver/silver__transfers_ibc_packets.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = ['tx_id','msg_index'],
+    merge_exclude_columns = ["inserted_timestamp"],
     incremental_strategy = 'merge',
     cluster_by = ['block_timestamp::DATE'],
     tags = ['core']
@@ -88,6 +89,12 @@ SELECT
     packet_sequence,
     packet_src_channel,
     packet_dst_channel,
-    _inserted_timestamp
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['tx_id','msg_index']
+    ) }} AS transfers_ibc_packets_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
 FROM
     all_data