Merge pull request #18 from FlipsideCrypto/Update-PKs-and-add-d+i

update to PKs and change to d+i

.github/workflows/dbt_test_daily.yml

@@ -43,7 +43,7 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt test -m tag:test_recency tag:test_quality --vars '{"TEST_HOURS_THRESHOLD":24}'
+          dbt test -m tag:test_recency tag:test_quality --vars '{"TEST_HOURS_THRESHOLD":72}'
           
         continue-on-error: true
 

models/gold/core/core__fact_changes.sql

@@ -1,9 +1,7 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = ['tx_hash','change_index'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate","block_timestamp::DATE"],
-    merge_exclude_columns = ["inserted_timestamp"],
+    unique_key = 'version',
+    incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash,change_type,inner_change_type,change_address,change_module,change_resource,payload_function);",
     tags = ['core']

models/gold/core/core__fact_events.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = 'tx_hash',
+    unique_key = 'version',
     incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash,event_type,event_address,event_module,event_resource,payload_function);",

models/gold/core/core__fact_transactions.sql

@@ -1,9 +1,7 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = ['tx_hash','block_timestamp::DATE'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
-    merge_exclude_columns = ["inserted_timestamp"],
+    unique_key = 'version',
+    incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash,payload_function,sender);",
     tags = ['core']

models/gold/core/core__fact_transactions_block_metadata.sql

@@ -1,8 +1,7 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = ['tx_hash','block_timestamp::DATE'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate","block_timestamp::DATE"],
+    unique_key = 'version',
+    incremental_strategy = 'delete+insert',
     merge_exclude_columns = ['inserted_timestamp'],
     cluster_by = ['block_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash);",
@@ -26,7 +25,7 @@ SELECT
     id,
     previous_block_votes_bitvec,
     proposer,
-    round,
+    ROUND,
     vm_status,
     state_change_hash,
     accumulator_root_hash,

models/gold/core/core__fact_transactions_state_checkpoint.sql

@@ -1,8 +1,7 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = ['tx_hash','block_timestamp::DATE'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
+    unique_key = 'version',
+    incremental_strategy = 'delete+insert',
     merge_exclude_columns = ["inserted_timestamp"],
     cluster_by = ['block_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash);",

models/gold/core/core__fact_transfers.sql

@@ -1,9 +1,7 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = ['tx_hash','event_index','version','block_timestamp::DATE'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
-    merge_exclude_columns = ["inserted_timestamp"],
+    unique_key = 'version',
+    incremental_strategy = 'delete+insert',
     cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, version, account_address,token_address);",
     tags = ['core']
@@ -43,9 +41,7 @@ AND modified_timestamp >= (
         {{ this }}
 )
 {% endif %}
-
 UNION ALL
-
 SELECT
     block_number,
     block_timestamp,
@@ -75,4 +71,4 @@ AND modified_timestamp >= (
     FROM
         {{ this }}
 )
-{% endif %}
+{% endif %}

models/silver/core/silver__changes.sql

@@ -1,57 +0,0 @@
-{{ config(
-    materialized = 'incremental',
-    unique_key = ['tx_hash', 'change_index'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],    
-    merge_exclude_columns = ['inserted_timestamp'],
-    cluster_by = ['modified_timestamp::DATE'],
-    tags = ['core'],
-    enabled = false
-) }}
--- depends_on: {{ ref('core__fact_transactions') }}
-SELECT
-    A.block_number,
-    block_timestamp,
-    tx_hash,
-    version,
-    success,
-    tx_type,
-    payload_function,
-    index AS change_index,
-    DATA :data AS change_data,
-    DATA :type :: STRING AS change_type,
-    DATA :address :: STRING AS address,
-    DATA :handle :: STRING AS handle,
-    DATA :"type" :: STRING AS inner_change_type,
-    SPLIT_PART(
-        inner_change_type,
-        '::',
-        1
-    ) :: STRING AS change_address,
-    SPLIT_PART(
-        inner_change_type,
-        '::',
-        2
-    ) :: STRING AS change_module,
-    SUBSTRING(inner_change_type, len(change_address) + len(change_module) + 5) AS change_resource,
-    VALUE :key :: STRING AS key,
-    VALUE :value :: STRING AS VALUE,
-    VALUE :state_key_hash :: STRING AS state_key_hash,
-    {{ dbt_utils.generate_surrogate_key(
-        ['tx_hash','change_index']
-    ) }} AS changes_id,
-    inserted_timestamp,
-    modified_timestamp
-FROM
-    {{ ref(
-        'core__fact_transactions'
-    ) }}
-
-{% if is_incremental() %}
-WHERE
-  A.modified_timestamp >= (
-    SELECT
-      DATEADD('minute', -15, MAX(modified_timestamp))
-    FROM
-      {{ this }})
-{% endif %}

models/silver/core/silver__events.sql

@@ -1,60 +0,0 @@
-{{ config(
-    materialized = 'incremental',
-    unique_key = ['tx_hash','event_index'],
-    incremental_strategy = 'merge',
-    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],    
-    merge_exclude_columns = ['inserted_timestamp'],
-    cluster_by = ['modified_timestamp::DATE'],
-    tags = ['core'],
-    enabled = false
-) }}
--- depends_on: {{ ref('core__fact_transactions') }}
-SELECT
-    A.block_number,
-    A.block_timestamp,
-    A.tx_hash,
-    version,
-    success,
-    A.tx_type,
-    A.payload_function,
-    b.index AS event_index,
-    b.value :type :: STRING AS event_type,
-    SPLIT_PART(
-        event_type,
-        '::',
-        1
-    ) :: STRING AS event_address,
-    SPLIT_PART(
-        event_type,
-        '::',
-        2
-    ) :: STRING AS event_module,
-    SPLIT_PART(
-        event_type,
-        '::',
-        3
-    ) :: STRING AS event_resource,
-    b.value :data AS event_data,
-    -- b.value :guid :: STRING AS event_guid, -- extract into account_address + creation_number
-    b.value :guid :account_address :: STRING AS account_address,
-    b.value :guid :creation_number :: bigint AS creation_number,
-    b.value :sequence_number :: bigint AS sequence_number,
-    {{ dbt_utils.generate_surrogate_key(
-        ['tx_hash','event_index']
-    ) }} AS events_id,
-    inserted_timestamp,
-    modified_timestamp
-FROM
-    {{ ref(
-        'core__fact_transactions'
-    ) }} A,
-    LATERAL FLATTEN (events) b
-
-{% if is_incremental() %}
-WHERE
-    modified_timestamp >= (
-        SELECT
-            DATEADD('minute', -15, MAX(modified_timestamp))
-        FROM
-            {{ this }})
-{% endif %}

models/silver/core/silver__fungiblestore_owners.sql

@@ -1,12 +1,11 @@
 {{ config(
   materialized = 'incremental',
-  unique_key = ['tx_hash','change_index'],
+  unique_key = ['version','change_index'],
   incremental_strategy = 'merge',
   cluster_by = ['block_timestamp::DATE'],
   post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(store_address);",
   tags = ['core']
 ) }}
-
 -- depends_on: {{ ref('core__fact_changes') }}
 
 SELECT

models/silver/core/silver__transactions.sql

@@ -1,9 +1,7 @@
 {{ config(
   materialized = 'incremental',
-  unique_key = ['tx_hash', 'block_timestamp::DATE'],
-  incremental_strategy = 'merge',
-  incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"],
-  merge_exclude_columns = ['inserted_timestamp'],
+  unique_key = 'version',
+  incremental_strategy = 'delete+insert',
   cluster_by = ['modified_timestamp::DATE'],
   tags = ['core']
 ) }}
@@ -11,7 +9,7 @@
 WITH from_transactions AS (
 
   SELECT
-    value :BLOCK_NUMBER :: bigint AS block_number,
+    VALUE :BLOCK_NUMBER :: bigint AS block_number,
     TO_TIMESTAMP(
       VALUE :BLOCK_TIMESTAMP :: STRING
     ) AS block_timestamp,
@@ -21,23 +19,23 @@ WITH from_transactions AS (
     DATA,
     inserted_timestamp AS file_last_updated
   FROM
-  {% if is_incremental() %}
-    {{ ref('bronze__transactions') }}
-  {% else %}
-    {{ ref('bronze__transactions_FR') }}
+
+{% if is_incremental() %}
+{{ ref('bronze__transactions') }}
+{% else %}
+  {{ ref('bronze__transactions_FR') }}
+{% endif %}
+WHERE
+  version BETWEEN VALUE :FIRST_VERSION :: bigint
+  AND VALUE :LAST_VERSION :: bigint
+
+{% if is_incremental() %}
+AND inserted_timestamp >= (
+  SELECT
+    DATEADD('minute', -5, MAX(modified_timestamp))
+  FROM
+    {{ this }})
   {% endif %}
-
-  WHERE
-    version BETWEEN VALUE :FIRST_VERSION :: bigint
-    AND VALUE :LAST_VERSION :: bigint
-
-  {% if is_incremental() %}
-  AND inserted_timestamp >= (
-    SELECT
-      DATEADD('minute', -5, MAX(modified_timestamp))
-    FROM
-      {{ this }})
-    {% endif %}
 ),
 transformed AS (
   SELECT
@@ -70,7 +68,7 @@ transformed AS (
     DATA :id :: STRING AS id,
     DATA :previous_block_votes_bitvec AS previous_block_votes_bitvec,
     DATA :proposer :: STRING AS proposer,
-    DATA :round :: INT AS round,
+    DATA :round :: INT AS ROUND,
     DATA,
     file_last_updated
   FROM
@@ -111,6 +109,6 @@ SELECT
   SYSDATE() AS modified_timestamp,
   '{{ invocation_id }}' AS _invocation_id
 FROM
-  transformed qualify(ROW_NUMBER() over (PARTITION BY tx_hash
+  transformed qualify(ROW_NUMBER() over (PARTITION BY version
 ORDER BY
   file_last_updated DESC)) = 1

models/silver/core/silver__transfers.sql

@@ -1,8 +1,7 @@
 {{ config(
   materialized = 'incremental',
-  unique_key = ['tx_hash','event_index','block_timestamp::DATE'],
-  incremental_strategy = 'merge',
-  merge_exclude_columns = ["inserted_timestamp"],
+  unique_key = 'version',
+  incremental_strategy = 'delete+insert',
   cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
   tags = ['core']
 ) }}

models/silver/core/silver__transfers_fungible.sql

@@ -1,16 +1,13 @@
 {{ config(
   materialized = 'incremental',
-  unique_key = ['tx_hash','event_index','block_timestamp::DATE'],
-  incremental_strategy = 'merge',
-  merge_exclude_columns = ["inserted_timestamp"],
+  unique_key = 'version',
+  incremental_strategy = 'delete+insert',
   cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
   tags = ['core']
 ) }}
-
 -- depends_on: {{ ref('core__fact_events') }}
 -- depends_on: {{ ref('silver__fungiblestore_owners') }}
 -- depends_on: {{ ref('silver__fungiblestore_metadata') }}
-
 {% if execute %}
   {% set base_query %}
   CREATE

models/silver/core/silver_core.yml

@@ -310,8 +310,7 @@ models:
         tests:
           - not_null:
               tags: ['test_quality']
-          - dbt_utils.expression_is_true:
-              expression: ">= 0"
+          - unique:
               tags: ['test_quality']
       - name: tx_type
         data_type: VARCHAR