diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index bf2481c..1b73c8c 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -43,7 +43,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test -m tag:test_recency tag:test_quality --vars '{"TEST_HOURS_THRESHOLD":24}' + dbt test -m tag:test_recency tag:test_quality --vars '{"TEST_HOURS_THRESHOLD":72}' continue-on-error: true diff --git a/models/gold/core/core__fact_changes.sql b/models/gold/core/core__fact_changes.sql index 1a4db02..9911b92 100644 --- a/models/gold/core/core__fact_changes.sql +++ b/models/gold/core/core__fact_changes.sql @@ -1,9 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','change_index'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate","block_timestamp::DATE"], - merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'version', + incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash,change_type,inner_change_type,change_address,change_module,change_resource,payload_function);", tags = ['core'] diff --git a/models/gold/core/core__fact_events.sql b/models/gold/core/core__fact_events.sql index 9fc3119..f628cc3 100644 --- a/models/gold/core/core__fact_events.sql +++ b/models/gold/core/core__fact_events.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = 'tx_hash', + unique_key = 'version', incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash,event_type,event_address,event_module,event_resource,payload_function);", diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 1335f18..3155ff3 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -1,9 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','block_timestamp::DATE'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], - merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'version', + incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash,payload_function,sender);", tags = ['core'] diff --git a/models/gold/core/core__fact_transactions_block_metadata.sql b/models/gold/core/core__fact_transactions_block_metadata.sql index 0a3d051..db5255d 100644 --- a/models/gold/core/core__fact_transactions_block_metadata.sql +++ b/models/gold/core/core__fact_transactions_block_metadata.sql @@ -1,8 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','block_timestamp::DATE'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate","block_timestamp::DATE"], + unique_key = 'version', + incremental_strategy = 'delete+insert', merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash);", @@ -26,7 +25,7 @@ SELECT id, previous_block_votes_bitvec, proposer, - round, + ROUND, vm_status, state_change_hash, accumulator_root_hash, diff --git a/models/gold/core/core__fact_transactions_state_checkpoint.sql b/models/gold/core/core__fact_transactions_state_checkpoint.sql index 92f03ab..23ae7cb 100644 --- a/models/gold/core/core__fact_transactions_state_checkpoint.sql +++ b/models/gold/core/core__fact_transactions_state_checkpoint.sql @@ -1,8 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','block_timestamp::DATE'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], + unique_key = 'version', + incremental_strategy = 'delete+insert', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(version,tx_hash);", diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql index 40c7ba0..6b4efb1 100644 --- a/models/gold/core/core__fact_transfers.sql +++ b/models/gold/core/core__fact_transfers.sql @@ -1,9 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','event_index','version','block_timestamp::DATE'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], - merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'version', + incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, version, account_address,token_address);", tags = ['core'] @@ -43,9 +41,7 @@ AND modified_timestamp >= ( {{ this }} ) {% endif %} - UNION ALL - SELECT block_number, block_timestamp, @@ -75,4 +71,4 @@ AND modified_timestamp >= ( FROM {{ this }} ) -{% endif %} \ No newline at end of file +{% endif %} diff --git a/models/silver/core/silver__changes.sql b/models/silver/core/silver__changes.sql deleted file mode 100644 index 149ee38..0000000 --- a/models/silver/core/silver__changes.sql +++ /dev/null @@ -1,57 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = ['tx_hash', 'change_index'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], - merge_exclude_columns = ['inserted_timestamp'], - cluster_by = ['modified_timestamp::DATE'], - tags = ['core'], - enabled = false -) }} --- depends_on: {{ ref('core__fact_transactions') }} -SELECT - A.block_number, - block_timestamp, - tx_hash, - version, - success, - tx_type, - payload_function, - index AS change_index, - DATA :data AS change_data, - DATA :type :: STRING AS change_type, - DATA :address :: STRING AS address, - DATA :handle :: STRING AS handle, - DATA :"type" :: STRING AS inner_change_type, - SPLIT_PART( - inner_change_type, - '::', - 1 - ) :: STRING AS change_address, - SPLIT_PART( - inner_change_type, - '::', - 2 - ) :: STRING AS change_module, - SUBSTRING(inner_change_type, len(change_address) + len(change_module) + 5) AS change_resource, - VALUE :key :: STRING AS key, - VALUE :value :: STRING AS VALUE, - VALUE :state_key_hash :: STRING AS state_key_hash, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash','change_index'] - ) }} AS changes_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref( - 'core__fact_transactions' - ) }} - -{% if is_incremental() %} -WHERE - A.modified_timestamp >= ( - SELECT - DATEADD('minute', -15, MAX(modified_timestamp)) - FROM - {{ this }}) -{% endif %} diff --git a/models/silver/core/silver__events.sql b/models/silver/core/silver__events.sql deleted file mode 100644 index acdb15d..0000000 --- a/models/silver/core/silver__events.sql +++ /dev/null @@ -1,60 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = ['tx_hash','event_index'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], - merge_exclude_columns = ['inserted_timestamp'], - cluster_by = ['modified_timestamp::DATE'], - tags = ['core'], - enabled = false -) }} --- depends_on: {{ ref('core__fact_transactions') }} -SELECT - A.block_number, - A.block_timestamp, - A.tx_hash, - version, - success, - A.tx_type, - A.payload_function, - b.index AS event_index, - b.value :type :: STRING AS event_type, - SPLIT_PART( - event_type, - '::', - 1 - ) :: STRING AS event_address, - SPLIT_PART( - event_type, - '::', - 2 - ) :: STRING AS event_module, - SPLIT_PART( - event_type, - '::', - 3 - ) :: STRING AS event_resource, - b.value :data AS event_data, - -- b.value :guid :: STRING AS event_guid, -- extract into account_address + creation_number - b.value :guid :account_address :: STRING AS account_address, - b.value :guid :creation_number :: bigint AS creation_number, - b.value :sequence_number :: bigint AS sequence_number, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash','event_index'] - ) }} AS events_id, - inserted_timestamp, - modified_timestamp -FROM - {{ ref( - 'core__fact_transactions' - ) }} A, - LATERAL FLATTEN (events) b - -{% if is_incremental() %} -WHERE - modified_timestamp >= ( - SELECT - DATEADD('minute', -15, MAX(modified_timestamp)) - FROM - {{ this }}) -{% endif %} diff --git a/models/silver/core/silver__fungiblestore_owners.sql b/models/silver/core/silver__fungiblestore_owners.sql index ba38ee5..294f4f5 100644 --- a/models/silver/core/silver__fungiblestore_owners.sql +++ b/models/silver/core/silver__fungiblestore_owners.sql @@ -1,12 +1,11 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','change_index'], + unique_key = ['version','change_index'], incremental_strategy = 'merge', cluster_by = ['block_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(store_address);", tags = ['core'] ) }} - -- depends_on: {{ ref('core__fact_changes') }} SELECT diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index aed1fde..5d227ef 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -1,9 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash', 'block_timestamp::DATE'], - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate", "block_timestamp::DATE"], - merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'version', + incremental_strategy = 'delete+insert', cluster_by = ['modified_timestamp::DATE'], tags = ['core'] ) }} @@ -11,7 +9,7 @@ WITH from_transactions AS ( SELECT - value :BLOCK_NUMBER :: bigint AS block_number, + VALUE :BLOCK_NUMBER :: bigint AS block_number, TO_TIMESTAMP( VALUE :BLOCK_TIMESTAMP :: STRING ) AS block_timestamp, @@ -21,23 +19,23 @@ WITH from_transactions AS ( DATA, inserted_timestamp AS file_last_updated FROM - {% if is_incremental() %} - {{ ref('bronze__transactions') }} - {% else %} - {{ ref('bronze__transactions_FR') }} + +{% if is_incremental() %} +{{ ref('bronze__transactions') }} +{% else %} + {{ ref('bronze__transactions_FR') }} +{% endif %} +WHERE + version BETWEEN VALUE :FIRST_VERSION :: bigint + AND VALUE :LAST_VERSION :: bigint + +{% if is_incremental() %} +AND inserted_timestamp >= ( + SELECT + DATEADD('minute', -5, MAX(modified_timestamp)) + FROM + {{ this }}) {% endif %} - - WHERE - version BETWEEN VALUE :FIRST_VERSION :: bigint - AND VALUE :LAST_VERSION :: bigint - - {% if is_incremental() %} - AND inserted_timestamp >= ( - SELECT - DATEADD('minute', -5, MAX(modified_timestamp)) - FROM - {{ this }}) - {% endif %} ), transformed AS ( SELECT @@ -70,7 +68,7 @@ transformed AS ( DATA :id :: STRING AS id, DATA :previous_block_votes_bitvec AS previous_block_votes_bitvec, DATA :proposer :: STRING AS proposer, - DATA :round :: INT AS round, + DATA :round :: INT AS ROUND, DATA, file_last_updated FROM @@ -111,6 +109,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - transformed qualify(ROW_NUMBER() over (PARTITION BY tx_hash + transformed qualify(ROW_NUMBER() over (PARTITION BY version ORDER BY file_last_updated DESC)) = 1 diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index a688a22..f50f87f 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -1,8 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','event_index','block_timestamp::DATE'], - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'version', + incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], tags = ['core'] ) }} diff --git a/models/silver/core/silver__transfers_fungible.sql b/models/silver/core/silver__transfers_fungible.sql index d9d484d..f2e14b9 100644 --- a/models/silver/core/silver__transfers_fungible.sql +++ b/models/silver/core/silver__transfers_fungible.sql @@ -1,16 +1,13 @@ {{ config( materialized = 'incremental', - unique_key = ['tx_hash','event_index','block_timestamp::DATE'], - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'version', + incremental_strategy = 'delete+insert', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], tags = ['core'] ) }} - -- depends_on: {{ ref('core__fact_events') }} -- depends_on: {{ ref('silver__fungiblestore_owners') }} -- depends_on: {{ ref('silver__fungiblestore_metadata') }} - {% if execute %} {% set base_query %} CREATE diff --git a/models/silver/core/silver_core.yml b/models/silver/core/silver_core.yml index bd982d0..d0777ee 100644 --- a/models/silver/core/silver_core.yml +++ b/models/silver/core/silver_core.yml @@ -310,8 +310,7 @@ models: tests: - not_null: tags: ['test_quality'] - - dbt_utils.expression_is_true: - expression: ">= 0" + - unique: tags: ['test_quality'] - name: tx_type data_type: VARCHAR