diff --git a/dbt_project.yml b/dbt_project.yml index a2576fc..c48e7c3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -57,6 +57,16 @@ models: columns: true +on_schema_change: "append_new_columns" + movement_models: + gold: # This will apply to ALL models under the gold directory + +tests: + - dbt_utils.recency: + datepart: hour + field: BLOCK_TIMESTAMP + interval: 3 + severity: error + tags: ['recency'] + # In this example config, we tell dbt to build all models in the example/ directory # as tables. These settings can be overridden in the individual model files # using the `{{ config(...) }}` macro. diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index b4e49bb..4215d4e 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -2,7 +2,6 @@ version: 2 models: - name: core__fact_blocks description: '{{ doc("core__fact_blocks") }}' - columns: - name: block_number description: '{{ doc("block_number") }}' diff --git a/models/gold/core/core__fact_changes.yml b/models/gold/core/core__fact_changes.yml index 9839df5..5ebfc18 100644 --- a/models/gold/core/core__fact_changes.yml +++ b/models/gold/core/core__fact_changes.yml @@ -2,7 +2,6 @@ version: 2 models: - name: core__fact_changes description: '{{ doc("core__fact_changes") }}' - columns: - name: block_number description: '{{ doc("block_number") }}' diff --git a/models/gold/core/core__fact_events.yml b/models/gold/core/core__fact_events.yml index 2adacb3..29d3c07 100644 --- a/models/gold/core/core__fact_events.yml +++ b/models/gold/core/core__fact_events.yml @@ -2,7 +2,6 @@ version: 2 models: - name: core__fact_events description: '{{ doc("core__fact_events") }}' - columns: - name: block_number description: '{{ doc("block_number") }}' diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index d7516c8..e66dd43 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -2,12 +2,18 @@ version: 2 models: - name: core__fact_transactions description: '{{ doc("core__fact_transactions") }}' - + tests: + - dbt_utils.recency: + datepart: hour + field: BLOCK_TIMESTAMP + interval: 3 + severity: error + tags: ['recency'] columns: - name: block_number description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP - description: '{{ doc("block_timestamp") }}' + description: '{{ doc("block_timestamp") }}' - name: TX_HASH description: '{{ doc("tx_hash") }}' - name: VERSION diff --git a/models/gold/core/core__fact_transactions_block_metadata.sql b/models/gold/core/core__fact_transactions_block_metadata.sql index 2ec74e4..977a73d 100644 --- a/models/gold/core/core__fact_transactions_block_metadata.sql +++ b/models/gold/core/core__fact_transactions_block_metadata.sql @@ -14,23 +14,23 @@ SELECT A.block_timestamp, A.version, A.tx_hash, - A.data :success :: BOOLEAN AS success, + A.success, A.tx_type, - A.data :sender :: STRING AS sender, - A.data :signature :: STRING AS signature, - A.data :payload AS payload, - A.data :payload :function :: STRING AS payload_function, - A.data :changes AS changes, - A.data :events AS events, - A.data :failed_proposer_indices :: STRING AS failed_proposer_indices, - A.data :id :: STRING AS id, - A.data :previous_block_votes_bitvec :: STRING AS previous_block_votes_bitvec, - A.data :proposer :: STRING AS proposer, - A.data :ROUND :: INT AS ROUND, - A.data :vm_status :: STRING AS vm_status, - A.data :state_change_hash :: STRING AS state_change_hash, - A.data :accumulator_root_hash :: STRING AS accumulator_root_hash, - A.data :event_root_hash :: STRING AS event_root_hash, + A.sender, + A.signature, + A.payload, + A.payload_function, + A.changes, + A.events, + A.failed_proposer_indices, + A.id, + A.previous_block_votes_bitvec, + A.proposer, + A.ROUND, + A.vm_status, + A.state_change_hash, + A.accumulator_root_hash, + A.event_root_hash, {{ dbt_utils.generate_surrogate_key( ['tx_hash'] ) }} AS fact_transactions_block_metadata_id, diff --git a/models/gold/core/core__fact_transactions_block_metadata.yml b/models/gold/core/core__fact_transactions_block_metadata.yml index 2d36f91..3c12197 100644 --- a/models/gold/core/core__fact_transactions_block_metadata.yml +++ b/models/gold/core/core__fact_transactions_block_metadata.yml @@ -2,7 +2,6 @@ version: 2 models: - name: core__fact_transactions_block_metadata description: '{{ doc("core__fact_transactions_block_metadata") }}' - columns: - name: block_number description: '{{ doc("block_number") }}' diff --git a/models/gold/core/core__fact_transactions_state_checkpoint.yml b/models/gold/core/core__fact_transactions_state_checkpoint.yml index 05b4d39..ebd5745 100644 --- a/models/gold/core/core__fact_transactions_state_checkpoint.yml +++ b/models/gold/core/core__fact_transactions_state_checkpoint.yml @@ -2,7 +2,6 @@ version: 2 models: - name: core__fact_transactions_state_checkpoint description: '{{ doc("core__fact_transactions_state_checkpoint") }}' - columns: - name: block_number description: '{{ doc("block_number") }}' diff --git a/models/silver/core/silver__blocks.yml b/models/silver/core/silver__blocks.yml index 343c8c7..4fc3d00 100644 --- a/models/silver/core/silver__blocks.yml +++ b/models/silver/core/silver__blocks.yml @@ -1,25 +1,44 @@ version: 2 + models: - name: silver__blocks + config: + where: "modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}" tests: - - sequence_gaps: - column: block_number + - dbt_utils.sequential_values: + column_name: block_number + interval: 1 + config: + severity: error + error_if: ">100" + tags: ['test_recent'] columns: - name: block_number tests: - not_null: - where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + tags: ['test_integrity'] - name: block_hash - - name: block_timestamp_num + tests: + - not_null: + tags: ['test_integrity'] + - unique: + tags: ['test_integrity'] - name: block_timestamp tests: - not_null: - where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + tags: ['test_integrity'] + - name: tx_count_from_transactions_array + tests: + - dbt_utils.expression_is_true: + expression: "tx_count_from_transactions_array >= 0" + - name: tx_count_from_versions + tests: + - dbt_utils.expression_is_true: + expression: "tx_count_from_versions >= 0" + - name: block_timestamp_num - name: first_version - name: last_version - - name: tx_count_from_transactions_array - - name: tx_count_from_versions - name: blocks_id - name: inserted_timestamp - name: modified_timestamp - - name: _invocation_id + - name: _invocation_id \ No newline at end of file diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index e2b6368..cabd4e7 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -112,6 +112,11 @@ transformed AS ( DATA :accumulator_root_hash :: STRING AS accumulator_root_hash, DATA :event_root_hash :: STRING AS event_root_hash, DATA :state_checkpoint_hash :: STRING AS state_checkpoint_hash, + DATA :failed_proposer_indices :: STRING AS failed_proposer_indices, + DATA :id :: STRING AS id, + DATA :previous_block_votes_bitvec :: STRING AS previous_block_votes_bitvec, + DATA :proposer :: STRING AS proposer, + DATA :ROUND :: INT AS ROUND, DATA, file_last_updated FROM @@ -138,6 +143,11 @@ SELECT accumulator_root_hash, event_root_hash, state_checkpoint_hash, + failed_proposer_indices, + id, + previous_block_votes_bitvec, + proposer, + ROUND, DATA, {{ dbt_utils.generate_surrogate_key( ['tx_hash'] diff --git a/models/silver/core/silver__transactions.yml b/models/silver/core/silver__transactions.yml index 1b080a0..ba36983 100644 --- a/models/silver/core/silver__transactions.yml +++ b/models/silver/core/silver__transactions.yml @@ -39,6 +39,11 @@ models: - name: accumulator_root_hash - name: event_root_hash - name: state_checkpoint_hash + - name: failed_proposer_indices + - name: id + - name: previous_block_votes_bitvec + - name: proposer + - name: ROUND - name: data - name: transactions_id - name: inserted_timestamp diff --git a/models/streamline/silver/realtime/streamline__transactions_realtime.sql b/models/streamline/silver/realtime/streamline__transactions_realtime.sql index 41d2dcb..1a7d6f9 100644 --- a/models/streamline/silver/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/silver/realtime/streamline__transactions_realtime.sql @@ -16,17 +16,11 @@ WITH blocks AS ( SELECT A.block_number, tx_count_from_versions AS tx_count, - --tx_count_from_versions -100 AS tx_count, --REMOVE FOR PUBLIC MAINNET - --first_version + 100 version_start --REMOVE FOR PUBLIC MAINNET first_version AS version_start FROM {{ ref('silver__blocks') }} A - WHERE - -- tx_count_from_versions > 100 --REMOVE FOR PUBLIC MAINNET - block_number != 0 --REMOVE FOR PUBLIC MAINNET ), numbers AS ( - -- Recursive CTE to generate numbers. We'll use the maximum txcount value to limit our recursion. SELECT 1 AS n UNION ALL diff --git a/packages.yml b/packages.yml index 4758956..8001507 100644 --- a/packages.yml +++ b/packages.yml @@ -4,7 +4,7 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.29.0 + revision: v1.32.0 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] - package: Snowflake-Labs/dbt_constraints