diff --git a/macros/incremental_utils.sql b/macros/incremental_utils.sql index 6f78f3c..a1a3368 100644 --- a/macros/incremental_utils.sql +++ b/macros/incremental_utils.sql @@ -2,7 +2,7 @@ -- dbt makes it easy to query your target table by using the "{{ this }}" variable. {% if is_incremental() %} -{{ time_col }} > ( +{{ time_col }} >= ( SELECT MAX( {{ time_col }} diff --git a/models/bronze/bronze__transactions.sql b/models/bronze/bronze__transactions.sql index cbe3da1..50c20de 100644 --- a/models/bronze/bronze__transactions.sql +++ b/models/bronze/bronze__transactions.sql @@ -4,7 +4,7 @@ SELECT record_id, - tx_id, + tx_id AS txn_hash, tx_block_index, offset_id, block_id, diff --git a/models/core/actions_events.sql b/models/core/actions_events.sql index 839c396..4b7b796 100644 --- a/models/core/actions_events.sql +++ b/models/core/actions_events.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'], unique_key = 'action_id', - tags = ['actions'] + tags = ['actions'], + enabled = false ) }} WITH txs AS ( diff --git a/models/core/actions_events_addkey.sql b/models/core/actions_events_addkey.sql index b67cba2..e333d60 100644 --- a/models/core/actions_events_addkey.sql +++ b/models/core/actions_events_addkey.sql @@ -2,7 +2,8 @@ materialized = 'incremental', unique_key = 'action_id', cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'], - tags = ['actions_events'] + tags = ['actions_events'], + enabled = false ) }} WITH action_events AS ( diff --git a/models/core/actions_events_function_call.sql b/models/core/actions_events_function_call.sql index 5d95831..a5bd27b 100644 --- a/models/core/actions_events_function_call.sql +++ b/models/core/actions_events_function_call.sql @@ -3,7 +3,8 @@ incremental_strategy = 'delete+insert', unique_key = 'action_id', cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'], - tags = ['near','actions','events','functioncall'] + tags = ['near','actions','events','functioncall'], + enabled = false ) }} WITH action_events AS ( diff --git a/models/core/receipts.sql b/models/core/receipts.sql index 9c63041..2d8fb20 100644 --- a/models/core/receipts.sql +++ b/models/core/receipts.sql @@ -3,7 +3,8 @@ unique_key = 'receipt_object_id', incremental_strategy = 'delete+insert', tags = ['core', 'transactions'], - cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'] + cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'], + enabled = false ) }} WITH txs AS ( diff --git a/models/core/transfers.sql b/models/core/transfers.sql index 464bc42..e717343 100644 --- a/models/core/transfers.sql +++ b/models/core/transfers.sql @@ -2,7 +2,8 @@ materialized = 'incremental', cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'], unique_key = 'action_id', - tags = ['core', 'transfers'] + tags = ['core', 'transfers'], + enabled = false ) }} WITH action_events AS( diff --git a/models/descriptions/ingested_at.md b/models/descriptions/_ingested_at.md similarity index 72% rename from models/descriptions/ingested_at.md rename to models/descriptions/_ingested_at.md index 9d7851b..5fc2dca 100644 --- a/models/descriptions/ingested_at.md +++ b/models/descriptions/_ingested_at.md @@ -1,4 +1,4 @@ -{% docs ingested_at %} +{% docs _ingested_at %} The timestamp for when the block was ingested. diff --git a/models/descriptions/_insterted_timestamp.md b/models/descriptions/_insterted_timestamp.md new file mode 100644 index 0000000..1bb8a05 --- /dev/null +++ b/models/descriptions/_insterted_timestamp.md @@ -0,0 +1,5 @@ +{% docs _inserted_timestamp %} + +The timestamp for when the block was inserted into the data warehouse by the Chainwalker. + +{% enddocs %} diff --git a/models/metrics/metrics_active_wallets.sql b/models/metrics/metrics_active_wallets.sql index 3e28917..d3f1be3 100644 --- a/models/metrics/metrics_active_wallets.sql +++ b/models/metrics/metrics_active_wallets.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', tags = ['metrics'], - cluster_by = ['date'] + cluster_by = ['date'], + enabled = false ) }} WITH txs AS ( diff --git a/models/metrics/metrics_daily_gas.sql b/models/metrics/metrics_daily_gas.sql index 2e1e101..d477796 100644 --- a/models/metrics/metrics_daily_gas.sql +++ b/models/metrics/metrics_daily_gas.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', tags = ['metrics', 'transactions'], - cluster_by = ['date'] + cluster_by = ['date'], + enabled = false ) }} WITH txs AS ( diff --git a/models/metrics/metrics_daily_transactions.sql b/models/metrics/metrics_daily_transactions.sql index a6348a1..9a7edc1 100644 --- a/models/metrics/metrics_daily_transactions.sql +++ b/models/metrics/metrics_daily_transactions.sql @@ -2,7 +2,8 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', tags = ['metrics', 'transactions'], - cluster_by = ['date'] + cluster_by = ['date'], + enabled = false ) }} WITH txs AS ( diff --git a/models/core/blocks.sql b/models/silver/silver__blocks.sql similarity index 86% rename from models/core/blocks.sql rename to models/silver/silver__blocks.sql index 5eb15c7..1dec5c6 100644 --- a/models/core/blocks.sql +++ b/models/silver/silver__blocks.sql @@ -2,8 +2,7 @@ materialized = 'incremental', unique_key = 'block_height', incremental_strategy = 'delete+insert', - tags = ['core'], - cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'] + cluster_by = ['_inserted_timestamp::DATE'] ) }} WITH base_blocks AS ( @@ -11,9 +10,14 @@ WITH base_blocks AS ( SELECT * FROM - {{ ref("stg_blocks") }} + {{ ref('bronze__blocks') }} WHERE - {{ incremental_load_filter("ingested_at") }} + {{ incremental_load_filter('_inserted_timestamp') }} + qualify ROW_NUMBER() over ( + PARTITION BY block_id + ORDER BY + _inserted_timestamp DESC + ) = 1 ), FINAL AS ( SELECT @@ -52,7 +56,8 @@ FINAL AS ( header :total_supply :: FLOAT AS total_supply, header :validator_proposals AS validator_proposals, header :validator_reward :: FLOAT AS validator_reward, - ingested_at + _ingested_at, + _inserted_timestamp FROM base_blocks ) diff --git a/models/core/blocks.yml b/models/silver/silver__blocks.yml similarity index 96% rename from models/core/blocks.yml rename to models/silver/silver__blocks.yml index 934fee4..7cb6116 100644 --- a/models/core/blocks.yml +++ b/models/silver/silver__blocks.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: blocks + - name: silver__blocks description: |- This table records all the blocks of Near blockchain. tests: @@ -50,7 +50,7 @@ models: column_type_list: - NUMBER - FLOAT - + - name: block_author description: "{{ doc('block_author')}}" tests: @@ -161,7 +161,7 @@ models: - ARRAY - VARIANT - OBJECT - + - name: gas_price description: "{{ doc('gas_price')}}" tests: @@ -170,7 +170,7 @@ models: column_type_list: - NUMBER - FLOAT - + - name: last_ds_final_block description: "{{ doc('last_ds_final_block')}}" tests: @@ -197,7 +197,7 @@ models: column_type_list: - NUMBER - FLOAT - + - name: next_bp_hash description: "{{ doc('next_bp_hash')}}" tests: @@ -241,7 +241,7 @@ models: column_type_list: - NUMBER - FLOAT - + - name: prev_state_root description: "{{ doc('prev_state_root')}}" tests: @@ -267,7 +267,7 @@ models: column_type_list: - NUMBER - FLOAT - + - name: signature description: "{{ doc('signature')}}" tests: @@ -285,7 +285,7 @@ models: column_type_list: - NUMBER - FLOAT - + - name: validator_proposals description: "{{ doc('validator_proposals')}}" tests: @@ -305,8 +305,16 @@ models: - NUMBER - FLOAT - - name: ingested_at - description: "{{ doc('ingested_at')}}" + - name: _ingested_at + description: "{{ doc('_ingested_at')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: _inserted_timestamp + description: "{{ doc('_inserted_timestamp')}}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: diff --git a/models/core/transactions.sql b/models/silver/silver__transactions.sql similarity index 77% rename from models/core/transactions.sql rename to models/silver/silver__transactions.sql index 6667ec0..475bc69 100644 --- a/models/core/transactions.sql +++ b/models/silver/silver__transactions.sql @@ -2,16 +2,28 @@ materialized = 'incremental', unique_key = 'txn_hash', incremental_strategy = 'delete+insert', - tags = ['core', 'transactions'], - cluster_by = ['ingested_at::DATE', 'block_timestamp::DATE'] + cluster_by = ['_inserted_timestamp::DATE'] ) }} -WITH transactions AS ( +WITH base_transactions AS ( + SELECT + * + FROM + {{ ref('bronze__transactions') }} + WHERE + {{ incremental_load_filter('_inserted_timestamp') }} + qualify ROW_NUMBER() over ( + PARTITION BY txn_hash + ORDER BY + _inserted_timestamp DESC + ) = 1 +), +transactions AS ( SELECT block_id AS block_height, tx :outcome :block_hash :: STRING AS block_hash, - tx_id AS txn_hash, + txn_hash, block_timestamp, tx :nonce :: NUMBER AS nonce, tx :signature :: STRING AS signature, @@ -26,11 +38,10 @@ WITH transactions AS ( tx :actions, 0 ) :FunctionCall :gas :: NUMBER AS attached_gas, - ingested_at + _ingested_at, + _inserted_timestamp FROM - {{ ref('stg_txs') }} - WHERE - {{ incremental_load_filter("ingested_at") }} + base_transactions ), receipts AS ( SELECT @@ -68,7 +79,8 @@ FINAL AS ( t.attached_gas, gas_used ) AS attached_gas, - t.ingested_at + t._ingested_at, + t._inserted_timestamp FROM transactions AS t JOIN receipts AS r diff --git a/models/core/transactions.yml b/models/silver/silver__transactions.yml similarity index 92% rename from models/core/transactions.yml rename to models/silver/silver__transactions.yml index 83de8ba..b1ce6b4 100644 --- a/models/core/transactions.yml +++ b/models/silver/silver__transactions.yml @@ -1,8 +1,7 @@ - version: 2 models: - - name: transactions + - name: silver__transactions description: "Near transactions." tests: - dbt_utils.unique_combination_of_columns: @@ -143,8 +142,16 @@ models: - NUMBER - FLOAT - - name: ingested_at - description: "{{ doc('ingested_at')}}" + - name: _ingested_at + description: "{{ doc('_ingested_at')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: _inserted_timestamp + description: "{{ doc('_inserted_timestamp')}}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: diff --git a/models/staging/stg_blocks.sql b/models/staging/stg_blocks.sql index 23c1178..3551fed 100644 --- a/models/staging/stg_blocks.sql +++ b/models/staging/stg_blocks.sql @@ -3,7 +3,8 @@ unique_key = 'block_id', incremental_strategy = 'delete+insert', tags = ['core'], - cluster_by = ['block_timestamp'] + cluster_by = ['block_timestamp'], + enabled = false ) }} WITH FINAL AS ( diff --git a/models/staging/stg_txs.sql b/models/staging/stg_txs.sql index 43fb88e..06ccbbd 100644 --- a/models/staging/stg_txs.sql +++ b/models/staging/stg_txs.sql @@ -3,7 +3,8 @@ unique_key = 'tx_id', incremental_strategy = 'delete+insert', tags = ['core', 'transactions'], - cluster_by = ['block_timestamp'] + cluster_by = ['block_timestamp'], + enabled = false ) }} WITH FINAL AS (