diff --git a/.dbt/profiles.yml b/.dbt/profiles.yml index 67e7b23..69d3a83 100644 --- a/.dbt/profiles.yml +++ b/.dbt/profiles.yml @@ -10,7 +10,7 @@ near: region: "{{ env_var('REGION') }}" database: "{{ env_var('DATABASE_DEV') }}" warehouse: "{{ env_var('WAREHOUSE_DEV') }}" - schema: DEV + schema: SILVER threads: 4 client_session_keep_alive: False query_tag: gh_actions @@ -23,7 +23,7 @@ near: region: "{{ env_var('REGION') }}" database: "{{ env_var('DATABASE_PROD') }}" warehouse: "{{ env_var('WAREHOUSE_PROD') }}" - schema: PROD + schema: SILVER threads: 4 client_session_keep_alive: False query_tag: gh_actions diff --git a/README.md b/README.md index d71d888..93981aa 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,10 @@ SF_ACCOUNT=zsniary-metricsdao SF_USERNAME= SF_PASSWORD= SF_REGION=us-east-1 -SF_DATABASE=NEAR +SF_DATABASE=NEAR_DEV SF_WAREHOUSE=DEFAULT SF_ROLE=PUBLIC -SF_SCHEMA=DEV +SF_SCHEMA=SILVER ``` 3. New to DBT? It's pretty dope. Read up on it [here](https://www.getdbt.com/docs/) diff --git a/macros/tx_gaps.sql b/macros/tx_gaps.sql index 00e1a4f..448380d 100644 --- a/macros/tx_gaps.sql +++ b/macros/tx_gaps.sql @@ -9,7 +9,7 @@ WITH block_base AS ( {{ column_block }}, {{ column_tx_count }} FROM - {{ ref('blocks') }} + {{ ref('silver__blocks') }} ), model_name AS ( SELECT diff --git a/models/core/core__fact_actions_events.sql b/models/core/core__fact_actions_events.sql index 0cac010..86e06af 100644 --- a/models/core/core__fact_actions_events.sql +++ b/models/core/core__fact_actions_events.sql @@ -12,6 +12,7 @@ WITH actions_events AS ( SELECT action_id, tx_hash, + block_id, block_timestamp, action_index, action_name, diff --git a/models/core/core__fact_actions_events.yml b/models/core/core__fact_actions_events.yml index a792fe6..7a74aa0 100644 --- a/models/core/core__fact_actions_events.yml +++ b/models/core/core__fact_actions_events.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: diff --git a/models/core/core__fact_actions_events_addkey.sql b/models/core/core__fact_actions_events_addkey.sql index 08a7ba0..d578459 100644 --- a/models/core/core__fact_actions_events_addkey.sql +++ b/models/core/core__fact_actions_events_addkey.sql @@ -12,6 +12,7 @@ WITH actions_events_addkey AS ( SELECT action_id, tx_hash, + block_id, block_timestamp, nonce, public_key, diff --git a/models/core/core__fact_actions_events_addkey.yml b/models/core/core__fact_actions_events_addkey.yml index 418700b..f69f0d4 100644 --- a/models/core/core__fact_actions_events_addkey.yml +++ b/models/core/core__fact_actions_events_addkey.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: diff --git a/models/core/core__fact_actions_events_function_call.sql b/models/core/core__fact_actions_events_function_call.sql index e8d3b76..c0f128d 100644 --- a/models/core/core__fact_actions_events_function_call.sql +++ b/models/core/core__fact_actions_events_function_call.sql @@ -12,6 +12,7 @@ WITH actions_events_function_call AS ( SELECT action_id, tx_hash, + block_id, block_timestamp, action_name, method_name, diff --git a/models/core/core__fact_actions_events_function_call.yml b/models/core/core__fact_actions_events_function_call.yml index 9cfbe81..767f431 100644 --- a/models/core/core__fact_actions_events_function_call.yml +++ b/models/core/core__fact_actions_events_function_call.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: diff --git a/models/core/core__fact_blocks.sql b/models/core/core__fact_blocks.sql index 84937e8..0f38869 100644 --- a/models/core/core__fact_blocks.sql +++ b/models/core/core__fact_blocks.sql @@ -10,7 +10,7 @@ WITH blocks AS ( {{ ref('silver__blocks') }} ) SELECT - block_height, + block_id, block_timestamp, block_hash, tx_count, diff --git a/models/core/core__fact_blocks.yml b/models/core/core__fact_blocks.yml index a526799..028d919 100644 --- a/models/core/core__fact_blocks.yml +++ b/models/core/core__fact_blocks.yml @@ -7,14 +7,14 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - block_height + - block_id - sequence_gaps: - column_name: block_height + column_name: block_id where: BLOCK_TIMESTAMP < CURRENT_DATE columns: - - name: block_height - description: "{{ doc('block_height')}}" + - name: block_id + description: "{{ doc('block_id')}}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: diff --git a/models/core/core__fact_receipts.sql b/models/core/core__fact_receipts.sql index 2fce5ed..a75f985 100644 --- a/models/core/core__fact_receipts.sql +++ b/models/core/core__fact_receipts.sql @@ -11,7 +11,7 @@ WITH receipts AS ( ) SELECT block_timestamp, - block_hash, + block_id, tx_hash, receipt_object_id, receipt_outcome_id, diff --git a/models/core/core__fact_receipts.yml b/models/core/core__fact_receipts.yml index ac064e7..7026149 100644 --- a/models/core/core__fact_receipts.yml +++ b/models/core/core__fact_receipts.yml @@ -17,14 +17,14 @@ models: column_type_list: - TIMESTAMP_NTZ - - name: block_hash - description: "{{ doc('block_hash')}}" + - name: block_id + description: "{{ doc('block_id')}}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - - STRING - - VARCHAR + - NUMBER + - FLOAT - name: tx_hash description: "{{ doc('tx_hash')}}" diff --git a/models/core/core__fact_transactions.sql b/models/core/core__fact_transactions.sql index 4dad1f3..20e6661 100644 --- a/models/core/core__fact_transactions.sql +++ b/models/core/core__fact_transactions.sql @@ -11,7 +11,7 @@ WITH transactions AS ( ) SELECT tx_hash, - block_height, + block_id, block_hash, block_timestamp, nonce, diff --git a/models/core/core__fact_transactions.yml b/models/core/core__fact_transactions.yml index 6c10406..9039c25 100644 --- a/models/core/core__fact_transactions.yml +++ b/models/core/core__fact_transactions.yml @@ -9,8 +9,8 @@ models: - tx_hash columns: - - name: block_height - description: "{{ doc('block_height')}}" + - name: block_id + description: "{{ doc('block_id')}}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: @@ -37,7 +37,7 @@ models: - STRING - VARCHAR - tx_gaps: - column_block: block_height + column_block: block_id column_tx_count: tx_count where: BLOCK_TIMESTAMP < CURRENT_DATE diff --git a/models/core/core__fact_transfers.sql b/models/core/core__fact_transfers.sql index 9c03404..b6e429f 100644 --- a/models/core/core__fact_transfers.sql +++ b/models/core/core__fact_transfers.sql @@ -12,6 +12,7 @@ WITH transfers AS ( SELECT tx_hash, action_id, + block_id, block_timestamp, tx_signer, tx_receiver, diff --git a/models/core/core__fact_transfers.yml b/models/core/core__fact_transfers.yml index 7d87151..83e98ba 100644 --- a/models/core/core__fact_transfers.yml +++ b/models/core/core__fact_transfers.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: diff --git a/models/legacy/legacy__blocks.sql b/models/legacy/legacy__blocks.sql index 8263cdc..6d90d48 100644 --- a/models/legacy/legacy__blocks.sql +++ b/models/legacy/legacy__blocks.sql @@ -10,7 +10,7 @@ WITH blocks AS ( {{ ref('silver__blocks') }} ) SELECT - block_height, + block_id AS block_height, block_timestamp, block_hash, tx_count, diff --git a/models/legacy/legacy__transactions.sql b/models/legacy/legacy__transactions.sql index cbed675..68e49bd 100644 --- a/models/legacy/legacy__transactions.sql +++ b/models/legacy/legacy__transactions.sql @@ -11,7 +11,7 @@ WITH transactions AS ( ) SELECT tx_hash AS txn_hash, - block_height, + block_id AS block_height, block_hash, block_timestamp, nonce, diff --git a/models/silver/silver__actions_events.sql b/models/silver/silver__actions_events.sql index a844b92..f711899 100644 --- a/models/silver/silver__actions_events.sql +++ b/models/silver/silver__actions_events.sql @@ -17,6 +17,7 @@ WITH txs AS ( actions AS ( SELECT tx_hash, + block_id, block_timestamp, INDEX AS action_index, CASE @@ -43,6 +44,7 @@ FINAL AS ( action_index ) AS action_id, tx_hash, + block_id, block_timestamp, action_index, action_name, diff --git a/models/silver/silver__actions_events.yml b/models/silver/silver__actions_events.yml index f281c03..206ddf7 100644 --- a/models/silver/silver__actions_events.yml +++ b/models/silver/silver__actions_events.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: diff --git a/models/silver/silver__actions_events_addkey.sql b/models/silver/silver__actions_events_addkey.sql index 28c4b8c..7f6286c 100644 --- a/models/silver/silver__actions_events_addkey.sql +++ b/models/silver/silver__actions_events_addkey.sql @@ -19,6 +19,7 @@ addkey_events AS ( SELECT action_id, tx_hash, + block_id, block_timestamp, action_data :access_key :nonce :: NUMBER AS nonce, action_data :public_key :: STRING AS public_key, diff --git a/models/silver/silver__actions_events_addkey.yml b/models/silver/silver__actions_events_addkey.yml index bf906ee..17cfe85 100644 --- a/models/silver/silver__actions_events_addkey.yml +++ b/models/silver/silver__actions_events_addkey.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: diff --git a/models/silver/silver__actions_events_function_call.sql b/models/silver/silver__actions_events_function_call.sql index b52fdb3..eb53cea 100644 --- a/models/silver/silver__actions_events_function_call.sql +++ b/models/silver/silver__actions_events_function_call.sql @@ -29,6 +29,7 @@ decoding AS ( SELECT action_id, tx_hash, + block_id, block_timestamp, action_name, method_name, diff --git a/models/silver/silver__actions_events_function_call.yml b/models/silver/silver__actions_events_function_call.yml index 1f0aa76..ed04966 100644 --- a/models/silver/silver__actions_events_function_call.yml +++ b/models/silver/silver__actions_events_function_call.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index 1dec5c6..30bb0f7 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = 'block_height', + unique_key = 'block_id', incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'] ) }} @@ -21,7 +21,7 @@ WITH base_blocks AS ( ), FINAL AS ( SELECT - header :height :: INTEGER AS block_height, + block_id, div0( header :timestamp :: INTEGER, 1000000000 diff --git a/models/silver/silver__blocks.yml b/models/silver/silver__blocks.yml index 7cb6116..04bccc7 100644 --- a/models/silver/silver__blocks.yml +++ b/models/silver/silver__blocks.yml @@ -7,14 +7,14 @@ models: tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - block_height + - block_id - sequence_gaps: - column_name: block_height + column_name: block_id where: BLOCK_TIMESTAMP < CURRENT_DATE columns: - - name: block_height - description: "{{ doc('block_height')}}" + - name: block_id + description: "{{ doc('block_id')}}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: diff --git a/models/silver/silver__receipts.sql b/models/silver/silver__receipts.sql index 2183ac4..c4bfa0a 100644 --- a/models/silver/silver__receipts.sql +++ b/models/silver/silver__receipts.sql @@ -18,6 +18,7 @@ receipts AS ( SELECT block_timestamp, block_hash, + block_id, tx_hash, VALUE :id :: STRING AS receipt_object_id, VALUE :outcome :receipt_ids AS receipt_outcome_id, diff --git a/models/silver/silver__receipts.yml b/models/silver/silver__receipts.yml index a2e5815..d7e6a9d 100644 --- a/models/silver/silver__receipts.yml +++ b/models/silver/silver__receipts.yml @@ -35,6 +35,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: receipt_object_id description: "{{ doc('receipt_object_id')}}" tests: diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index 951514d..123a6e9 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -21,7 +21,7 @@ WITH base_transactions AS ( ), transactions AS ( SELECT - block_id AS block_height, + block_id AS block_id, tx :outcome :block_hash :: STRING AS block_hash, tx_hash, block_timestamp, @@ -60,7 +60,7 @@ receipts AS ( ), FINAL AS ( SELECT - t.block_height, + t.block_id, t.block_hash, t.tx_hash, t.block_timestamp, diff --git a/models/silver/silver__transactions.yml b/models/silver/silver__transactions.yml index a6b6c22..7742af5 100644 --- a/models/silver/silver__transactions.yml +++ b/models/silver/silver__transactions.yml @@ -9,8 +9,8 @@ models: - tx_hash columns: - - name: block_height - description: "{{ doc('block_height')}}" + - name: block_id + description: "{{ doc('block_id')}}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: @@ -37,7 +37,7 @@ models: - STRING - VARCHAR - tx_gaps: - column_block: block_height + column_block: block_id column_tx_count: tx_count where: BLOCK_TIMESTAMP < CURRENT_DATE diff --git a/models/silver/silver__transfers.sql b/models/silver/silver__transfers.sql index 0908c76..9468f49 100644 --- a/models/silver/silver__transfers.sql +++ b/models/silver/silver__transfers.sql @@ -20,6 +20,7 @@ WITH action_events AS( actions AS ( SELECT t.tx_hash, + t.block_id, A.action_id, t.block_timestamp, t.tx_receiver, @@ -44,6 +45,7 @@ actions AS ( FINAL AS ( SELECT tx_hash, + block_id, action_id, block_timestamp, tx_signer, diff --git a/models/silver/silver__transfers.yml b/models/silver/silver__transfers.yml index aa456b9..91d10e9 100644 --- a/models/silver/silver__transfers.yml +++ b/models/silver/silver__transfers.yml @@ -29,6 +29,15 @@ models: - STRING - VARCHAR + - name: block_id + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: block_timestamp description: "{{ doc('block_timestamp')}}" tests: