From 7e7d0e6114b6875f9673b16e83f5e220f0380461 Mon Sep 17 00:00:00 2001 From: Eevui Date: Sun, 18 Sep 2022 20:53:32 +0800 Subject: [PATCH 1/6] staking_actions: initial track --- models/silver/silver__staking_actions.sql | 161 ++++++++++++++++++++++ models/silver/silver__staking_actions.yml | 72 ++++++++++ 2 files changed, 233 insertions(+) create mode 100644 models/silver/silver__staking_actions.sql create mode 100644 models/silver/silver__staking_actions.yml diff --git a/models/silver/silver__staking_actions.sql b/models/silver/silver__staking_actions.sql new file mode 100644 index 0000000..26a7731 --- /dev/null +++ b/models/silver/silver__staking_actions.sql @@ -0,0 +1,161 @@ +{{ config( + materialized = 'incremental', + incremental = 'merge', + cluster_by = ['block_timestamp'], + unique_key = 'tx_hash' +) }} + +with actions_events_function_call as ( + select + tx_hash, + method_name, + _inserted_timestamp + from {{ ref('silver__actions_events_function_call') }} + where {{ incremental_load_filter('_inserted_timestamp') }} + and method_name in ( + 'deposit_and_stake', + 'stake', + 'unstake', + 'unstake_all' + ) +), + +base_txs as ( + select + * + from {{ ref('silver__transactions') }} + where {{ incremental_load_filter('_inserted_timestamp') }} +), + +txs as ( + select + * + from base_txs + where (tx_receiver like '%.pool.near' or tx_receiver like '%.poolv1.near') +), + +pool_txs as ( + select + txs.tx_hash as tx_hash, + block_timestamp, + tx_receiver, + tx_signer, + tx, + method_name, + txs._inserted_timestamp as _inserted_timestamp + from txs + inner join actions_events_function_call + on txs.tx_hash = actions_events_function_call.tx_hash +), + +deposit_and_stake_txs as ( + select + tx_hash, + block_timestamp, + tx_receiver as pool_address, + tx_signer, + coalesce( + regexp_substr(tx:receipt[0]:outcome:logs[1], 'staking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[2], 'staking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[3], 'staking (\\d+)', 1, 1, 'e')::number + ) as stake_amount, + 'Stake' as action, + _inserted_timestamp + from pool_txs + where method_name = 'deposit_and_stake' + and tx:receipt[0]:outcome:status:SuccessValue is not null +), + +stake_txs as ( + select + tx_hash, + block_timestamp, + tx_receiver as pool_address, + tx_signer, + coalesce( + regexp_substr(tx:receipt[0]:outcome:logs[1], 'staking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[2], 'staking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[3], 'staking (\\d+)', 1, 1, 'e')::number + ) as stake_amount, + 'Stake' as action, + _inserted_timestamp + from pool_txs + where method_name = 'stake' + and tx:receipt[0]:outcome:status:SuccessValue is not null +), + +stake_all_txs as ( + select + tx_hash, + block_timestamp, + tx_receiver as pool_address, + tx_signer, + coalesce( + regexp_substr(tx:receipt[0]:outcome:logs[0], 'staking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[1], 'staking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[2], 'staking (\\d+)', 1, 1, 'e')::number + ) as stake_amount, + 'Stake' as action, + _inserted_timestamp + from pool_txs + where method_name = 'stake_all' + and tx:receipt[0]:outcome:status:SuccessValue is not null +), + +unstake_txs as ( + select + tx_hash, + block_timestamp, + tx_receiver as pool_address, + tx_signer, + coalesce( + regexp_substr(tx:receipt[0]:outcome:logs[0], 'unstaking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[1], 'unstaking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[2], 'unstaking (\\d+)', 1, 1, 'e')::number + ) as stake_amount, + 'Unstake' as action, + _inserted_timestamp + from pool_txs + where method_name = 'unstake' + and tx:receipt[0]:outcome:status:SuccessValue is not null +), + +unstake_all_txs as ( + select + tx_hash, + block_timestamp, + tx_receiver as pool_address, + tx_signer, + coalesce( + regexp_substr(tx:receipt[0]:outcome:logs[0], 'unstaking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[1], 'unstaking (\\d+)', 1, 1, 'e')::number, + regexp_substr(tx:receipt[0]:outcome:logs[2], 'unstaking (\\d+)', 1, 1, 'e')::number + ) as stake_amount, + 'Unstake' as action, + _inserted_timestamp + from pool_txs + where method_name = 'unstake_all' + and tx:receipt[0]:outcome:status:SuccessValue is not null +), + +final as ( + select + * + from deposit_and_stake_txs + union + select + * + from stake_all_txs + union + select + * + from unstake_txs + union + select + * + from unstake_all_txs +) + +select +* +from final diff --git a/models/silver/silver__staking_actions.yml b/models/silver/silver__staking_actions.yml new file mode 100644 index 0000000..770fb82 --- /dev/null +++ b/models/silver/silver__staking_actions.yml @@ -0,0 +1,72 @@ +version: 2 + +models: + - name: silver__staking_actions + description: |- + This table records all the staking and delegation transactions occurring in NEAR. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_hash + + columns: + - name: tx_hash + description: "{{ doc('tx_hash') }}" + tests: + - unique + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: pool_address + description: "{{ doc('staking_pool_address') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: tx_signer + description: "{{ doc('tx_signer') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: stake_amount + description: "{{ doc('staking_stake_amount') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: action + description: "{{ doc('staking_action') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: _inserted_timestamp + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ From 7ed4dca13e679643ca0581391a4409c5b681df68 Mon Sep 17 00:00:00 2001 From: Eevui Date: Fri, 30 Sep 2022 21:06:16 +0800 Subject: [PATCH 2/6] staking_actions: add additional docs --- models/descriptions/staking_action.md | 5 +++++ models/descriptions/staking_pool_address.md | 2 +- models/descriptions/staking_stake_amount.md | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 models/descriptions/staking_action.md create mode 100644 models/descriptions/staking_stake_amount.md diff --git a/models/descriptions/staking_action.md b/models/descriptions/staking_action.md new file mode 100644 index 0000000..fb4512b --- /dev/null +++ b/models/descriptions/staking_action.md @@ -0,0 +1,5 @@ +{% docs staking_action %} + +The staking action performed in this transaction. Can be `"Stake"` or `"Unstake"`. + +{% enddocs %} diff --git a/models/descriptions/staking_pool_address.md b/models/descriptions/staking_pool_address.md index d1fcb21..938ced4 100644 --- a/models/descriptions/staking_pool_address.md +++ b/models/descriptions/staking_pool_address.md @@ -1,5 +1,5 @@ {% docs staking_pool_address %} -The address of the staking pool. +The pool address delegated into. {% enddocs %} diff --git a/models/descriptions/staking_stake_amount.md b/models/descriptions/staking_stake_amount.md new file mode 100644 index 0000000..c2b3238 --- /dev/null +++ b/models/descriptions/staking_stake_amount.md @@ -0,0 +1,5 @@ +{% docs staking_stake_amount %} + +The amount staked or delegated toward securing the NEAR network. + +{% enddocs %} From 740aac6099763d9572668db0a52395a3bb000484 Mon Sep 17 00:00:00 2001 From: Eevui Date: Fri, 30 Sep 2022 21:06:43 +0800 Subject: [PATCH 3/6] staking_actions: add note to table description --- models/silver/silver__staking_actions.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/silver/silver__staking_actions.yml b/models/silver/silver__staking_actions.yml index 770fb82..8511053 100644 --- a/models/silver/silver__staking_actions.yml +++ b/models/silver/silver__staking_actions.yml @@ -4,6 +4,9 @@ models: - name: silver__staking_actions description: |- This table records all the staking and delegation transactions occurring in NEAR. + + Be aware, however, not all transactions here necessarily actually contribute to securing the NEAR network. + Each epoch can have a different set of validators. tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: From f48a821225587648b0a1d48c4f19e25bbf8151c4 Mon Sep 17 00:00:00 2001 From: Eevui Date: Wed, 12 Oct 2022 22:51:08 +0800 Subject: [PATCH 4/6] staking-actions: Add core table --- models/core/core__dim_staking_actions.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 models/core/core__dim_staking_actions.sql diff --git a/models/core/core__dim_staking_actions.sql b/models/core/core__dim_staking_actions.sql new file mode 100644 index 0000000..469f3b0 --- /dev/null +++ b/models/core/core__dim_staking_actions.sql @@ -0,0 +1,12 @@ +{{ config( + materialized = 'view', + secure = true +) }} + +with staking_actions as ( + select + * + from {{ ref('silver__staking_actions') }} +) + +select * from staking_actions From f150df08b122b6717d12b86932519d6767067db0 Mon Sep 17 00:00:00 2001 From: Eevui Date: Wed, 12 Oct 2022 23:00:40 +0800 Subject: [PATCH 5/6] staking-actions: add docs for core table --- models/core/core__dim_staking_actions.yml | 73 +++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 models/core/core__dim_staking_actions.yml diff --git a/models/core/core__dim_staking_actions.yml b/models/core/core__dim_staking_actions.yml new file mode 100644 index 0000000..de4202d --- /dev/null +++ b/models/core/core__dim_staking_actions.yml @@ -0,0 +1,73 @@ +version: 2 + +models: + - name: core__dim_staking_actions + description: |- + This table records all the staking and delegation transactions occurring in NEAR. + + Be aware, however, not all transactions here necessarily actually contribute to securing the NEAR network. + Each epoch can have a different set of validators. + + columns: + - name: tx_hash + description: "{{ doc('tx_hash') }}" + tests: + - unique + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: pool_address + description: "{{ doc('staking_pool_address') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: tx_signer + description: "{{ doc('tx_signer') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: stake_amount + description: "{{ doc('staking_stake_amount') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: action + description: "{{ doc('staking_action') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['Stake', 'Unstake'] + + - name: _inserted_timestamp + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ From 7b9496263c48b180705f468fb0bc558735a58126 Mon Sep 17 00:00:00 2001 From: Eevui Date: Thu, 13 Oct 2022 00:21:09 +0800 Subject: [PATCH 6/6] staking-actions: Fix null amounts --- models/silver/silver__staking_actions.sql | 30 ++++------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/models/silver/silver__staking_actions.sql b/models/silver/silver__staking_actions.sql index 26a7731..64606cb 100644 --- a/models/silver/silver__staking_actions.sql +++ b/models/silver/silver__staking_actions.sql @@ -54,11 +54,7 @@ deposit_and_stake_txs as ( block_timestamp, tx_receiver as pool_address, tx_signer, - coalesce( - regexp_substr(tx:receipt[0]:outcome:logs[1], 'staking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[2], 'staking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[3], 'staking (\\d+)', 1, 1, 'e')::number - ) as stake_amount, + regexp_substr(array_to_string(tx:receipt[0]:outcome:logs, ','), 'staking (\\d+)', 1, 1, 'e')::number as stake_amount, 'Stake' as action, _inserted_timestamp from pool_txs @@ -72,11 +68,7 @@ stake_txs as ( block_timestamp, tx_receiver as pool_address, tx_signer, - coalesce( - regexp_substr(tx:receipt[0]:outcome:logs[1], 'staking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[2], 'staking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[3], 'staking (\\d+)', 1, 1, 'e')::number - ) as stake_amount, + regexp_substr(array_to_string(tx:receipt[0]:outcome:logs, ','), 'staking (\\d+)', 1, 1, 'e')::number as stake_amount, 'Stake' as action, _inserted_timestamp from pool_txs @@ -90,11 +82,7 @@ stake_all_txs as ( block_timestamp, tx_receiver as pool_address, tx_signer, - coalesce( - regexp_substr(tx:receipt[0]:outcome:logs[0], 'staking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[1], 'staking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[2], 'staking (\\d+)', 1, 1, 'e')::number - ) as stake_amount, + regexp_substr(array_to_string(tx:receipt[0]:outcome:logs, ','), 'staking (\\d+)', 1, 1, 'e')::number as stake_amount, 'Stake' as action, _inserted_timestamp from pool_txs @@ -108,11 +96,7 @@ unstake_txs as ( block_timestamp, tx_receiver as pool_address, tx_signer, - coalesce( - regexp_substr(tx:receipt[0]:outcome:logs[0], 'unstaking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[1], 'unstaking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[2], 'unstaking (\\d+)', 1, 1, 'e')::number - ) as stake_amount, + regexp_substr(array_to_string(tx:receipt[0]:outcome:logs, ','), 'unstaking (\\d+)', 1, 1, 'e')::number as stake_amount, 'Unstake' as action, _inserted_timestamp from pool_txs @@ -126,11 +110,7 @@ unstake_all_txs as ( block_timestamp, tx_receiver as pool_address, tx_signer, - coalesce( - regexp_substr(tx:receipt[0]:outcome:logs[0], 'unstaking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[1], 'unstaking (\\d+)', 1, 1, 'e')::number, - regexp_substr(tx:receipt[0]:outcome:logs[2], 'unstaking (\\d+)', 1, 1, 'e')::number - ) as stake_amount, + regexp_substr(array_to_string(tx:receipt[0]:outcome:logs, ','), 'unstaking (\\d+)', 1, 1, 'e')::number as stake_amount, 'Unstake' as action, _inserted_timestamp from pool_txs