From 812678f631f72413baf488baa1fb5f392fed2fd8 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Thu, 23 Mar 2023 15:05:20 -0600 Subject: [PATCH 1/7] staking table pass 1, draft --- .../curated/silver__staking_actions_v2.sql | 58 +++++++++++++ .../curated/silver__staking_comp_temp.sql | 33 +++++++ .../silver/curated/silver__staking_notes.md | 85 +++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 models/silver/curated/silver__staking_actions_v2.sql create mode 100644 models/silver/curated/silver__staking_comp_temp.sql create mode 100644 models/silver/curated/silver__staking_notes.md diff --git a/models/silver/curated/silver__staking_actions_v2.sql b/models/silver/curated/silver__staking_actions_v2.sql new file mode 100644 index 0000000..a073f46 --- /dev/null +++ b/models/silver/curated/silver__staking_actions_v2.sql @@ -0,0 +1,58 @@ +{{config(materialized='table', + unique_key='tx_hash', + incremental_strategy='delete+insert', + tags=['curated'], + cluster_by=['_partition_by_block_number', 'block_timestamp::date'] + ) +}} +with +stake_actions as ( +select * from near.silver.actions_events_s3 +where lower(action_name) = 'stake' +), +staking as ( +select + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id as staking_pool, + signer_id, + status_value, + logs, + split(logs[0], ' ') as log_parts, + split(log_parts[0]::string, '@')[1]::string as log_signer_id, + log_parts[1]::string as log_action, + split(log_parts[2]::string,'.')[0]::number as log_amount, + log_amount / pow(10, 24) as log_amount_near, + length(split(log_parts[2]::string,'.')[0]::string) as _log_amount_length, + signer_id = log_signer_id as _log_signer_id_match, + _load_timestamp, + _partition_by_block_number +from near.silver.streamline_receipts_final + where true + and tx_hash in (select distinct tx_hash from stake_actions) + and receiver_id != signer_id + and receiver_id ilike '%.pool%.near' + and receiver_id not in ('usn-unofficial.poolowner.near', 'usn-unofficial.pool.near') + and array_size(logs) > 0 + -- exclude epoch rewards, todo for separate model + and logs[0] not like 'Epoch%' + and logs[0] not ilike '%Record%reward from farm%' + -- explicit: + and log_action in ('staking', 'unstaking', 'deposited') + +) +select * from staking +-- where block_timestamp::date >= current_date - interval '7 days' +-- limit 500; + -- where tx_hash in ( + -- 'GeiLBPrvF3MAbcPA74Nm1qNdXbsVhxYXhjdbDWGkyxwC', -- forg stake + -- 'C1uy4w3wqdBictXy4Jyiqj2yNZGEHzQxiEdgR79uy3wB', -- forg unstake + -- '2Zqi2S7LDAUby3kpM2bYSLojX6TtTSLE7x5zTnSD5KKj', -- mdao stake + -- 'H2ozLWxhk6UbVAnTgzKFSKsuooi7kYsruJgfx2A1HEhy', -- random + -- '5P7rw88ZBLzLsMZgfV5VsEXeLiXQ4KefVvq1puyBQJtU', -- arca unstake 1 + -- 'ABcv2Aozny6Uvi5Lb8SC6YuVBbpY1uUTSS9fAQ2Phndy', -- arca stake 1 + -- '3smwZoSeWD5bzSScSnB3yRZJqVsjkuAZu33v78aqhUtj' -- dnd unstake 25 + -- ) + diff --git a/models/silver/curated/silver__staking_comp_temp.sql b/models/silver/curated/silver__staking_comp_temp.sql new file mode 100644 index 0000000..acac2c3 --- /dev/null +++ b/models/silver/curated/silver__staking_comp_temp.sql @@ -0,0 +1,33 @@ + +with staked_v1 as ( +select + block_timestamp::date as date, + action, + count(distinct tx_hash) as txs, + sum(stake_amount/pow(10,24)) as amount +from near.core.dim_staking_actions +group by 1, 2 +), +staked_v2 as ( +select + block_timestamp::date as date, + iff(log_action in ('deposited', 'staking'), 'Stake', 'Unstake') as action, + count(distinct tx_hash) as txs, + sum(log_amount_near) as amount +from near_dev.silver.staking_actions_v2 +group by 1, 2 +) +select + v1.date as date_v1, + v1.action as action_v1, + v1.txs as txs_v1, + v1.amount as amount_v1, + v2.date as date_v2, + v2.action as action_v2, + v2.txs as txs_v2, + v2.amount as amount_v2, + coalesce(amount_v1,0) = coalesce(amount_v2,0) as amount_match +from staked_v1 v1 +full join staked_v2 v2 +on v1.date = v2.date and v1.action = v2.action +order by 1,2; \ No newline at end of file diff --git a/models/silver/curated/silver__staking_notes.md b/models/silver/curated/silver__staking_notes.md new file mode 100644 index 0000000..f58cbe9 --- /dev/null +++ b/models/silver/curated/silver__staking_notes.md @@ -0,0 +1,85 @@ +# staking fix + +## methods +### staking actions +*current* + - deposit_and_stake + - stake + - unstake + - unstake_all + + *investigate* + - on_stake_action + - has the most actions at 626k while deposit and stake is 2 at 400k. Could be the missing events + - args are always empty, but this event could possible be used to find other staking events + - liquid_unstake + - on_staking_pool_withdraw + - on_staking_pool_unstake + - withdraw_all_from_staking_pool + - on_get_account_unstaked_balance_to_withdraw_by_owner + - distribute_staking + - on_staking_pool_deposit_and_stake + +### staking pools +*current* + - create_staking_pool + - update_reward_fee_fraction + + *investigate* + - set_staking_pool(s) + +*interesting* + - refresh_staking_pool_balance + - proxy_get_pool_info_callback + - get_account_unstaked_balance + - get_pool_info_callback + +*unlikely, but lots of actions* + - get_stable_pool + - get_pool_shares + - balance_stable_pool + - get_pool + +# solution +honing in on a solution by + - looking for a `Stake` action in the events + - receipts where a recipient is a staking pool + - logs exist + +## potential issues + +### log_action not in ('unstaking', 'deposited') + - + + +### where not _log_signer_id_match +There are instances where the receipt signer does not match the address named in the log. 36.6k to be exact. +In once instance - the staker is `linear-protocol.near` while the signer is `operator.linear-protocol.near`. Others it is not so clear what the relationship is. + - some (unk how many) might be addresses that are not yet defined `.near` addresses + - seeing some other sub-addresses like `storage.herewallet.near` vs `owner.herewallet.near` +- i don't need to filter on this test so it's fine + +### epoch rewards +There are also Epoch rewards in this pattern, which will make a good table on their own. +Logs are something like this: +> Epoch N: Contract received total rewards of X tokens. etc... +Example tx: '6UfgPwpwSxEb9WPvVh9jTHceTSK8U3wDLc7V8zQ6pGCu' + - looks like this is what the discord user looked at to try and calculate total staked amount (close, was not exact) + - query in snowflake/near/staking + + ### usn staking? + usn-unofficial.poolowner.near + - https://nearblocks.io/txns/5Zy7Vi8jCxiY6ErP7VvUUADM25yD3QawobGbnfaDxSTT#execution + - https://nearblocks.io/txns/HbNHechdxs24HAf4Gk2P5ogq2gVnkE7S7hCmtynmRzid + - exclude this pool as i dont think it is near staking + - TODO - maybe name the table NEAR staking? or something like that + +### aurora pool +aurora.pool.near +Seem to follow a pattern like +> Record 0 TOKEN reward from farm #0" + - is there an instance where the reward is not 0? + - TODO - check where receiver_id is `aurora.pool.near` + - seeing this Record farm reward on `nativo.pool.near` as well + - `stardust.pool.near` +=> and logs[0] not ilike '%Record%reward from farm%' From 93c9f0bf54f344ef3c7256dba1fa7456ffb68e42 Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Thu, 23 Mar 2023 17:18:22 -0600 Subject: [PATCH 2/7] flatten logs --- .../curated/silver__staking_actions_v2.sql | 76 +++++++++++++------ .../curated/silver__staking_diff_temp.sql | 34 +++++++++ 2 files changed, 88 insertions(+), 22 deletions(-) create mode 100644 models/silver/curated/silver__staking_diff_temp.sql diff --git a/models/silver/curated/silver__staking_actions_v2.sql b/models/silver/curated/silver__staking_actions_v2.sql index a073f46..d52e383 100644 --- a/models/silver/curated/silver__staking_actions_v2.sql +++ b/models/silver/curated/silver__staking_actions_v2.sql @@ -5,47 +5,79 @@ cluster_by=['_partition_by_block_number', 'block_timestamp::date'] ) }} + with +-- actions as ( +-- select * from near.silver.actions_events_s3 +-- where tx_hash in (select distinct tx_hash from near_dev.silver.staking_diff_temp) +-- and block_timestamp::date <= '2023-03-22' +-- ) +-- -- select count(distinct tx_hash) from actions; +-- , stake_actions as ( select * from near.silver.actions_events_s3 where lower(action_name) = 'stake' -), -staking as ( +) +-- select count(distinct tx_hash) from stake_actions; +-- 24,404 vs 26,886, losing about 2400 txs (from diff) with no Stake action +, +flatten_logs as ( select tx_hash, block_id, block_timestamp, receipt_object_id, - receiver_id as staking_pool, + receiver_id, signer_id, status_value, logs, - split(logs[0], ' ') as log_parts, - split(log_parts[0]::string, '@')[1]::string as log_signer_id, - log_parts[1]::string as log_action, - split(log_parts[2]::string,'.')[0]::number as log_amount, - log_amount / pow(10, 24) as log_amount_near, - length(split(log_parts[2]::string,'.')[0]::string) as _log_amount_length, - signer_id = log_signer_id as _log_signer_id_match, + value as log, _load_timestamp, _partition_by_block_number -from near.silver.streamline_receipts_final +from near.silver.streamline_receipts_final, lateral flatten(logs) where true and tx_hash in (select distinct tx_hash from stake_actions) - and receiver_id != signer_id - and receiver_id ilike '%.pool%.near' - and receiver_id not in ('usn-unofficial.poolowner.near', 'usn-unofficial.pool.near') and array_size(logs) > 0 - -- exclude epoch rewards, todo for separate model - and logs[0] not like 'Epoch%' - and logs[0] not ilike '%Record%reward from farm%' - -- explicit: - and log_action in ('staking', 'unstaking', 'deposited') +), +staking_data as ( + select + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id, + signer_id, + status_value, + log, + split(log, ' ') as log_parts, + split(log_parts[0]::string, '@')[1]::string as log_signer_id, + log_parts[1]::string as log_action, + split(log_parts[2]::string,'.')[0]::number as log_amount, + log_amount / pow(10, 24) as log_amount_near, + length(log_amount::string) as _log_amount_length, + signer_id = log_signer_id as _log_signer_id_match, + _load_timestamp, + _partition_by_block_number + from flatten_logs + where true + and receiver_id != signer_id + and receiver_id ilike '%.pool%.near' + and receiver_id not in ('usn-unofficial.poolowner.near', 'usn-unofficial.pool.near') + and log like '@%' + -- test against the below, shouldn't need to filter but idk + -- and log_action in ('staking', 'unstaking', 'deposited') + -- -- exclude epoch rewards, todo for separate model + -- and log not like 'Epoch%' + -- and log not ilike '%Record%reward from farm%' + -- and log not like 'Contract total staked balance is%' + -- and log not like 'Total rewards fee is%' ) -select * from staking --- where block_timestamp::date >= current_date - interval '7 days' --- limit 500; +select +* + from staking_data + + -- where tx_hash in ( -- 'GeiLBPrvF3MAbcPA74Nm1qNdXbsVhxYXhjdbDWGkyxwC', -- forg stake -- 'C1uy4w3wqdBictXy4Jyiqj2yNZGEHzQxiEdgR79uy3wB', -- forg unstake diff --git a/models/silver/curated/silver__staking_diff_temp.sql b/models/silver/curated/silver__staking_diff_temp.sql new file mode 100644 index 0000000..a7408ac --- /dev/null +++ b/models/silver/curated/silver__staking_diff_temp.sql @@ -0,0 +1,34 @@ +{{config(materialized='table')}} +; +with +staking_v1 as ( + select + tx_hash, + block_timestamp, + pool_address, + tx_signer, + stake_amount / pow(10,24) as amount, + action, + 'v1' as model +from near.silver.staking_actions_s3 +), +staking_v2 as ( + select + tx_hash, + block_timestamp, + receiver_id as pool_address, + signer_id as tx_signer, + log_amount_near as amount, + iff(log_action in ('deposited', 'staking'), 'Stake', 'Unstake') as action, + 'v2' as model + from near_dev.silver.staking_actions_v2 +), +diff as ( + select distinct tx_hash from staking_v1 + except + select distinct tx_hash from staking_v2 +) +select * from near.silver.streamline_receipts_final +where tx_hash in (select tx_hash from diff) +order by block_timestamp desc, tx_hash +limit 100 \ No newline at end of file From 191bace77d753df832e5af3a0e13185e358481b3 Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Thu, 23 Mar 2023 19:42:07 -0600 Subject: [PATCH 3/7] directly to logs --- .../curated/silver__staking_actions_v2.sql | 47 ++++++------------- .../curated/silver__staking_comp_temp.sql | 18 +++++++ .../curated/silver__staking_diff_temp.sql | 2 +- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/models/silver/curated/silver__staking_actions_v2.sql b/models/silver/curated/silver__staking_actions_v2.sql index d52e383..4b1b5a3 100644 --- a/models/silver/curated/silver__staking_actions_v2.sql +++ b/models/silver/curated/silver__staking_actions_v2.sql @@ -14,13 +14,13 @@ with -- ) -- -- select count(distinct tx_hash) from actions; -- , -stake_actions as ( -select * from near.silver.actions_events_s3 -where lower(action_name) = 'stake' -) --- select count(distinct tx_hash) from stake_actions; --- 24,404 vs 26,886, losing about 2400 txs (from diff) with no Stake action -, +-- stake_actions as ( +-- select * from near.silver.actions_events_s3 +-- where lower(action_name) = 'stake' +-- ) +-- -- select count(distinct tx_hash) from stake_actions; +-- -- 24,404 vs 26,886, losing about 2400 txs (from diff) with no Stake action +-- , flatten_logs as ( select tx_hash, @@ -36,9 +36,13 @@ select _partition_by_block_number from near.silver.streamline_receipts_final, lateral flatten(logs) where true - and tx_hash in (select distinct tx_hash from stake_actions) + -- and tx_hash in (select distinct tx_hash from stake_actions) and array_size(logs) > 0 -), + and receiver_id ilike '%.pool%.near' + and receiver_id not in ('usn-unofficial.poolowner.near', 'usn-unofficial.pool.near') +) + +, staking_data as ( select tx_hash, @@ -48,12 +52,13 @@ staking_data as ( receiver_id, signer_id, status_value, + logs, log, split(log, ' ') as log_parts, split(log_parts[0]::string, '@')[1]::string as log_signer_id, log_parts[1]::string as log_action, split(log_parts[2]::string,'.')[0]::number as log_amount, - log_amount / pow(10, 24) as log_amount_near, + log_amount::float / pow(10, 24) as log_amount_near, length(log_amount::string) as _log_amount_length, signer_id = log_signer_id as _log_signer_id_match, _load_timestamp, @@ -61,30 +66,8 @@ staking_data as ( from flatten_logs where true and receiver_id != signer_id - and receiver_id ilike '%.pool%.near' - and receiver_id not in ('usn-unofficial.poolowner.near', 'usn-unofficial.pool.near') and log like '@%' - -- test against the below, shouldn't need to filter but idk - -- and log_action in ('staking', 'unstaking', 'deposited') - - -- -- exclude epoch rewards, todo for separate model - -- and log not like 'Epoch%' - -- and log not ilike '%Record%reward from farm%' - -- and log not like 'Contract total staked balance is%' - -- and log not like 'Total rewards fee is%' ) select * from staking_data - - - -- where tx_hash in ( - -- 'GeiLBPrvF3MAbcPA74Nm1qNdXbsVhxYXhjdbDWGkyxwC', -- forg stake - -- 'C1uy4w3wqdBictXy4Jyiqj2yNZGEHzQxiEdgR79uy3wB', -- forg unstake - -- '2Zqi2S7LDAUby3kpM2bYSLojX6TtTSLE7x5zTnSD5KKj', -- mdao stake - -- 'H2ozLWxhk6UbVAnTgzKFSKsuooi7kYsruJgfx2A1HEhy', -- random - -- '5P7rw88ZBLzLsMZgfV5VsEXeLiXQ4KefVvq1puyBQJtU', -- arca unstake 1 - -- 'ABcv2Aozny6Uvi5Lb8SC6YuVBbpY1uUTSS9fAQ2Phndy', -- arca stake 1 - -- '3smwZoSeWD5bzSScSnB3yRZJqVsjkuAZu33v78aqhUtj' -- dnd unstake 25 - -- ) - diff --git a/models/silver/curated/silver__staking_comp_temp.sql b/models/silver/curated/silver__staking_comp_temp.sql index acac2c3..37fcb18 100644 --- a/models/silver/curated/silver__staking_comp_temp.sql +++ b/models/silver/curated/silver__staking_comp_temp.sql @@ -30,4 +30,22 @@ select from staked_v1 v1 full join staked_v2 v2 on v1.date = v2.date and v1.action = v2.action +order by 1,2; + +with staked as +(select + block_timestamp::date as date, + iff(log_action = 'staking','Stake','Unstake') as action, + sum(log_amount_near) as amount +from near_dev.silver.staking_actions_v2 +where log_action in ('staking', 'unstaking') +group by 1, 2 +) + +select + date, + action, + amount, + sum(amount) over (partition by action order by date asc rows between unbounded preceding and current row) as staking_balance +from staked order by 1,2; \ No newline at end of file diff --git a/models/silver/curated/silver__staking_diff_temp.sql b/models/silver/curated/silver__staking_diff_temp.sql index a7408ac..e5cc7b3 100644 --- a/models/silver/curated/silver__staking_diff_temp.sql +++ b/models/silver/curated/silver__staking_diff_temp.sql @@ -31,4 +31,4 @@ diff as ( select * from near.silver.streamline_receipts_final where tx_hash in (select tx_hash from diff) order by block_timestamp desc, tx_hash -limit 100 \ No newline at end of file +limit 500; \ No newline at end of file From 86b62ef5748662062c2c8454aeef82f877e389b9 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Tue, 4 Apr 2023 16:06:27 -0600 Subject: [PATCH 4/7] core views and docs --- models/core/core__dim_staking_actions.yml | 42 +----- models/core/core__fact_staking_actions.sql | 31 ++++ models/core/core__fact_staking_actions.yml | 33 +++++ .../core/core__fact_staking_pool_balances.sql | 29 ++++ .../core/core__fact_staking_pool_balances.yml | 27 ++++ models/descriptions/__overview__.md | 5 +- models/descriptions/_log_signer_id_match.md | 5 + models/descriptions/amount_adj.md | 5 + models/descriptions/amount_raw.md | 5 + models/descriptions/balance.md | 5 + models/descriptions/log.md | 5 + models/descriptions/log_signer_id.md | 5 + models/descriptions/logs.md | 2 +- models/descriptions/pool_address.md | 5 + models/descriptions/staking_action.md | 2 +- .../silver/curated/silver__pool_balances.sql | 55 +++++++ .../silver/curated/silver__pool_balances.yml | 57 ++++++++ models/silver/curated/silver__pool_events.sql | 45 ++++++ models/silver/curated/silver__pool_events.yml | 73 ++++++++++ .../curated/silver__staking_actions_v2.sql | 135 ++++++++++-------- .../curated/silver__staking_actions_v2.yml | 96 +++++++++++++ .../curated/silver__staking_comp_temp.sql | 51 ------- .../silver/curated/silver__staking_notes.md | 85 ----------- 23 files changed, 563 insertions(+), 240 deletions(-) create mode 100644 models/core/core__fact_staking_actions.sql create mode 100644 models/core/core__fact_staking_actions.yml create mode 100644 models/core/core__fact_staking_pool_balances.sql create mode 100644 models/core/core__fact_staking_pool_balances.yml create mode 100644 models/descriptions/_log_signer_id_match.md create mode 100644 models/descriptions/amount_adj.md create mode 100644 models/descriptions/amount_raw.md create mode 100644 models/descriptions/balance.md create mode 100644 models/descriptions/log.md create mode 100644 models/descriptions/log_signer_id.md create mode 100644 models/descriptions/pool_address.md create mode 100644 models/silver/curated/silver__pool_balances.sql create mode 100644 models/silver/curated/silver__pool_balances.yml create mode 100644 models/silver/curated/silver__pool_events.sql create mode 100644 models/silver/curated/silver__pool_events.yml create mode 100644 models/silver/curated/silver__staking_actions_v2.yml delete mode 100644 models/silver/curated/silver__staking_comp_temp.sql delete mode 100644 models/silver/curated/silver__staking_notes.md diff --git a/models/core/core__dim_staking_actions.yml b/models/core/core__dim_staking_actions.yml index bbe1f96..20258f4 100644 --- a/models/core/core__dim_staking_actions.yml +++ b/models/core/core__dim_staking_actions.yml @@ -3,63 +3,23 @@ version: 2 models: - name: core__dim_staking_actions description: |- - This table records all the staking and delegation transactions occurring in NEAR. - - Be aware, however, not all transactions here necessarily actually contribute to securing the NEAR network. - Each epoch can have a different set of validators. + Deprecating soon! This table is being deprecated due to inaccuracies. The underlying model has been disabled and will not produce new data past early April 2023. Please use the new fact staking table(s). columns: - name: TX_HASH description: "{{ doc('tx_hash') }}" - tests: - - unique - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - name: POOL_ADDRESS description: "{{ doc('staking_pool_address') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: TX_SIGNER description: "{{ doc('tx_signer') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: STAKE_AMOUNT description: "{{ doc('staking_stake_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - name: ACTION description: "{{ doc('staking_action') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - dbt_expectations.expect_column_values_to_be_in_set: - value_set: ['Stake', 'Unstake'] diff --git a/models/core/core__fact_staking_actions.sql b/models/core/core__fact_staking_actions.sql new file mode 100644 index 0000000..33d2e04 --- /dev/null +++ b/models/core/core__fact_staking_actions.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view', + secure = true, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING' + } + } + }, + tags = ['core'] +) }} + +WITH staking_actions AS ( + + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id AS pool_address, + signer_id, + action, + amount_adj AS amount + FROM + {{ ref('silver__staking_actions_v2') }} +) +SELECT + * +FROM + staking_actions diff --git a/models/core/core__fact_staking_actions.yml b/models/core/core__fact_staking_actions.yml new file mode 100644 index 0000000..a6bed5c --- /dev/null +++ b/models/core/core__fact_staking_actions.yml @@ -0,0 +1,33 @@ +version: 2 + +models: + - name: core__fact_staking_actions + description: |- + An updated version of the staking actions table which looks at all logs, instead of just the first receipt. + There are four actions taken when staking: staking->deposit->unstaking->withdraw. + Note - in this core view the amount is decimal adjusted by 10^24. + + columns: + - name: TX_HASH + description: "{{ doc('tx_hash') }}" + + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + + - name: RECEIPT_OBJECT_ID + description: "{{ doc('receipt_object_id') }}" + + - name: POOL_ADDRESS + description: "{{ doc('pool_address') }}" + + - name: SIGNER_ID + description: "{{ doc('signer_id') }}" + + - name: ACTION + description: "{{ doc('staking_action') }}" + + - name: AMOUNT + description: "{{ doc('amount') }}" diff --git a/models/core/core__fact_staking_pool_balances.sql b/models/core/core__fact_staking_pool_balances.sql new file mode 100644 index 0000000..6689f61 --- /dev/null +++ b/models/core/core__fact_staking_pool_balances.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view', + secure = true, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING' + } + } + }, + tags = ['core'] +) }} + +WITH balance_changes AS ( + + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id AS pool_address, + amount_adj AS balance + FROM + {{ ref('silver__pool_balances') }} +) +SELECT + * +FROM + balance_changes diff --git a/models/core/core__fact_staking_pool_balances.yml b/models/core/core__fact_staking_pool_balances.yml new file mode 100644 index 0000000..bfb3efd --- /dev/null +++ b/models/core/core__fact_staking_pool_balances.yml @@ -0,0 +1,27 @@ +version: 2 + +models: + - name: core__fact_staking_pool_balances + description: |- + Staking pool balances as extracted from receipt logs when an individual makes a staking action. + To calculate balance at a point in time, isolate a single record for each pool. This table is transactional-based, so balances are updated with every staking event by users. + Note - the amount in balance is decimal adjusted by 10^24. + + columns: + - name: tx_hash + description: "{{ doc('tx_hash') }}" + + - name: block_id + description: "{{ doc('block_id') }}" + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + + - name: receipt_object_id + description: "{{ doc('receipt_object_id') }}" + + - name: pool_address + description: "{{ doc('pool_address') }}" + + - name: balance + description: "{{ doc('balance') }}" diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index b7631bd..7e3be82 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -21,7 +21,6 @@ There is more information on how to use dbt docs in the last section of this doc **Dimension Tables:** - [dim_address_labels](#!/model/model.near.core__dim_address_labels) -- [dim_staking_actions](#!/model/model.near.core__dim_staking_actions) - [dim_staking_pools](#!/model/model.near.core__dim_staking_pools) - [dim_token_labels](#!/model/model.near.core__dim_token_labels) @@ -34,6 +33,8 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_token_metadata](#!/model/model.near.core__fact_token_metadata) - [fact_transactions](#!/model/model.near.core__fact_transactions) - [fact_transfers](#!/model/model.near.core__fact_transfers) +- [fact_staking_actions](#!/model/model.near.core__fact_staking_actions) +- [fact_staking_pool_balances](#!/model/model.near.core__fact_staking_pool_balances) **Convenience Tables:** @@ -45,6 +46,8 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_addkey_events](#!/model/model.near.social__fact_addkey_events) - [fact_decoded_actions](#!/model/model.near.social__fact_decoded_actions) - [fact_profile_changes](#!/model/model.near.social__fact_profile_changes) +- [fact_posts](#!/model/model.near.social__fact_posts) +- [fact_widget_deployments](#!/model/model.near.social__fact_widget_deployments) ### Beta Tables (`NEAR`.`BETA`.``) diff --git a/models/descriptions/_log_signer_id_match.md b/models/descriptions/_log_signer_id_match.md new file mode 100644 index 0000000..f2e5209 --- /dev/null +++ b/models/descriptions/_log_signer_id_match.md @@ -0,0 +1,5 @@ +{% docs _log_signer_id_match %} + +A boolean indicating whether the signer_id of the log matches the signer_id of the transaction. Internal / silver column only. + +{% enddocs %} diff --git a/models/descriptions/amount_adj.md b/models/descriptions/amount_adj.md new file mode 100644 index 0000000..6e6dc9a --- /dev/null +++ b/models/descriptions/amount_adj.md @@ -0,0 +1,5 @@ +{% docs amount_adj %} + +A decimal adjusted amount (of tokens, price, etc.). + +{% enddocs %} diff --git a/models/descriptions/amount_raw.md b/models/descriptions/amount_raw.md new file mode 100644 index 0000000..57ef3ba --- /dev/null +++ b/models/descriptions/amount_raw.md @@ -0,0 +1,5 @@ +{% docs amount_raw %} + +An unadjusted amount (of tokens, price, etc.) for the relevant record. This is the number as it appears on chain and is not decimal adjusted. + +{% enddocs %} diff --git a/models/descriptions/balance.md b/models/descriptions/balance.md new file mode 100644 index 0000000..9c0d6f7 --- /dev/null +++ b/models/descriptions/balance.md @@ -0,0 +1,5 @@ +{% docs balance %} + +The balance of a pool or address at a given block. + +{% enddocs %} diff --git a/models/descriptions/log.md b/models/descriptions/log.md new file mode 100644 index 0000000..dd71474 --- /dev/null +++ b/models/descriptions/log.md @@ -0,0 +1,5 @@ +{% docs log %} + +A single log extracted from a receipt. + +{% enddocs %} diff --git a/models/descriptions/log_signer_id.md b/models/descriptions/log_signer_id.md new file mode 100644 index 0000000..6ce880a --- /dev/null +++ b/models/descriptions/log_signer_id.md @@ -0,0 +1,5 @@ +{% docs log_signer_id %} + +The NEAR address indicated in the log. + +{% enddocs %} diff --git a/models/descriptions/logs.md b/models/descriptions/logs.md index e75f00f..15c30ec 100644 --- a/models/descriptions/logs.md +++ b/models/descriptions/logs.md @@ -1,5 +1,5 @@ {% docs logs %} -Logs (array) for this transaction. +Logs (array) for this receipt. {% enddocs %} diff --git a/models/descriptions/pool_address.md b/models/descriptions/pool_address.md new file mode 100644 index 0000000..2e45b5c --- /dev/null +++ b/models/descriptions/pool_address.md @@ -0,0 +1,5 @@ +{% docs pool_address %} + +The NEAR address of the staking pool. + +{% enddocs %} diff --git a/models/descriptions/staking_action.md b/models/descriptions/staking_action.md index fb4512b..f34f643 100644 --- a/models/descriptions/staking_action.md +++ b/models/descriptions/staking_action.md @@ -1,5 +1,5 @@ {% docs staking_action %} -The staking action performed in this transaction. Can be `"Stake"` or `"Unstake"`. +The staking action performed in this transaction. Can be `"Stake"` or `"Unstake"` if in the deprecating `dim_staking_actions` table, or `staking`, `unstaking`, `deposited`, `withdrawing` if in the new `fact_staking_actions` table. These method names are taken directly from the log in which they occur. {% enddocs %} diff --git a/models/silver/curated/silver__pool_balances.sql b/models/silver/curated/silver__pool_balances.sql new file mode 100644 index 0000000..3e3c472 --- /dev/null +++ b/models/silver/curated/silver__pool_balances.sql @@ -0,0 +1,55 @@ +{{ config( + materialized = 'table', + unique_key = 'tx_hash', + incremental_strategy = 'delete+insert', + tags = ['curated'], + cluster_by = ['_partition_by_block_number', 'block_timestamp::date'] +) }} + +WITH pool_events AS ( + + SELECT + * + FROM + {{ ref('silver__pool_events') }} + WHERE + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {{ partition_load_manual('no_buffer') }} + {% else %} + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} +), +FINAL AS ( + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id, + signer_id, + LOG, + TO_NUMBER( + REGEXP_SUBSTR( + LOG, + 'Contract total staked balance is (\\d+)', + 1, + 1, + 'e', + 1 + ) + ) AS amount_raw, + amount_raw / pow( + 10, + 24 + ) AS amount_adj, + _load_timestamp, + _partition_by_block_number + FROM + pool_events + WHERE + LOG LIKE 'Contract total staked balance is%' +) +SELECT + * +FROM + FINAL diff --git a/models/silver/curated/silver__pool_balances.yml b/models/silver/curated/silver__pool_balances.yml new file mode 100644 index 0000000..08ccf04 --- /dev/null +++ b/models/silver/curated/silver__pool_balances.yml @@ -0,0 +1,57 @@ +version: 2 + +models: + - name: silver__pool_balances + description: |- + Staking pool balances as extracted from receipt logs when an individual makes a staking action. + To calculate balance at a point in time, isolate a single record for each pool. This table is transactional-based, so balances are updated with every staking event by users. + + columns: + - name: tx_hash + description: "{{ doc('tx_hash') }}" + + - name: block_id + description: "{{ doc('block_id') }}" + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + + - name: receipt_object_id + description: "{{ doc('receipt_object_id') }}" + + - name: receiver_id + description: "{{ doc('receiver_id') }}" + + - name: signer_id + description: "{{ doc('signer_id') }}" + + - name: LOG + description: "{{ doc('log') }}" + + - name: amount_raw + description: "{{ doc('amount_raw') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: amount_adj + description: "{{ doc('amount_adj') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - DOUBLE + + - name: _LOAD_TIMESTAMP + description: "{{ doc('_load_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" diff --git a/models/silver/curated/silver__pool_events.sql b/models/silver/curated/silver__pool_events.sql new file mode 100644 index 0000000..874b88e --- /dev/null +++ b/models/silver/curated/silver__pool_events.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'table', + unique_key = 'tx_hash', + incremental_strategy = 'delete+insert', + tags = ['curated'], + cluster_by = ['_partition_by_block_number', 'block_timestamp::date'] +) }} + +WITH receipts AS ( + + SELECT + * + FROM + {{ ref('silver__streamline_receipts_final') }} + WHERE + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {{ partition_load_manual('no_buffer') }} + {% else %} + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} +), +FINAL AS ( + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id, + signer_id, + status_value, + logs, + VALUE AS LOG, + _load_timestamp, + _partition_by_block_number + FROM + receipts, + LATERAL FLATTEN(logs) + WHERE + ARRAY_SIZE(logs) > 0 + AND receiver_id ILIKE '%.pool%.near' +) +SELECT + * +FROM + FINAL diff --git a/models/silver/curated/silver__pool_events.yml b/models/silver/curated/silver__pool_events.yml new file mode 100644 index 0000000..cf74750 --- /dev/null +++ b/models/silver/curated/silver__pool_events.yml @@ -0,0 +1,73 @@ +version: 2 + +models: + - name: silver__pool_events + description: |- + Flattened logs from receipt events where the recipient is a staking pool, for use in multiple downstream models. + + columns: + - name: tx_hash + description: "{{ doc('tx_hash') }}" + tests: + - not_null: + where: _load_timestamp <= current_timestamp - interval '2 hours' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + + - name: block_id + description: "{{ doc('block_id') }}" + tests: + - not_null + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + + - name: receipt_object_id + description: "{{ doc('receipt_object_id') }}" + tests: + - not_null + + - name: receiver_id + description: "{{ doc('receiver_id') }}" + tests: + - not_null + + - name: signer_id + description: "{{ doc('signer_id') }}" + tests: + - not_null + + - name: status_value + description: "{{ doc('status_value') }}" + tests: + - not_null + + - name: logs + description: "{{ doc('logs') }}" + tests: + - not_null + + - name: LOG + description: "{{ doc('log') }}" + tests: + - not_null + + - name: _LOAD_TIMESTAMP + description: "{{ doc('_load_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" diff --git a/models/silver/curated/silver__staking_actions_v2.sql b/models/silver/curated/silver__staking_actions_v2.sql index 4b1b5a3..ee97d04 100644 --- a/models/silver/curated/silver__staking_actions_v2.sql +++ b/models/silver/curated/silver__staking_actions_v2.sql @@ -1,50 +1,26 @@ -{{config(materialized='table', - unique_key='tx_hash', - incremental_strategy='delete+insert', - tags=['curated'], - cluster_by=['_partition_by_block_number', 'block_timestamp::date'] - ) -}} +{{ config( + materialized = 'table', + unique_key = 'tx_hash', + incremental_strategy = 'delete+insert', + tags = ['curated'], + cluster_by = ['_partition_by_block_number', 'block_timestamp::date'] +) }} -with --- actions as ( --- select * from near.silver.actions_events_s3 --- where tx_hash in (select distinct tx_hash from near_dev.silver.staking_diff_temp) --- and block_timestamp::date <= '2023-03-22' --- ) --- -- select count(distinct tx_hash) from actions; --- , --- stake_actions as ( --- select * from near.silver.actions_events_s3 --- where lower(action_name) = 'stake' --- ) --- -- select count(distinct tx_hash) from stake_actions; --- -- 24,404 vs 26,886, losing about 2400 txs (from diff) with no Stake action --- , -flatten_logs as ( -select - tx_hash, - block_id, - block_timestamp, - receipt_object_id, - receiver_id, - signer_id, - status_value, - logs, - value as log, - _load_timestamp, - _partition_by_block_number -from near.silver.streamline_receipts_final, lateral flatten(logs) - where true - -- and tx_hash in (select distinct tx_hash from stake_actions) - and array_size(logs) > 0 - and receiver_id ilike '%.pool%.near' - and receiver_id not in ('usn-unofficial.poolowner.near', 'usn-unofficial.pool.near') -) +WITH pool_events AS ( -, -staking_data as ( - select + SELECT + * + FROM + {{ ref('silver__pool_events') }} + WHERE + {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {{ partition_load_manual('no_buffer') }} + {% else %} + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} +), +staking_actions AS ( + SELECT tx_hash, block_id, block_timestamp, @@ -53,21 +29,60 @@ staking_data as ( signer_id, status_value, logs, - log, - split(log, ' ') as log_parts, - split(log_parts[0]::string, '@')[1]::string as log_signer_id, - log_parts[1]::string as log_action, - split(log_parts[2]::string,'.')[0]::number as log_amount, - log_amount::float / pow(10, 24) as log_amount_near, - length(log_amount::string) as _log_amount_length, - signer_id = log_signer_id as _log_signer_id_match, + LOG, + SPLIT( + LOG, + ' ' + ) AS log_parts, + SPLIT( + log_parts [0] :: STRING, + '@' + ) [1] :: STRING AS log_signer_id, + log_parts [1] :: STRING AS action, + SPLIT( + log_parts [2] :: STRING, + '.' + ) [0] :: NUMBER AS amount_raw, + amount_raw :: FLOAT / pow( + 10, + 24 + ) AS amount_adj, + LENGTH( + amount_raw :: STRING + ) AS decimals, + signer_id = log_signer_id AS _log_signer_id_match, _load_timestamp, _partition_by_block_number - from flatten_logs - where true - and receiver_id != signer_id - and log like '@%' + FROM + pool_events + WHERE + TRUE + AND receiver_id != signer_id + AND LOG LIKE '@%' +), +FINAL AS ( + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id, + signer_id, + status_value, + logs, + LOG, + log_signer_id, + action, + amount_raw, + amount_adj, + decimals, + _log_signer_id_match, + _load_timestamp, + _partition_by_block_number + FROM + staking_actions ) -select -* - from staking_data +SELECT + * +FROM + FINAL diff --git a/models/silver/curated/silver__staking_actions_v2.yml b/models/silver/curated/silver__staking_actions_v2.yml new file mode 100644 index 0000000..4977c64 --- /dev/null +++ b/models/silver/curated/silver__staking_actions_v2.yml @@ -0,0 +1,96 @@ +version: 2 + +models: + - name: silver__staking_actions_v2 + description: |- + An updated version of the staking actions table which looks at all logs, instead of just the first receipt. + There are four actions taken when staking: staking->deposit->unstaking->withdraw. + + columns: + - name: tx_hash + description: "{{ doc('tx_hash') }}" + + - name: block_id + description: "{{ doc('block_id') }}" + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + + - name: receipt_object_id + description: "{{ doc('receipt_object_id') }}" + + - name: receiver_id + description: "{{ doc('receiver_id') }}" + + - name: signer_id + description: "{{ doc('signer_id') }}" + + - name: status_value + description: "{{ doc('status_value') }}" + + - name: logs + description: "{{ doc('logs') }}" + + - name: LOG + description: "{{ doc('log') }}" + + - name: log_signer_id + description: "{{ doc('log_signer_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + + - name: action + description: "{{ doc('staking_action') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - accepted_values: + values: + - deposited + - withdrawing + - staking + - unstaking + + - name: amount_raw + description: "{{ doc('amount_raw') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: amount_adj + description: "{{ doc('amount_adj') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - DOUBLE + + - name: decimals + description: "{{ doc('decimals') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: _log_signer_id_match + description: "{{ doc('_log_signer_id_match') }}" + + - name: _LOAD_TIMESTAMP + description: "{{ doc('_load_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" diff --git a/models/silver/curated/silver__staking_comp_temp.sql b/models/silver/curated/silver__staking_comp_temp.sql deleted file mode 100644 index 37fcb18..0000000 --- a/models/silver/curated/silver__staking_comp_temp.sql +++ /dev/null @@ -1,51 +0,0 @@ - -with staked_v1 as ( -select - block_timestamp::date as date, - action, - count(distinct tx_hash) as txs, - sum(stake_amount/pow(10,24)) as amount -from near.core.dim_staking_actions -group by 1, 2 -), -staked_v2 as ( -select - block_timestamp::date as date, - iff(log_action in ('deposited', 'staking'), 'Stake', 'Unstake') as action, - count(distinct tx_hash) as txs, - sum(log_amount_near) as amount -from near_dev.silver.staking_actions_v2 -group by 1, 2 -) -select - v1.date as date_v1, - v1.action as action_v1, - v1.txs as txs_v1, - v1.amount as amount_v1, - v2.date as date_v2, - v2.action as action_v2, - v2.txs as txs_v2, - v2.amount as amount_v2, - coalesce(amount_v1,0) = coalesce(amount_v2,0) as amount_match -from staked_v1 v1 -full join staked_v2 v2 -on v1.date = v2.date and v1.action = v2.action -order by 1,2; - -with staked as -(select - block_timestamp::date as date, - iff(log_action = 'staking','Stake','Unstake') as action, - sum(log_amount_near) as amount -from near_dev.silver.staking_actions_v2 -where log_action in ('staking', 'unstaking') -group by 1, 2 -) - -select - date, - action, - amount, - sum(amount) over (partition by action order by date asc rows between unbounded preceding and current row) as staking_balance -from staked -order by 1,2; \ No newline at end of file diff --git a/models/silver/curated/silver__staking_notes.md b/models/silver/curated/silver__staking_notes.md deleted file mode 100644 index f58cbe9..0000000 --- a/models/silver/curated/silver__staking_notes.md +++ /dev/null @@ -1,85 +0,0 @@ -# staking fix - -## methods -### staking actions -*current* - - deposit_and_stake - - stake - - unstake - - unstake_all - - *investigate* - - on_stake_action - - has the most actions at 626k while deposit and stake is 2 at 400k. Could be the missing events - - args are always empty, but this event could possible be used to find other staking events - - liquid_unstake - - on_staking_pool_withdraw - - on_staking_pool_unstake - - withdraw_all_from_staking_pool - - on_get_account_unstaked_balance_to_withdraw_by_owner - - distribute_staking - - on_staking_pool_deposit_and_stake - -### staking pools -*current* - - create_staking_pool - - update_reward_fee_fraction - - *investigate* - - set_staking_pool(s) - -*interesting* - - refresh_staking_pool_balance - - proxy_get_pool_info_callback - - get_account_unstaked_balance - - get_pool_info_callback - -*unlikely, but lots of actions* - - get_stable_pool - - get_pool_shares - - balance_stable_pool - - get_pool - -# solution -honing in on a solution by - - looking for a `Stake` action in the events - - receipts where a recipient is a staking pool - - logs exist - -## potential issues - -### log_action not in ('unstaking', 'deposited') - - - - -### where not _log_signer_id_match -There are instances where the receipt signer does not match the address named in the log. 36.6k to be exact. -In once instance - the staker is `linear-protocol.near` while the signer is `operator.linear-protocol.near`. Others it is not so clear what the relationship is. - - some (unk how many) might be addresses that are not yet defined `.near` addresses - - seeing some other sub-addresses like `storage.herewallet.near` vs `owner.herewallet.near` -- i don't need to filter on this test so it's fine - -### epoch rewards -There are also Epoch rewards in this pattern, which will make a good table on their own. -Logs are something like this: -> Epoch N: Contract received total rewards of X tokens. etc... -Example tx: '6UfgPwpwSxEb9WPvVh9jTHceTSK8U3wDLc7V8zQ6pGCu' - - looks like this is what the discord user looked at to try and calculate total staked amount (close, was not exact) - - query in snowflake/near/staking - - ### usn staking? - usn-unofficial.poolowner.near - - https://nearblocks.io/txns/5Zy7Vi8jCxiY6ErP7VvUUADM25yD3QawobGbnfaDxSTT#execution - - https://nearblocks.io/txns/HbNHechdxs24HAf4Gk2P5ogq2gVnkE7S7hCmtynmRzid - - exclude this pool as i dont think it is near staking - - TODO - maybe name the table NEAR staking? or something like that - -### aurora pool -aurora.pool.near -Seem to follow a pattern like -> Record 0 TOKEN reward from farm #0" - - is there an instance where the reward is not 0? - - TODO - check where receiver_id is `aurora.pool.near` - - seeing this Record farm reward on `nativo.pool.near` as well - - `stardust.pool.near` -=> and logs[0] not ilike '%Record%reward from farm%' From 3001c50b287e609a3c4cbf41fecf3b9f3a09b3e4 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Tue, 4 Apr 2023 16:08:55 -0600 Subject: [PATCH 5/7] del temp model --- .../curated/silver__staking_diff_temp.sql | 34 ------------------- 1 file changed, 34 deletions(-) delete mode 100644 models/silver/curated/silver__staking_diff_temp.sql diff --git a/models/silver/curated/silver__staking_diff_temp.sql b/models/silver/curated/silver__staking_diff_temp.sql deleted file mode 100644 index e5cc7b3..0000000 --- a/models/silver/curated/silver__staking_diff_temp.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{config(materialized='table')}} -; -with -staking_v1 as ( - select - tx_hash, - block_timestamp, - pool_address, - tx_signer, - stake_amount / pow(10,24) as amount, - action, - 'v1' as model -from near.silver.staking_actions_s3 -), -staking_v2 as ( - select - tx_hash, - block_timestamp, - receiver_id as pool_address, - signer_id as tx_signer, - log_amount_near as amount, - iff(log_action in ('deposited', 'staking'), 'Stake', 'Unstake') as action, - 'v2' as model - from near_dev.silver.staking_actions_v2 -), -diff as ( - select distinct tx_hash from staking_v1 - except - select distinct tx_hash from staking_v2 -) -select * from near.silver.streamline_receipts_final -where tx_hash in (select tx_hash from diff) -order by block_timestamp desc, tx_hash -limit 500; \ No newline at end of file From 9f19acc500a34d7ee74c1b9690d8ec3709155c41 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Tue, 4 Apr 2023 17:53:46 -0600 Subject: [PATCH 6/7] col name --- models/core/core__fact_staking_pool_balances.sql | 2 +- models/core/core__fact_staking_pool_balances.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/core/core__fact_staking_pool_balances.sql b/models/core/core__fact_staking_pool_balances.sql index 6689f61..79a0c3c 100644 --- a/models/core/core__fact_staking_pool_balances.sql +++ b/models/core/core__fact_staking_pool_balances.sql @@ -18,7 +18,7 @@ WITH balance_changes AS ( block_id, block_timestamp, receipt_object_id, - receiver_id AS pool_address, + receiver_id AS address, amount_adj AS balance FROM {{ ref('silver__pool_balances') }} diff --git a/models/core/core__fact_staking_pool_balances.yml b/models/core/core__fact_staking_pool_balances.yml index bfb3efd..291c9c2 100644 --- a/models/core/core__fact_staking_pool_balances.yml +++ b/models/core/core__fact_staking_pool_balances.yml @@ -20,7 +20,7 @@ models: - name: receipt_object_id description: "{{ doc('receipt_object_id') }}" - - name: pool_address + - name: address description: "{{ doc('pool_address') }}" - name: balance From 34befa1afdf96725f1a30995d2ab695f635d5664 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Tue, 4 Apr 2023 18:06:13 -0600 Subject: [PATCH 7/7] address --- models/core/core__fact_staking_actions.sql | 2 +- models/core/core__fact_staking_actions.yml | 2 +- models/core/core__fact_staking_pool_balances.yml | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/models/core/core__fact_staking_actions.sql b/models/core/core__fact_staking_actions.sql index 33d2e04..9f9cdaa 100644 --- a/models/core/core__fact_staking_actions.sql +++ b/models/core/core__fact_staking_actions.sql @@ -18,7 +18,7 @@ WITH staking_actions AS ( block_id, block_timestamp, receipt_object_id, - receiver_id AS pool_address, + receiver_id AS address, signer_id, action, amount_adj AS amount diff --git a/models/core/core__fact_staking_actions.yml b/models/core/core__fact_staking_actions.yml index a6bed5c..69a55e3 100644 --- a/models/core/core__fact_staking_actions.yml +++ b/models/core/core__fact_staking_actions.yml @@ -20,7 +20,7 @@ models: - name: RECEIPT_OBJECT_ID description: "{{ doc('receipt_object_id') }}" - - name: POOL_ADDRESS + - name: ADDRESS description: "{{ doc('pool_address') }}" - name: SIGNER_ID diff --git a/models/core/core__fact_staking_pool_balances.yml b/models/core/core__fact_staking_pool_balances.yml index 291c9c2..46b431d 100644 --- a/models/core/core__fact_staking_pool_balances.yml +++ b/models/core/core__fact_staking_pool_balances.yml @@ -8,20 +8,20 @@ models: Note - the amount in balance is decimal adjusted by 10^24. columns: - - name: tx_hash + - name: TX_HASH description: "{{ doc('tx_hash') }}" - - name: block_id + - name: BLOCK_ID description: "{{ doc('block_id') }}" - - name: block_timestamp + - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" - - name: receipt_object_id + - name: RECEIPT_OBJECT_ID description: "{{ doc('receipt_object_id') }}" - - name: address + - name: ADDRESS description: "{{ doc('pool_address') }}" - - name: balance + - name: BALANCE description: "{{ doc('balance') }}"