From 6389d8f689bda90ead626c0db96a60ecc87f6573 Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Thu, 27 Jul 2023 16:18:20 -0600 Subject: [PATCH] add silver epoch parsing model --- models/descriptions/_epoch_id.md | 5 ++ models/descriptions/epoch_number.md | 5 ++ models/descriptions/reward_tokens.md | 5 ++ models/descriptions/total_staked_balance.md | 5 ++ models/descriptions/total_staking_shares.md | 5 ++ .../silver__pool_balance_daily.sql | 0 .../silver__pool_balance_daily.yml | 0 .../{ => staking}/silver__pool_balances.sql | 0 .../{ => staking}/silver__pool_balances.yml | 0 .../{ => staking}/silver__pool_events.sql | 1 + .../{ => staking}/silver__pool_events.yml | 0 .../silver__prices_oracle_s3.sql | 0 .../silver__prices_oracle_s3.yml | 0 .../silver__staking_actions_s3.sql | 0 .../silver__staking_actions_s3.yml | 0 .../silver__staking_actions_v2.sql | 0 .../silver__staking_actions_v2.yml | 0 .../staking/silver__staking_epochs.sql | 61 +++++++++++++++ .../staking/silver__staking_epochs.yml | 77 +++++++++++++++++++ .../silver__staking_pools_s3.sql | 0 .../silver__staking_pools_s3.yml | 0 21 files changed, 164 insertions(+) create mode 100644 models/descriptions/_epoch_id.md create mode 100644 models/descriptions/epoch_number.md create mode 100644 models/descriptions/reward_tokens.md create mode 100644 models/descriptions/total_staked_balance.md create mode 100644 models/descriptions/total_staking_shares.md rename models/silver/curated/{ => staking}/silver__pool_balance_daily.sql (100%) rename models/silver/curated/{ => staking}/silver__pool_balance_daily.yml (100%) rename models/silver/curated/{ => staking}/silver__pool_balances.sql (100%) rename models/silver/curated/{ => staking}/silver__pool_balances.yml (100%) rename models/silver/curated/{ => staking}/silver__pool_events.sql (96%) rename models/silver/curated/{ => staking}/silver__pool_events.yml (100%) rename models/silver/curated/{ => staking}/silver__prices_oracle_s3.sql (100%) rename models/silver/curated/{ => staking}/silver__prices_oracle_s3.yml (100%) rename models/silver/curated/{ => staking}/silver__staking_actions_s3.sql (100%) rename models/silver/curated/{ => staking}/silver__staking_actions_s3.yml (100%) rename models/silver/curated/{ => staking}/silver__staking_actions_v2.sql (100%) rename models/silver/curated/{ => staking}/silver__staking_actions_v2.yml (100%) create mode 100644 models/silver/curated/staking/silver__staking_epochs.sql create mode 100644 models/silver/curated/staking/silver__staking_epochs.yml rename models/silver/curated/{ => staking}/silver__staking_pools_s3.sql (100%) rename models/silver/curated/{ => staking}/silver__staking_pools_s3.yml (100%) diff --git a/models/descriptions/_epoch_id.md b/models/descriptions/_epoch_id.md new file mode 100644 index 0000000..58f3ffc --- /dev/null +++ b/models/descriptions/_epoch_id.md @@ -0,0 +1,5 @@ +{% docs _epoch_id %} + +An internal identifier for the epoch. This is a hashed representation of the pool_id and epoch number. + +{% enddocs %} diff --git a/models/descriptions/epoch_number.md b/models/descriptions/epoch_number.md new file mode 100644 index 0000000..ce0f8b8 --- /dev/null +++ b/models/descriptions/epoch_number.md @@ -0,0 +1,5 @@ +{% docs epoch_number %} + +The epoch number for this staking reward payout. Epoch increment in order every 43,200 blocks. + +{% enddocs %} diff --git a/models/descriptions/reward_tokens.md b/models/descriptions/reward_tokens.md new file mode 100644 index 0000000..447600b --- /dev/null +++ b/models/descriptions/reward_tokens.md @@ -0,0 +1,5 @@ +{% docs reward_tokens %} + +The total amount of tokens rewarded to the validator for this staking reward payout. Number is presented raw and not decimal adjusted in the fact table. + +{% enddocs %} diff --git a/models/descriptions/total_staked_balance.md b/models/descriptions/total_staked_balance.md new file mode 100644 index 0000000..13e2a0b --- /dev/null +++ b/models/descriptions/total_staked_balance.md @@ -0,0 +1,5 @@ +{% docs total_staked_balance %} + +The total staked NEAR at the time of the reward payout. Number is presented raw and not decimal adjusted in the fact table. + +{% enddocs %} diff --git a/models/descriptions/total_staking_shares.md b/models/descriptions/total_staking_shares.md new file mode 100644 index 0000000..260ca25 --- /dev/null +++ b/models/descriptions/total_staking_shares.md @@ -0,0 +1,5 @@ +{% docs total_staking_shares %} + +Total staking shares that comprise the staked balance of the pool. Number is presented raw and not decimal adjusted in the fact table. + +{% enddocs %} diff --git a/models/silver/curated/silver__pool_balance_daily.sql b/models/silver/curated/staking/silver__pool_balance_daily.sql similarity index 100% rename from models/silver/curated/silver__pool_balance_daily.sql rename to models/silver/curated/staking/silver__pool_balance_daily.sql diff --git a/models/silver/curated/silver__pool_balance_daily.yml b/models/silver/curated/staking/silver__pool_balance_daily.yml similarity index 100% rename from models/silver/curated/silver__pool_balance_daily.yml rename to models/silver/curated/staking/silver__pool_balance_daily.yml diff --git a/models/silver/curated/silver__pool_balances.sql b/models/silver/curated/staking/silver__pool_balances.sql similarity index 100% rename from models/silver/curated/silver__pool_balances.sql rename to models/silver/curated/staking/silver__pool_balances.sql diff --git a/models/silver/curated/silver__pool_balances.yml b/models/silver/curated/staking/silver__pool_balances.yml similarity index 100% rename from models/silver/curated/silver__pool_balances.yml rename to models/silver/curated/staking/silver__pool_balances.yml diff --git a/models/silver/curated/silver__pool_events.sql b/models/silver/curated/staking/silver__pool_events.sql similarity index 96% rename from models/silver/curated/silver__pool_events.sql rename to models/silver/curated/staking/silver__pool_events.sql index 2d1b36d..48cc0ed 100644 --- a/models/silver/curated/silver__pool_events.sql +++ b/models/silver/curated/staking/silver__pool_events.sql @@ -18,6 +18,7 @@ WITH receipts AS ( {% else %} {{ incremental_load_filter('_load_timestamp') }} {% endif %} + AND receipt_succeeded ), FINAL AS ( SELECT diff --git a/models/silver/curated/silver__pool_events.yml b/models/silver/curated/staking/silver__pool_events.yml similarity index 100% rename from models/silver/curated/silver__pool_events.yml rename to models/silver/curated/staking/silver__pool_events.yml diff --git a/models/silver/curated/silver__prices_oracle_s3.sql b/models/silver/curated/staking/silver__prices_oracle_s3.sql similarity index 100% rename from models/silver/curated/silver__prices_oracle_s3.sql rename to models/silver/curated/staking/silver__prices_oracle_s3.sql diff --git a/models/silver/curated/silver__prices_oracle_s3.yml b/models/silver/curated/staking/silver__prices_oracle_s3.yml similarity index 100% rename from models/silver/curated/silver__prices_oracle_s3.yml rename to models/silver/curated/staking/silver__prices_oracle_s3.yml diff --git a/models/silver/curated/silver__staking_actions_s3.sql b/models/silver/curated/staking/silver__staking_actions_s3.sql similarity index 100% rename from models/silver/curated/silver__staking_actions_s3.sql rename to models/silver/curated/staking/silver__staking_actions_s3.sql diff --git a/models/silver/curated/silver__staking_actions_s3.yml b/models/silver/curated/staking/silver__staking_actions_s3.yml similarity index 100% rename from models/silver/curated/silver__staking_actions_s3.yml rename to models/silver/curated/staking/silver__staking_actions_s3.yml diff --git a/models/silver/curated/silver__staking_actions_v2.sql b/models/silver/curated/staking/silver__staking_actions_v2.sql similarity index 100% rename from models/silver/curated/silver__staking_actions_v2.sql rename to models/silver/curated/staking/silver__staking_actions_v2.sql diff --git a/models/silver/curated/silver__staking_actions_v2.yml b/models/silver/curated/staking/silver__staking_actions_v2.yml similarity index 100% rename from models/silver/curated/silver__staking_actions_v2.yml rename to models/silver/curated/staking/silver__staking_actions_v2.yml diff --git a/models/silver/curated/staking/silver__staking_epochs.sql b/models/silver/curated/staking/silver__staking_epochs.sql new file mode 100644 index 0000000..14aaa60 --- /dev/null +++ b/models/silver/curated/staking/silver__staking_epochs.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'table', + unique_key = '_epoch_id', + incremental_strategy = 'delete+insert', + tags = ['curated'], + cluster_by = ['block_id'] +) }} + +WITH pool_events AS ( + + SELECT + * + FROM + {{ ref('silver__pool_events') }} + WHERE + {% if var("MANUAL_FIX") %} + {{ partition_load_manual('no_buffer') }} + {% else %} + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} + AND LOG LIKE 'Epoch%' +), +FINAL AS ( + SELECT + tx_hash, + block_id, + block_timestamp, + receiver_id AS pool_id, + SUBSTR(REGEXP_SUBSTR(LOG, 'Epoch [0-9]+'), 7) :: NUMBER AS epoch_number, + REGEXP_SUBSTR( + REGEXP_SUBSTR( + LOG, + 'Contract received total rewards of [0-9]+' + ), + '[0-9]+' + ) :: NUMBER AS reward_tokens, + REGEXP_SUBSTR( + REGEXP_SUBSTR( + LOG, + 'New total staked balance is [0-9]+' + ), + '[0-9]+' + ) :: NUMBER AS total_staked_balance, + REGEXP_SUBSTR( + REGEXP_SUBSTR( + LOG, + 'Total number of shares [0-9]+' + ), + '[0-9]+' + ) :: NUMBER AS total_staking_shares, + LOG, + _load_timestamp, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key(['pool_id', 'epoch_number']) }} AS _epoch_id + FROM + pool_events +) +SELECT + * +FROM + FINAL diff --git a/models/silver/curated/staking/silver__staking_epochs.yml b/models/silver/curated/staking/silver__staking_epochs.yml new file mode 100644 index 0000000..011c677 --- /dev/null +++ b/models/silver/curated/staking/silver__staking_epochs.yml @@ -0,0 +1,77 @@ +version: 2 + +models: + - name: silver__staking_epochs + description: |- + This table extracts epoch information for each pool from the logs of the reward receipts. + Note, it appears that the epoch log is only recorded during an act of withdrawal or deposit by a delegator, anbd not on each epoch. + + tests: + - dbt_utils.recency: + datepart: day + field: block_timestamp + interval: 1 + + columns: + - name: TX_HASH + description: "{{ doc('tx_hash')}}" + tests: + - not_null + + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + tests: + - not_null + + - name: POOL_ID + description: "{{ doc('pool_id')}}" + tests: + - not_null + + - name: EPOCH_NUMBER + description: "{{ doc('epoch_number')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: REWARD_TOKENS + description: "{{ doc('reward_tokens')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: TOTAL_STAKED_BALANCE + description: "{{ doc('total_staked_balance')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: TOTAL_STAKING_SHARES + description: "{{ doc('total_staking_shares')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: LOG + description: "{{ doc('log')}}" + + - name: _LOAD_TIMESTAMP + description: "{{ doc('_load_timestamp')}}" + + - name: _PARTITION_BY_BLOCK_NUMBER + description: "{{ doc('_partition_by_block_number')}}" + + - name: _EPOCH_ID + description: "{{ doc('_epoch_id')}}" + tests: + - unique + - not_null diff --git a/models/silver/curated/silver__staking_pools_s3.sql b/models/silver/curated/staking/silver__staking_pools_s3.sql similarity index 100% rename from models/silver/curated/silver__staking_pools_s3.sql rename to models/silver/curated/staking/silver__staking_pools_s3.sql diff --git a/models/silver/curated/silver__staking_pools_s3.yml b/models/silver/curated/staking/silver__staking_pools_s3.yml similarity index 100% rename from models/silver/curated/silver__staking_pools_s3.yml rename to models/silver/curated/staking/silver__staking_pools_s3.yml