diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index bb47992..40f431e 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -4,8 +4,8 @@ run-name: dbt_test_scheduled on: workflow_dispatch: schedule: - # Run at 1600 UTC daily (see https://crontab.guru) - - cron: '0 16 * * *' + # Run at 1300 UTC daily (see https://crontab.guru) + - cron: '0 13 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" diff --git a/macros/partition_batch_load.sql b/macros/partition_batch_load.sql index 2094437..a8e0468 100644 --- a/macros/partition_batch_load.sql +++ b/macros/partition_batch_load.sql @@ -10,7 +10,7 @@ _partition_by_block_number BETWEEN ( AND ( ( SELECT - MAX(_partition_by_block_number) + MAX(_partition_by_block_number) - 10000 FROM {{ this }} ) + {{ batch_size }} diff --git a/models/descriptions/_epoch_id.md b/models/descriptions/_epoch_id.md new file mode 100644 index 0000000..58f3ffc --- /dev/null +++ b/models/descriptions/_epoch_id.md @@ -0,0 +1,5 @@ +{% docs _epoch_id %} + +An internal identifier for the epoch. This is a hashed representation of the pool_id and epoch number. + +{% enddocs %} diff --git a/models/descriptions/epoch_number.md b/models/descriptions/epoch_number.md new file mode 100644 index 0000000..ce0f8b8 --- /dev/null +++ b/models/descriptions/epoch_number.md @@ -0,0 +1,5 @@ +{% docs epoch_number %} + +The epoch number for this staking reward payout. Epoch increment in order every 43,200 blocks. + +{% enddocs %} diff --git a/models/descriptions/reward_tokens.md b/models/descriptions/reward_tokens.md new file mode 100644 index 0000000..447600b --- /dev/null +++ b/models/descriptions/reward_tokens.md @@ -0,0 +1,5 @@ +{% docs reward_tokens %} + +The total amount of tokens rewarded to the validator for this staking reward payout. Number is presented raw and not decimal adjusted in the fact table. + +{% enddocs %} diff --git a/models/descriptions/total_staked_balance.md b/models/descriptions/total_staked_balance.md new file mode 100644 index 0000000..13e2a0b --- /dev/null +++ b/models/descriptions/total_staked_balance.md @@ -0,0 +1,5 @@ +{% docs total_staked_balance %} + +The total staked NEAR at the time of the reward payout. Number is presented raw and not decimal adjusted in the fact table. + +{% enddocs %} diff --git a/models/descriptions/total_staking_shares.md b/models/descriptions/total_staking_shares.md new file mode 100644 index 0000000..260ca25 --- /dev/null +++ b/models/descriptions/total_staking_shares.md @@ -0,0 +1,5 @@ +{% docs total_staking_shares %} + +Total staking shares that comprise the staked balance of the pool. Number is presented raw and not decimal adjusted in the fact table. + +{% enddocs %} diff --git a/models/gold/core__dim_staking_actions.sql b/models/gold/core__dim_staking_actions.sql deleted file mode 100644 index 7818051..0000000 --- a/models/gold/core__dim_staking_actions.sql +++ /dev/null @@ -1,27 +0,0 @@ -{{ config( - materialized = 'view', - secure = true, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'STAKING' - } - } - }, - tags = ['core'] -) }} - -with staking_actions as ( - select - * - from {{ ref('silver__staking_actions_s3') }} -) - -select - tx_hash, - block_timestamp, - pool_address, - tx_signer, - stake_amount, - action - from staking_actions diff --git a/models/gold/core__dim_staking_actions.yml b/models/gold/core__dim_staking_actions.yml deleted file mode 100644 index 20258f4..0000000 --- a/models/gold/core__dim_staking_actions.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 2 - -models: - - name: core__dim_staking_actions - description: |- - Deprecating soon! This table is being deprecated due to inaccuracies. The underlying model has been disabled and will not produce new data past early April 2023. Please use the new fact staking table(s). - - columns: - - name: TX_HASH - description: "{{ doc('tx_hash') }}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - - - name: POOL_ADDRESS - description: "{{ doc('staking_pool_address') }}" - - - name: TX_SIGNER - description: "{{ doc('tx_signer') }}" - - - name: STAKE_AMOUNT - description: "{{ doc('staking_stake_amount') }}" - - - name: ACTION - description: "{{ doc('staking_action') }}" diff --git a/models/gold/core__ez_nft_mints.sql b/models/gold/core__ez_nft_mints.sql deleted file mode 100644 index 3e4379a..0000000 --- a/models/gold/core__ez_nft_mints.sql +++ /dev/null @@ -1,33 +0,0 @@ -{{ config( - materialized = 'view', - secure = true, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'NFT' - } - } - }, - tags = ['core'] -) }} - -with mints as ( - select * - from {{ ref('silver__nft_mints_s3') }} -) - -select - action_id, - tx_hash, - block_id, - block_timestamp, - method_name, - tx_signer, - tx_receiver, - project_name, - token_id, - nft_id, - nft_address, - network_fee, - tx_status -from mints diff --git a/models/gold/core__ez_nft_mints.yml b/models/gold/core__ez_nft_mints.yml deleted file mode 100644 index f78627e..0000000 --- a/models/gold/core__ez_nft_mints.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: 2 - -models: - - name: core__ez_nft_mints - description: |- - Deprecating soon! Please use the new fact_nft_mints table. - - columns: - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: ACTION_ID - description: "{{ doc('action_id')}}" - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - - - name: TX_SIGNER - description: "{{ doc('tx_signer')}}" - - - name: TX_RECEIVER - description: "{{ doc('tx_receiver')}}" - - - name: TX_STATUS - description: "{{ doc('tx_status')}}" - - - name: PROJECT_NAME - description: "{{ doc('nft_project_name') }}" - - - name: NFT_ADDRESS - description: "{{ doc('nft_address') }}" - - - name: NFT_ID - description: "{{ doc('nft_id') }}" - - - name: TOKEN_ID - description: "{{ doc ('nft_token_id') }}" diff --git a/models/gold/core__ez_usn_supply.sql b/models/gold/core__ez_usn_supply.sql deleted file mode 100644 index dff0441..0000000 --- a/models/gold/core__ez_usn_supply.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - materialized = 'view', - secure = true, - tags = ['core'] -) }} - -WITH transactions AS ( - - SELECT - * - FROM - {{ ref('silver__usn_supply_s3') }} -) -SELECT - block_timestamp, - block_id, - tx_hash, - status, - event, - from_address, - to_address, - amount -FROM - transactions diff --git a/models/gold/core__ez_usn_supply.yml b/models/gold/core__ez_usn_supply.yml deleted file mode 100644 index e3737d1..0000000 --- a/models/gold/core__ez_usn_supply.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: 2 - -models: - - name: core__ez_usn_supply - description: |- - Deprecating soon! - - columns: - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: STATUS - description: "{{ doc('tx_status')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: FROM_ADDRESS - description: "{{ doc('from_address')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TO_ADDRESS - description: "{{ doc('to_address')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: EVENT - description: "{{ doc('event')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: AMOUNT - description: "{{ doc('amount')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT \ No newline at end of file diff --git a/models/silver/curated/silver__prices_oracle_s3.yml b/models/silver/curated/silver__prices_oracle_s3.yml index 2a65547..64ba0c9 100644 --- a/models/silver/curated/silver__prices_oracle_s3.yml +++ b/models/silver/curated/silver__prices_oracle_s3.yml @@ -24,7 +24,9 @@ models: - name: TX_HASH description: "{{ doc('tx_hash')}}" tests: - - not_null + - not_null: + where: _load_timestamp <= current_timestamp - interval '2 hours' + - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING diff --git a/models/silver/curated/silver__pool_balance_daily.sql b/models/silver/curated/staking/silver__pool_balance_daily.sql similarity index 100% rename from models/silver/curated/silver__pool_balance_daily.sql rename to models/silver/curated/staking/silver__pool_balance_daily.sql diff --git a/models/silver/curated/silver__pool_balance_daily.yml b/models/silver/curated/staking/silver__pool_balance_daily.yml similarity index 100% rename from models/silver/curated/silver__pool_balance_daily.yml rename to models/silver/curated/staking/silver__pool_balance_daily.yml diff --git a/models/silver/curated/silver__pool_balances.sql b/models/silver/curated/staking/silver__pool_balances.sql similarity index 100% rename from models/silver/curated/silver__pool_balances.sql rename to models/silver/curated/staking/silver__pool_balances.sql diff --git a/models/silver/curated/silver__pool_balances.yml b/models/silver/curated/staking/silver__pool_balances.yml similarity index 100% rename from models/silver/curated/silver__pool_balances.yml rename to models/silver/curated/staking/silver__pool_balances.yml diff --git a/models/silver/curated/silver__pool_events.sql b/models/silver/curated/staking/silver__pool_events.sql similarity index 96% rename from models/silver/curated/silver__pool_events.sql rename to models/silver/curated/staking/silver__pool_events.sql index 2d1b36d..48cc0ed 100644 --- a/models/silver/curated/silver__pool_events.sql +++ b/models/silver/curated/staking/silver__pool_events.sql @@ -18,6 +18,7 @@ WITH receipts AS ( {% else %} {{ incremental_load_filter('_load_timestamp') }} {% endif %} + AND receipt_succeeded ), FINAL AS ( SELECT diff --git a/models/silver/curated/silver__pool_events.yml b/models/silver/curated/staking/silver__pool_events.yml similarity index 100% rename from models/silver/curated/silver__pool_events.yml rename to models/silver/curated/staking/silver__pool_events.yml diff --git a/models/silver/curated/silver__staking_actions_s3.sql b/models/silver/curated/staking/silver__staking_actions_s3.sql similarity index 100% rename from models/silver/curated/silver__staking_actions_s3.sql rename to models/silver/curated/staking/silver__staking_actions_s3.sql diff --git a/models/silver/curated/silver__staking_actions_s3.yml b/models/silver/curated/staking/silver__staking_actions_s3.yml similarity index 100% rename from models/silver/curated/silver__staking_actions_s3.yml rename to models/silver/curated/staking/silver__staking_actions_s3.yml diff --git a/models/silver/curated/silver__staking_actions_v2.sql b/models/silver/curated/staking/silver__staking_actions_v2.sql similarity index 100% rename from models/silver/curated/silver__staking_actions_v2.sql rename to models/silver/curated/staking/silver__staking_actions_v2.sql diff --git a/models/silver/curated/silver__staking_actions_v2.yml b/models/silver/curated/staking/silver__staking_actions_v2.yml similarity index 100% rename from models/silver/curated/silver__staking_actions_v2.yml rename to models/silver/curated/staking/silver__staking_actions_v2.yml diff --git a/models/silver/curated/staking/silver__staking_epochs.sql b/models/silver/curated/staking/silver__staking_epochs.sql new file mode 100644 index 0000000..14aaa60 --- /dev/null +++ b/models/silver/curated/staking/silver__staking_epochs.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'table', + unique_key = '_epoch_id', + incremental_strategy = 'delete+insert', + tags = ['curated'], + cluster_by = ['block_id'] +) }} + +WITH pool_events AS ( + + SELECT + * + FROM + {{ ref('silver__pool_events') }} + WHERE + {% if var("MANUAL_FIX") %} + {{ partition_load_manual('no_buffer') }} + {% else %} + {{ incremental_load_filter('_load_timestamp') }} + {% endif %} + AND LOG LIKE 'Epoch%' +), +FINAL AS ( + SELECT + tx_hash, + block_id, + block_timestamp, + receiver_id AS pool_id, + SUBSTR(REGEXP_SUBSTR(LOG, 'Epoch [0-9]+'), 7) :: NUMBER AS epoch_number, + REGEXP_SUBSTR( + REGEXP_SUBSTR( + LOG, + 'Contract received total rewards of [0-9]+' + ), + '[0-9]+' + ) :: NUMBER AS reward_tokens, + REGEXP_SUBSTR( + REGEXP_SUBSTR( + LOG, + 'New total staked balance is [0-9]+' + ), + '[0-9]+' + ) :: NUMBER AS total_staked_balance, + REGEXP_SUBSTR( + REGEXP_SUBSTR( + LOG, + 'Total number of shares [0-9]+' + ), + '[0-9]+' + ) :: NUMBER AS total_staking_shares, + LOG, + _load_timestamp, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key(['pool_id', 'epoch_number']) }} AS _epoch_id + FROM + pool_events +) +SELECT + * +FROM + FINAL diff --git a/models/silver/curated/staking/silver__staking_epochs.yml b/models/silver/curated/staking/silver__staking_epochs.yml new file mode 100644 index 0000000..011c677 --- /dev/null +++ b/models/silver/curated/staking/silver__staking_epochs.yml @@ -0,0 +1,77 @@ +version: 2 + +models: + - name: silver__staking_epochs + description: |- + This table extracts epoch information for each pool from the logs of the reward receipts. + Note, it appears that the epoch log is only recorded during an act of withdrawal or deposit by a delegator, anbd not on each epoch. + + tests: + - dbt_utils.recency: + datepart: day + field: block_timestamp + interval: 1 + + columns: + - name: TX_HASH + description: "{{ doc('tx_hash')}}" + tests: + - not_null + + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + tests: + - not_null + + - name: POOL_ID + description: "{{ doc('pool_id')}}" + tests: + - not_null + + - name: EPOCH_NUMBER + description: "{{ doc('epoch_number')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: REWARD_TOKENS + description: "{{ doc('reward_tokens')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: TOTAL_STAKED_BALANCE + description: "{{ doc('total_staked_balance')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: TOTAL_STAKING_SHARES + description: "{{ doc('total_staking_shares')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: LOG + description: "{{ doc('log')}}" + + - name: _LOAD_TIMESTAMP + description: "{{ doc('_load_timestamp')}}" + + - name: _PARTITION_BY_BLOCK_NUMBER + description: "{{ doc('_partition_by_block_number')}}" + + - name: _EPOCH_ID + description: "{{ doc('_epoch_id')}}" + tests: + - unique + - not_null diff --git a/models/silver/curated/silver__staking_pools_s3.sql b/models/silver/curated/staking/silver__staking_pools_s3.sql similarity index 100% rename from models/silver/curated/silver__staking_pools_s3.sql rename to models/silver/curated/staking/silver__staking_pools_s3.sql diff --git a/models/silver/curated/silver__staking_pools_s3.yml b/models/silver/curated/staking/silver__staking_pools_s3.yml similarity index 100% rename from models/silver/curated/silver__staking_pools_s3.yml rename to models/silver/curated/staking/silver__staking_pools_s3.yml diff --git a/models/silver/social/silver_social__profile_changes.sql b/models/silver/social/silver_social__profile_changes.sql index d43d4ce..ba378f8 100644 --- a/models/silver/social/silver_social__profile_changes.sql +++ b/models/silver/social/silver_social__profile_changes.sql @@ -17,8 +17,7 @@ WITH decoded_actions AS ( {% else %} {{ incremental_load_filter('_load_timestamp') }} {% endif %} - AND - node = 'profile' + AND node = 'profile' ), flatten_profile_json AS ( SELECT diff --git a/models/silver/social/silver_social__profile_changes.yml b/models/silver/social/silver_social__profile_changes.yml index 53c2b57..c2f61a1 100644 --- a/models/silver/social/silver_social__profile_changes.yml +++ b/models/silver/social/silver_social__profile_changes.yml @@ -38,7 +38,8 @@ models: - name: profile_data description: "{{ doc('profile_data')}}" tests: - - not_null + - not_null: + where: profile_section not in ('horizon_tnc', 'team') - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - varchar