merge main

.github/workflows/dbt_test.yml

@@ -4,8 +4,8 @@ run-name: dbt_test_scheduled
 on:
   workflow_dispatch:
   schedule:
-    # Run at 1600 UTC daily (see https://crontab.guru)
-    - cron: '0 16 * * *'
+    # Run at 1300 UTC daily (see https://crontab.guru)
+    - cron: '0 13 * * *'
 env:
   USE_VARS: "${{ vars.USE_VARS }}"
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

macros/partition_batch_load.sql

@@ -10,7 +10,7 @@ _partition_by_block_number BETWEEN (
 AND (
     (
         SELECT
-            MAX(_partition_by_block_number)
+            MAX(_partition_by_block_number) - 10000
         FROM
             {{ this }}
     ) + {{ batch_size }}

models/descriptions/_epoch_id.md

@@ -0,0 +1,5 @@
+{% docs _epoch_id %}
+
+An internal identifier for the epoch. This is a hashed representation of the pool_id and epoch number.
+
+{% enddocs %}

models/descriptions/epoch_number.md

@@ -0,0 +1,5 @@
+{% docs epoch_number %}
+
+The epoch number for this staking reward payout. Epoch increment in order every 43,200 blocks.
+
+{% enddocs %}

models/descriptions/reward_tokens.md

@@ -0,0 +1,5 @@
+{% docs reward_tokens %}
+
+The total amount of tokens rewarded to the validator for this staking reward payout. Number is presented raw and not decimal adjusted in the fact table.
+
+{% enddocs %}

models/descriptions/total_staked_balance.md

@@ -0,0 +1,5 @@
+{% docs total_staked_balance %}
+
+The total staked NEAR at the time of the reward payout. Number is presented raw and not decimal adjusted in the fact table.
+
+{% enddocs %}

models/descriptions/total_staking_shares.md

@@ -0,0 +1,5 @@
+{% docs total_staking_shares %}
+
+Total staking shares that comprise the staked balance of the pool. Number is presented raw and not decimal adjusted in the fact table.
+
+{% enddocs %}

models/gold/core__dim_staking_actions.sql

@@ -1,27 +0,0 @@
-{{ config(
-    materialized = 'view',
-    secure = true,
-    meta={
-    'database_tags':{
-        'table': {
-            'PURPOSE': 'STAKING'
-            }
-        }
-    },
-    tags = ['core']
-) }}
-
-with staking_actions as (
-    select
-        *
-    from {{ ref('silver__staking_actions_s3') }}
-)
-
-select 
-    tx_hash,
-    block_timestamp,
-    pool_address,
-    tx_signer,
-    stake_amount,
-    action
- from staking_actions

models/gold/core__dim_staking_actions.yml

@@ -1,25 +0,0 @@
-version: 2
-
-models:
-  - name: core__dim_staking_actions
-    description: |-
-      Deprecating soon! This table is being deprecated due to inaccuracies. The underlying model has been disabled and will not produce new data past early April 2023. Please use the new fact staking table(s).
-
-    columns:
-      - name: TX_HASH
-        description: "{{ doc('tx_hash') }}"
-
-      - name: BLOCK_TIMESTAMP
-        description: "{{ doc('block_timestamp') }}"
-
-      - name: POOL_ADDRESS
-        description: "{{ doc('staking_pool_address') }}"
-
-      - name: TX_SIGNER
-        description: "{{ doc('tx_signer') }}"
-
-      - name: STAKE_AMOUNT
-        description: "{{ doc('staking_stake_amount') }}"
-
-      - name: ACTION
-        description: "{{ doc('staking_action') }}"

models/gold/core__ez_nft_mints.sql

@@ -1,33 +0,0 @@
-{{ config(
-    materialized = 'view',
-    secure = true,
-    meta={
-    'database_tags':{
-        'table': {
-            'PURPOSE': 'NFT'
-            }
-        }
-    },
-    tags = ['core']
-) }}
-
-with mints as (
-    select * 
-    from {{ ref('silver__nft_mints_s3') }}
-)
-
-select
-    action_id,
-    tx_hash,
-    block_id,
-    block_timestamp,
-    method_name,
-    tx_signer,
-    tx_receiver,
-    project_name,
-    token_id,
-    nft_id,
-    nft_address,
-    network_fee,
-    tx_status
-from mints

models/gold/core__ez_nft_mints.yml

@@ -1,40 +0,0 @@
-version: 2
-
-models:
-  - name: core__ez_nft_mints
-    description: |-
-      Deprecating soon! Please use the new fact_nft_mints table.
-
-    columns:    
-      - name: BLOCK_TIMESTAMP
-        description: "{{ doc('block_timestamp')}}"
-
-      - name: BLOCK_ID
-        description: "{{ doc('block_id')}}"
-
-      - name: ACTION_ID
-        description: "{{ doc('action_id')}}"
-
-      - name: TX_HASH
-        description: "{{ doc('tx_hash')}}"
-
-      - name: TX_SIGNER
-        description: "{{ doc('tx_signer')}}"
-
-      - name: TX_RECEIVER
-        description: "{{ doc('tx_receiver')}}"
-
-      - name: TX_STATUS
-        description: "{{ doc('tx_status')}}"
-
-      - name: PROJECT_NAME
-        description: "{{ doc('nft_project_name') }}"
-
-      - name: NFT_ADDRESS
-        description: "{{ doc('nft_address') }}"
-
-      - name: NFT_ID
-        description: "{{ doc('nft_id') }}"
-
-      - name: TOKEN_ID
-        description: "{{ doc ('nft_token_id') }}"

models/gold/core__ez_usn_supply.sql

@@ -1,24 +0,0 @@
-{{ config(
-    materialized = 'view',
-    secure = true,
-    tags = ['core']
-) }}
-
-WITH transactions AS (
-
-    SELECT
-        *
-    FROM
-        {{ ref('silver__usn_supply_s3') }}
-)
-SELECT
-    block_timestamp,
-    block_id,
-    tx_hash,
-    status,
-    event,
-    from_address,
-    to_address,
-    amount
-FROM
-    transactions

models/gold/core__ez_usn_supply.yml

@@ -1,77 +0,0 @@
-version: 2
-
-models:
-  - name: core__ez_usn_supply
-    description: |-
-      Deprecating soon! 
-
-    columns:    
-      - name: BLOCK_TIMESTAMP
-        description: "{{ doc('block_timestamp')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_NTZ
-
-      - name: BLOCK_ID
-        description: "{{ doc('block_id')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT
-
-      - name: STATUS
-        description: "{{ doc('tx_status')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
-
-      - name: TX_HASH
-        description: "{{ doc('tx_hash')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
-
-      - name: FROM_ADDRESS
-        description: "{{ doc('from_address')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
-
-      - name: TO_ADDRESS
-        description: "{{ doc('to_address')}}"
-        tests:
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
-      
-      - name: EVENT
-        description: "{{ doc('event')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
-
-      - name: AMOUNT
-        description: "{{ doc('amount')}}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
-                - FLOAT

models/silver/curated/silver__prices_oracle_s3.yml

@@ -24,7 +24,9 @@ models:
       - name: TX_HASH
         description: "{{ doc('tx_hash')}}"
         tests:
-          - not_null
+          - not_null:
+              where: _load_timestamp <= current_timestamp - interval '2 hours'
+              
           - dbt_expectations.expect_column_values_to_be_in_type_list:
               column_type_list:
                 - STRING

models/silver/curated/staking/silver__pool_events.sql

@@ -18,6 +18,7 @@ WITH receipts AS (
         {% else %}
             {{ incremental_load_filter('_load_timestamp') }}
         {% endif %}
+        AND receipt_succeeded
 ),
 FINAL AS (
     SELECT

models/silver/curated/staking/silver__staking_epochs.sql

@@ -0,0 +1,61 @@
+{{ config(
+    materialized = 'table',
+    unique_key = '_epoch_id',
+    incremental_strategy = 'delete+insert',
+    tags = ['curated'],
+    cluster_by = ['block_id']
+) }}
+
+WITH pool_events AS (
+
+    SELECT
+        *
+    FROM
+        {{ ref('silver__pool_events') }}
+    WHERE
+        {% if var("MANUAL_FIX") %}
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
+        AND LOG LIKE 'Epoch%'
+),
+FINAL AS (
+    SELECT
+        tx_hash,
+        block_id,
+        block_timestamp,
+        receiver_id AS pool_id,
+        SUBSTR(REGEXP_SUBSTR(LOG, 'Epoch [0-9]+'), 7) :: NUMBER AS epoch_number,
+        REGEXP_SUBSTR(
+            REGEXP_SUBSTR(
+                LOG,
+                'Contract received total rewards of [0-9]+'
+            ),
+            '[0-9]+'
+        ) :: NUMBER AS reward_tokens,
+        REGEXP_SUBSTR(
+            REGEXP_SUBSTR(
+                LOG,
+                'New total staked balance is [0-9]+'
+            ),
+            '[0-9]+'
+        ) :: NUMBER AS total_staked_balance,
+        REGEXP_SUBSTR(
+            REGEXP_SUBSTR(
+                LOG,
+                'Total number of shares [0-9]+'
+            ),
+            '[0-9]+'
+        ) :: NUMBER AS total_staking_shares,
+        LOG,
+        _load_timestamp,
+        _partition_by_block_number,
+        {{ dbt_utils.generate_surrogate_key(['pool_id', 'epoch_number']) }} AS _epoch_id
+    FROM
+        pool_events
+)
+SELECT
+    *
+FROM
+    FINAL

models/silver/curated/staking/silver__staking_epochs.yml

@@ -0,0 +1,77 @@
+version: 2
+
+models:
+  - name: silver__staking_epochs
+    description: |-
+      This table extracts epoch information for each pool from the logs of the reward receipts. 
+      Note, it appears that the epoch log is only recorded during an act of withdrawal or deposit by a delegator, anbd not on each epoch.
+
+    tests:
+        - dbt_utils.recency:
+            datepart: day
+            field: block_timestamp
+            interval: 1
+
+    columns:
+      - name: TX_HASH
+        description: "{{ doc('tx_hash')}}"
+        tests:
+          - not_null
+
+      - name: BLOCK_ID
+        description: "{{ doc('block_id')}}"
+        tests:
+          - not_null
+
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp')}}"
+        tests:
+          - not_null
+
+      - name: POOL_ID
+        description: "{{ doc('pool_id')}}"
+        tests:
+          - not_null
+
+      - name: EPOCH_NUMBER
+        description: "{{ doc('epoch_number')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: NUMBER
+
+      - name: REWARD_TOKENS
+        description: "{{ doc('reward_tokens')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: NUMBER
+
+      - name: TOTAL_STAKED_BALANCE
+        description: "{{ doc('total_staked_balance')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: NUMBER
+
+      - name: TOTAL_STAKING_SHARES
+        description: "{{ doc('total_staking_shares')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: NUMBER
+
+      - name: LOG
+        description: "{{ doc('log')}}"
+
+      - name: _LOAD_TIMESTAMP
+        description: "{{ doc('_load_timestamp')}}"
+
+      - name: _PARTITION_BY_BLOCK_NUMBER
+        description: "{{ doc('_partition_by_block_number')}}"
+
+      - name: _EPOCH_ID
+        description: "{{ doc('_epoch_id')}}"
+        tests:
+          - unique
+          - not_null

models/silver/social/silver_social__profile_changes.sql

@@ -17,8 +17,7 @@ WITH decoded_actions AS (
         {% else %}
             {{ incremental_load_filter('_load_timestamp') }}
         {% endif %}
-    AND
-        node = 'profile'
+        AND node = 'profile'
 ),
 flatten_profile_json AS (
     SELECT

models/silver/social/silver_social__profile_changes.yml

@@ -38,7 +38,8 @@ models:
       - name: profile_data
         description: "{{ doc('profile_data')}}"
         tests:
-          - not_null
+          - not_null:
+              where: profile_section not in ('horizon_tnc', 'team')
           - dbt_expectations.expect_column_values_to_be_in_type_list:
               column_type_list:
                 - varchar