Merge pull request #52 from FlipsideCrypto/An-2859/staking

An 2859/staking

models/core/core__dim_staking_actions.yml

@@ -3,60 +3,23 @@ version: 2
 models:
   - name: core__dim_staking_actions
     description: |-
-      Deprecating soon! The staking table is being rebuilt due to inaccuracies and will be re-released as as fact_ table. It is not advised you use this table anymore.
+      Deprecating soon! This table is being deprecated due to inaccuracies. The underlying model has been disabled and will not produce new data past early April 2023. Please use the new fact staking table(s).
 
     columns:
       - name: TX_HASH
         description: "{{ doc('tx_hash') }}"
-        tests:
-          - unique
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
 
       - name: BLOCK_TIMESTAMP
         description: "{{ doc('block_timestamp') }}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_NTZ
 
       - name: POOL_ADDRESS
         description: "{{ doc('staking_pool_address') }}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
 
       - name: TX_SIGNER
         description: "{{ doc('tx_signer') }}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
 
       - name: STAKE_AMOUNT
         description: "{{ doc('staking_stake_amount') }}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - NUMBER
 
       - name: ACTION
         description: "{{ doc('staking_action') }}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - VARCHAR
-          - dbt_expectations.expect_column_values_to_be_in_set:
-              value_set: ['Stake', 'Unstake']

models/core/core__fact_staking_actions.sql

@@ -0,0 +1,31 @@
+{{ config(
+    materialized = 'view',
+    secure = true,
+    meta={
+    'database_tags':{
+        'table': {
+            'PURPOSE': 'STAKING'
+            }
+        }
+    },
+    tags = ['core']
+) }}
+
+WITH staking_actions AS (
+
+    SELECT
+        tx_hash,
+        block_id,
+        block_timestamp,
+        receipt_object_id,
+        receiver_id AS address,
+        signer_id,
+        action,
+        amount_adj AS amount
+    FROM
+        {{ ref('silver__staking_actions_v2') }}
+)
+SELECT
+    *
+FROM
+    staking_actions

models/core/core__fact_staking_actions.yml

@@ -0,0 +1,33 @@
+version: 2
+
+models:
+  - name: core__fact_staking_actions
+    description: |-
+      An updated version of the staking actions table which looks at all logs, instead of just the first receipt.
+      There are four actions taken when staking: staking->deposit->unstaking->withdraw.
+      Note - in this core view the amount is decimal adjusted by 10^24.
+
+    columns:
+      - name: TX_HASH
+        description: "{{ doc('tx_hash') }}"
+
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+
+      - name: RECEIPT_OBJECT_ID
+        description: "{{ doc('receipt_object_id') }}"
+
+      - name: ADDRESS
+        description: "{{ doc('pool_address') }}"
+
+      - name: SIGNER_ID
+        description: "{{ doc('signer_id') }}"
+
+      - name: ACTION
+        description: "{{ doc('staking_action') }}"
+
+      - name: AMOUNT
+        description: "{{ doc('amount') }}"

models/core/core__fact_staking_pool_balances.sql

@@ -0,0 +1,29 @@
+{{ config(
+    materialized = 'view',
+    secure = true,
+    meta={
+    'database_tags':{
+        'table': {
+            'PURPOSE': 'STAKING'
+            }
+        }
+    },
+    tags = ['core']
+) }}
+
+WITH balance_changes AS (
+
+    SELECT
+        tx_hash,
+        block_id,
+        block_timestamp,
+        receipt_object_id,
+        receiver_id AS address,
+        amount_adj AS balance
+    FROM
+        {{ ref('silver__pool_balances') }}
+)
+SELECT
+    *
+FROM
+    balance_changes

models/core/core__fact_staking_pool_balances.yml

@@ -0,0 +1,27 @@
+version: 2
+
+models:
+  - name: core__fact_staking_pool_balances
+    description: |-
+      Staking pool balances as extracted from receipt logs when an individual makes a staking action.
+      To calculate balance at a point in time, isolate a single record for each pool. This table is transactional-based, so balances are updated with every staking event by users.
+      Note - the amount in balance is decimal adjusted by 10^24.
+
+    columns:
+      - name: TX_HASH
+        description: "{{ doc('tx_hash') }}"
+
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+
+      - name: RECEIPT_OBJECT_ID
+        description: "{{ doc('receipt_object_id') }}"
+
+      - name: ADDRESS
+        description: "{{ doc('pool_address') }}"
+
+      - name: BALANCE
+        description: "{{ doc('balance') }}"

models/descriptions/__overview__.md

@@ -21,7 +21,6 @@ There is more information on how to use dbt docs in the last section of this doc
 
 **Dimension Tables:**
 - [dim_address_labels](#!/model/model.near.core__dim_address_labels)
-- [dim_staking_actions](#!/model/model.near.core__dim_staking_actions)
 - [dim_staking_pools](#!/model/model.near.core__dim_staking_pools)
 - [dim_token_labels](#!/model/model.near.core__dim_token_labels)
 
@@ -34,6 +33,8 @@ There is more information on how to use dbt docs in the last section of this doc
 - [fact_token_metadata](#!/model/model.near.core__fact_token_metadata)
 - [fact_transactions](#!/model/model.near.core__fact_transactions)
 - [fact_transfers](#!/model/model.near.core__fact_transfers)
+- [fact_staking_actions](#!/model/model.near.core__fact_staking_actions)
+- [fact_staking_pool_balances](#!/model/model.near.core__fact_staking_pool_balances)
 
 
 **Convenience Tables:**
@@ -45,6 +46,8 @@ There is more information on how to use dbt docs in the last section of this doc
 - [fact_addkey_events](#!/model/model.near.social__fact_addkey_events)
 - [fact_decoded_actions](#!/model/model.near.social__fact_decoded_actions)
 - [fact_profile_changes](#!/model/model.near.social__fact_profile_changes)
+- [fact_posts](#!/model/model.near.social__fact_posts)
+- [fact_widget_deployments](#!/model/model.near.social__fact_widget_deployments)
 
 
 ### Beta Tables (`NEAR`.`BETA`.`<table_name>`)

models/descriptions/_log_signer_id_match.md

@@ -0,0 +1,5 @@
+{% docs _log_signer_id_match %}
+
+A boolean indicating whether the signer_id of the log matches the signer_id of the transaction. Internal / silver column only.
+
+{% enddocs %}

models/descriptions/amount_adj.md

@@ -0,0 +1,5 @@
+{% docs amount_adj %}
+
+A decimal adjusted amount (of tokens, price, etc.). 
+
+{% enddocs %}

models/descriptions/amount_raw.md

@@ -0,0 +1,5 @@
+{% docs amount_raw %}
+
+An unadjusted amount (of tokens, price, etc.) for the relevant record. This is the number as it appears on chain and is not decimal adjusted.
+
+{% enddocs %}

models/descriptions/balance.md

@@ -0,0 +1,5 @@
+{% docs balance %}
+
+The balance of a pool or address at a given block.
+
+{% enddocs %}

models/descriptions/log.md

@@ -0,0 +1,5 @@
+{% docs log %}
+
+A single log extracted from a receipt.
+
+{% enddocs %}

models/descriptions/log_signer_id.md

@@ -0,0 +1,5 @@
+{% docs log_signer_id %}
+
+The NEAR address indicated in the log.
+
+{% enddocs %}

models/descriptions/logs.md

@@ -1,5 +1,5 @@
 {% docs logs %}
 
-Logs (array) for this transaction.
+Logs (array) for this receipt.
 
 {% enddocs %}

models/descriptions/pool_address.md

@@ -0,0 +1,5 @@
+{% docs pool_address %}
+
+The NEAR address of the staking pool.
+
+{% enddocs %}

models/descriptions/staking_action.md

@@ -1,5 +1,5 @@
 {% docs staking_action %}
 
-The staking action performed in this transaction. Can be `"Stake"` or `"Unstake"`.
+The staking action performed in this transaction. Can be `"Stake"` or `"Unstake"` if in the deprecating `dim_staking_actions` table, or `staking`, `unstaking`, `deposited`, `withdrawing` if in the new `fact_staking_actions` table. These method names are taken directly from the log in which they occur.
 
 {% enddocs %}

models/silver/curated/silver__pool_balances.sql

@@ -0,0 +1,55 @@
+{{ config(
+    materialized = 'table',
+    unique_key = 'tx_hash',
+    incremental_strategy = 'delete+insert',
+    tags = ['curated'],
+    cluster_by = ['_partition_by_block_number', 'block_timestamp::date']
+) }}
+
+WITH pool_events AS (
+
+    SELECT
+        *
+    FROM
+        {{ ref('silver__pool_events') }}
+    WHERE
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
+),
+FINAL AS (
+    SELECT
+        tx_hash,
+        block_id,
+        block_timestamp,
+        receipt_object_id,
+        receiver_id,
+        signer_id,
+        LOG,
+        TO_NUMBER(
+            REGEXP_SUBSTR(
+                LOG,
+                'Contract total staked balance is (\\d+)',
+                1,
+                1,
+                'e',
+                1
+            )
+        ) AS amount_raw,
+        amount_raw / pow(
+            10,
+            24
+        ) AS amount_adj,
+        _load_timestamp,
+        _partition_by_block_number
+    FROM
+        pool_events
+    WHERE
+        LOG LIKE 'Contract total staked balance is%'
+)
+SELECT
+    *
+FROM
+    FINAL

models/silver/curated/silver__pool_balances.yml

@@ -0,0 +1,57 @@
+version: 2
+
+models:
+  - name: silver__pool_balances
+    description: |-
+      Staking pool balances as extracted from receipt logs when an individual makes a staking action.
+      To calculate balance at a point in time, isolate a single record for each pool. This table is transactional-based, so balances are updated with every staking event by users.
+
+    columns:
+      - name: tx_hash
+        description: "{{ doc('tx_hash') }}"
+
+      - name: block_id
+        description: "{{ doc('block_id') }}"
+
+      - name: block_timestamp
+        description: "{{ doc('block_timestamp') }}"
+
+      - name: receipt_object_id
+        description: "{{ doc('receipt_object_id') }}"
+
+      - name: receiver_id
+        description: "{{ doc('receiver_id') }}"
+
+      - name: signer_id
+        description: "{{ doc('signer_id') }}"
+
+      - name: LOG
+        description: "{{ doc('log') }}"
+
+      - name: amount_raw
+        description: "{{ doc('amount_raw') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+
+      - name: amount_adj
+        description: "{{ doc('amount_adj') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+                - DOUBLE
+
+      - name: _LOAD_TIMESTAMP
+        description: "{{ doc('_load_timestamp')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+
+      - name: _partition_by_block_number
+        description: "{{ doc('_partition_by_block_number') }}"

models/silver/curated/silver__pool_events.sql

@@ -0,0 +1,45 @@
+{{ config(
+    materialized = 'table',
+    unique_key = 'tx_hash',
+    incremental_strategy = 'delete+insert',
+    tags = ['curated'],
+    cluster_by = ['_partition_by_block_number', 'block_timestamp::date']
+) }}
+
+WITH receipts AS (
+
+    SELECT
+        *
+    FROM
+        {{ ref('silver__streamline_receipts_final') }}
+    WHERE
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
+),
+FINAL AS (
+    SELECT
+        tx_hash,
+        block_id,
+        block_timestamp,
+        receipt_object_id,
+        receiver_id,
+        signer_id,
+        status_value,
+        logs,
+        VALUE AS LOG,
+        _load_timestamp,
+        _partition_by_block_number
+    FROM
+        receipts,
+        LATERAL FLATTEN(logs)
+    WHERE
+        ARRAY_SIZE(logs) > 0
+        AND receiver_id ILIKE '%.pool%.near'
+)
+SELECT
+    *
+FROM
+    FINAL

models/silver/curated/silver__pool_events.yml

@@ -0,0 +1,73 @@
+version: 2
+
+models:
+  - name: silver__pool_events
+    description: |-
+      Flattened logs from receipt events where the recipient is a staking pool, for use in multiple downstream models.
+
+    columns:
+      - name: tx_hash
+        description: "{{ doc('tx_hash') }}"
+        tests:
+          - not_null:
+              where: _load_timestamp <= current_timestamp - interval '2 hours'
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - VARCHAR
+
+      - name: block_id
+        description: "{{ doc('block_id') }}"
+        tests:
+          - not_null
+
+      - name: block_timestamp
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+
+      - name: receipt_object_id
+        description: "{{ doc('receipt_object_id') }}"
+        tests:
+          - not_null
+
+      - name: receiver_id
+        description: "{{ doc('receiver_id') }}"
+        tests:
+          - not_null
+
+      - name: signer_id
+        description: "{{ doc('signer_id') }}"
+        tests:
+          - not_null
+
+      - name: status_value
+        description: "{{ doc('status_value') }}"
+        tests:
+          - not_null
+
+      - name: logs
+        description: "{{ doc('logs') }}"
+        tests:
+          - not_null
+
+      - name: LOG
+        description: "{{ doc('log') }}"
+        tests:
+          - not_null
+
+      - name: _LOAD_TIMESTAMP
+        description: "{{ doc('_load_timestamp')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+
+      - name: _partition_by_block_number
+        description: "{{ doc('_partition_by_block_number') }}"

models/silver/curated/silver__staking_actions_v2.sql

@@ -0,0 +1,88 @@
+{{ config(
+    materialized = 'table',
+    unique_key = 'tx_hash',
+    incremental_strategy = 'delete+insert',
+    tags = ['curated'],
+    cluster_by = ['_partition_by_block_number', 'block_timestamp::date']
+) }}
+
+WITH pool_events AS (
+
+    SELECT
+        *
+    FROM
+        {{ ref('silver__pool_events') }}
+    WHERE
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
+),
+staking_actions AS (
+    SELECT
+        tx_hash,
+        block_id,
+        block_timestamp,
+        receipt_object_id,
+        receiver_id,
+        signer_id,
+        status_value,
+        logs,
+        LOG,
+        SPLIT(
+            LOG,
+            ' '
+        ) AS log_parts,
+        SPLIT(
+            log_parts [0] :: STRING,
+            '@'
+        ) [1] :: STRING AS log_signer_id,
+        log_parts [1] :: STRING AS action,
+        SPLIT(
+            log_parts [2] :: STRING,
+            '.'
+        ) [0] :: NUMBER AS amount_raw,
+        amount_raw :: FLOAT / pow(
+            10,
+            24
+        ) AS amount_adj,
+        LENGTH(
+            amount_raw :: STRING
+        ) AS decimals,
+        signer_id = log_signer_id AS _log_signer_id_match,
+        _load_timestamp,
+        _partition_by_block_number
+    FROM
+        pool_events
+    WHERE
+        TRUE
+        AND receiver_id != signer_id
+        AND LOG LIKE '@%'
+),
+FINAL AS (
+    SELECT
+        tx_hash,
+        block_id,
+        block_timestamp,
+        receipt_object_id,
+        receiver_id,
+        signer_id,
+        status_value,
+        logs,
+        LOG,
+        log_signer_id,
+        action,
+        amount_raw,
+        amount_adj,
+        decimals,
+        _log_signer_id_match,
+        _load_timestamp,
+        _partition_by_block_number
+    FROM
+        staking_actions
+)
+SELECT
+    *
+FROM
+    FINAL

models/silver/curated/silver__staking_actions_v2.yml

@@ -0,0 +1,96 @@
+version: 2
+
+models:
+  - name: silver__staking_actions_v2
+    description: |-
+      An updated version of the staking actions table which looks at all logs, instead of just the first receipt.
+      There are four actions taken when staking: staking->deposit->unstaking->withdraw.
+
+    columns:
+      - name: tx_hash
+        description: "{{ doc('tx_hash') }}"
+
+      - name: block_id
+        description: "{{ doc('block_id') }}"
+
+      - name: block_timestamp
+        description: "{{ doc('block_timestamp') }}"
+
+      - name: receipt_object_id
+        description: "{{ doc('receipt_object_id') }}"
+
+      - name: receiver_id
+        description: "{{ doc('receiver_id') }}"
+
+      - name: signer_id
+        description: "{{ doc('signer_id') }}"
+
+      - name: status_value
+        description: "{{ doc('status_value') }}"
+
+      - name: logs
+        description: "{{ doc('logs') }}"
+
+      - name: LOG
+        description: "{{ doc('log') }}"
+
+      - name: log_signer_id
+        description: "{{ doc('log_signer_id') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - VARCHAR
+
+      - name: action
+        description: "{{ doc('staking_action') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - VARCHAR
+          - accepted_values:
+              values:
+                - deposited
+                - withdrawing
+                - staking
+                - unstaking
+
+      - name: amount_raw
+        description: "{{ doc('amount_raw') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+
+      - name: amount_adj
+        description: "{{ doc('amount_adj') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+                - DOUBLE
+
+      - name: decimals
+        description: "{{ doc('decimals') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+
+      - name: _log_signer_id_match
+        description: "{{ doc('_log_signer_id_match') }}"
+
+      - name: _LOAD_TIMESTAMP
+        description: "{{ doc('_load_timestamp')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+
+      - name: _partition_by_block_number
+        description: "{{ doc('_partition_by_block_number') }}"