From 64063a7dffdeca39d6f59f0a28113704646111a2 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Wed, 23 Aug 2023 11:38:04 +0200 Subject: [PATCH] docs, new views, crosschain's table update --- models/bronze/bronze__address_labels.sql | 2 +- models/descriptions/__overview__.md | 90 ++++++++++----- models/gold/defi/core__ez_dex_swaps.yml | 2 +- models/gold/defi/defi__ez_dex_swaps.sql | 73 ++++++++++++ models/gold/defi/defi__ez_dex_swaps.yml | 109 ++++++++++++++++++ .../governance/core__dim_staking_pools.yml | 4 +- .../governance/core__fact_lockup_actions.yml | 15 ++- .../governance/core__fact_staking_actions.yml | 4 +- .../core__fact_staking_pool_balances.yml | 4 +- ...core__fact_staking_pool_daily_balances.yml | 2 +- .../governance/gov__dim_staking_pools.sql | 29 +++++ .../governance/gov__dim_staking_pools.yml | 63 ++++++++++ .../governance/gov__fact_lockup_actions.sql | 35 ++++++ .../governance/gov__fact_lockup_actions.yml | 71 ++++++++++++ .../governance/gov__fact_staking_actions.sql | 31 +++++ .../governance/gov__fact_staking_actions.yml | 33 ++++++ .../gov__fact_staking_pool_balances.sql | 29 +++++ .../gov__fact_staking_pool_balances.yml | 27 +++++ .../gov__fact_staking_pool_daily_balances.sql | 26 +++++ .../gov__fact_staking_pool_daily_balances.yml | 16 +++ models/gold/nft/core__fact_nft_mints.yml | 14 +-- models/gold/nft/nft__fact_nft_mints.sql | 41 +++++++ models/gold/nft/nft__fact_nft_mints.yml | 103 +++++++++++++++++ models/gold/prices/core__fact_prices.yml | 2 +- models/gold/prices/price__fact_prices.sql | 36 ++++++ models/gold/prices/price__fact_prices.yml | 71 ++++++++++++ models/sources.yml | 2 +- requirements.txt | 1 + 28 files changed, 876 insertions(+), 59 deletions(-) create mode 100644 models/gold/defi/defi__ez_dex_swaps.sql create mode 100644 models/gold/defi/defi__ez_dex_swaps.yml create mode 100644 models/gold/governance/gov__dim_staking_pools.sql create mode 100644 models/gold/governance/gov__dim_staking_pools.yml create mode 100644 models/gold/governance/gov__fact_lockup_actions.sql create mode 100644 models/gold/governance/gov__fact_lockup_actions.yml create mode 100644 models/gold/governance/gov__fact_staking_actions.sql create mode 100644 models/gold/governance/gov__fact_staking_actions.yml create mode 100644 models/gold/governance/gov__fact_staking_pool_balances.sql create mode 100644 models/gold/governance/gov__fact_staking_pool_balances.yml create mode 100644 models/gold/governance/gov__fact_staking_pool_daily_balances.sql create mode 100644 models/gold/governance/gov__fact_staking_pool_daily_balances.yml create mode 100644 models/gold/nft/nft__fact_nft_mints.sql create mode 100644 models/gold/nft/nft__fact_nft_mints.yml create mode 100644 models/gold/prices/price__fact_prices.sql create mode 100644 models/gold/prices/price__fact_prices.yml create mode 100644 requirements.txt diff --git a/models/bronze/bronze__address_labels.sql b/models/bronze/bronze__address_labels.sql index 5c7677f..0ff5f92 100644 --- a/models/bronze/bronze__address_labels.sql +++ b/models/bronze/bronze__address_labels.sql @@ -16,7 +16,7 @@ SELECT FROM {{ source( 'crosschain', - 'address_labels' + 'dim_labels' ) }} WHERE blockchain = 'near' diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index b330c14..efe031d 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -3,10 +3,12 @@ # Welcome to the Flipside Crypto NEAR Models Documentation ## **What does this documentation cover?** + The documentation included here details the design of the NEAR - tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/near-models/) +tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/near-models/) ## **How do I use these docs?** + The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them. @@ -20,65 +22,89 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Tables (`NEAR`.`CORE`.``) **Dimension Tables:** + - [dim_address_labels](#!/model/model.near.core__dim_address_labels) -- [dim_staking_pools](#!/model/model.near.core__dim_staking_pools) - [dim_token_labels](#!/model/model.near.core__dim_token_labels) **Fact Tables:** + - [fact_actions_events_function_call](#!/model/model.near.core__fact_actions_events_function_call) - [fact_actions_events](#!/model/model.near.core__fact_actions_events) - [fact_blocks](#!/model/model.near.core__fact_blocks) -- [fact_lockup_actions](#!/model/model.near.core__fact_lockup_actions) -- [fact_nft_mints](#!/model/model.near.core__fact_nft_mints) -- [fact_prices](#!/model/model.near.core__fact_prices) +- [fact_logs](#!/model/model.near.core__fact_logs) - [fact_receipts](#!/model/model.near.core__fact_receipts) - [fact_token_metadata](#!/model/model.near.core__fact_token_metadata) - [fact_transactions](#!/model/model.near.core__fact_transactions) - [fact_transfers](#!/model/model.near.core__fact_transfers) -- [fact_staking_actions](#!/model/model.near.core__fact_staking_actions) -- [fact_staking_pool_balances](#!/model/model.near.core__fact_staking_pool_balances) -- [fact_staking_pool_daily_balances](#!/model/model.near.core__fact_staking_pool_daily_balances) -**Convenience Tables:** -- [ez_dex_swaps](#!/model/model.near.core__ez_dex_swaps) +### DeFi Tables (`NEAR`.`DEFI`.``) + +- [ez_swaps](#!/model/model.near.defi__ez_dex_swaps) + +### Governance Tables (`NEAR`.`GOV`.``) + +- [dim_staking_pools](#!/model/model.near.gov__dim_staking_pools) +- [fact_lockup_actions](#!/model/model.near.gov__fact_lockup_actions) +- [fact_staking_actions](#!/model/model.near.gov__fact_staking_actions) +- [fact_staking_pool_balances](#!/model/model.near.gov__fact_staking_pool_balances) +- [fact_staking_pool_daily_balances](#!/model/model.near.gov__fact_staking_pool_daily_balances) + +### NFT Tables (`NEAR`.`NFT`.``) + +- [fact_nft_mints](#!/model/model.near.nft__fact_nft_mints) + +### Price Tables (`NEAR`.`PRICE`.``) + +- [fact_prices](#!/model/model.near.price__fact_prices) ### Social Tables (`NEAR`.`SOCIAL`.``) + - [fact_addkey_events](#!/model/model.near.social__fact_addkey_events) - [fact_decoded_actions](#!/model/model.near.social__fact_decoded_actions) - [fact_profile_changes](#!/model/model.near.social__fact_profile_changes) - [fact_posts](#!/model/model.near.social__fact_posts) - [fact_widget_deployments](#!/model/model.near.social__fact_widget_deployments) +### Social Tables (`NEAR`.`SOCIAL`.``) + +- [fact_addkey_events](#!/model/model.near.social__fact_addkey_events) +- [fact_decoded_actions](#!/model/model.near.social__fact_decoded_actions) +- [fact_profile_changes](#!/model/model.near.social__fact_profile_changes) +- [fact_posts](#!/model/model.near.social__fact_posts) +- [fact_widget_deployments](#!/model/model.near.social__fact_widget_deployments) ### Beta Tables (`NEAR`.`BETA`.``) + - [github_activity](https://github.com/forgxyz/developer_report_near) ### Custom Functions + - [udtf_call_contract_function](#!/macro/macro.near.create_UDTF_CALL_CONTRACT_FUNCTION_BY_HEIGHT) -Call a contract method via the [public NEAR RPC endpoint](https://docs.near.org/api/rpc/setup), modeled after the official documentation, [here](https://docs.near.org/api/rpc/contracts#call-a-contract-function). - +Call a contract method via the [public NEAR RPC endpoint](https://docs.near.org/api/rpc/setup), modeled after the official documentation, [here](https://docs.near.org/api/rpc/contracts#call-a-contract-function). + This function accepts 3 or 4 parameters: - - `account_id` STR (required) - - This is the deployed contract_address you want to call. - - `method_name` STR (required) - - This is the method on the contract to call. +- `account_id` STR (required) +- This is the deployed contract_address you want to call. - - `args` OBJ (required) - - Any requred or optional input parameters that the contract method accepts. - - For best results, this should be formed by using the Snowflake function [`OBJECT_CONSTRUCT()`](https://docs.snowflake.com/en/sql-reference/functions/object_construct) +- `method_name` STR (required) +- This is the method on the contract to call. + +- `args` OBJ (required) +- Any requred or optional input parameters that the contract method accepts. +- For best results, this should be formed by using the Snowflake function [`OBJECT_CONSTRUCT()`](https://docs.snowflake.com/en/sql-reference/functions/object_construct) - `block_id` INT (optional) - - Pass a block height (note - hash not accepted) to call the method at a certain block in time. - - If nothing is passed, the default behavior is `final` per the explanation [here](https://docs.near.org/api/rpc/setup#using-finality-param). - - Note - when passing in a block id parameter, the archive node is called which may be considerably slower than the primary access node. - +- Pass a block height (note - hash not accepted) to call the method at a certain block in time. +- If nothing is passed, the default behavior is `final` per the explanation [here](https://docs.near.org/api/rpc/setup#using-finality-param). +- Note - when passing in a block id parameter, the archive node is called which may be considerably slower than the primary access node. -**Important Note** - this is the public access endpoint, use responsibly. +**Important Note** - this is the public access endpoint, use responsibly. #### Examples -Return 25 accounts that have authorized the contract `social.near`. + +Return 25 accounts that have authorized the contract `social.near`. ```sql SELECT @@ -100,6 +126,7 @@ FROM ``` Get the staked balance of 100 addresses on the pool `staked.poolv1.near` at block `85,000,000`. + ```sql SELECT DATA :result :block_height :: NUMBER AS block_height, @@ -131,7 +158,7 @@ FROM ## **Data Model Overview** The NEAR - models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** +models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** - Bronze: Data is loaded in from the source as a view - Silver: All necessary parsing, filtering, de-duping, and other transformations are done here @@ -139,16 +166,17 @@ The NEAR The dimension tables are sourced from a variety of on-chain and off-chain sources. -Convenience views (denoted ez_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. +Convenience views (denoted ez\_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. ## **Using dbt docs** + ### Navigation -You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project. +You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models in the project. ### Database Tab -This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database. +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown in this interface, as they do not exist in the database. ### Graph Exploration @@ -156,12 +184,12 @@ You can click the blue icon on the bottom-right corner of the page to view the l On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. -Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). +Once expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). Note that you can also right-click on models to interactively filter and explore the graph. - ### **More information** + - [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) diff --git a/models/gold/defi/core__ez_dex_swaps.yml b/models/gold/defi/core__ez_dex_swaps.yml index 116e6a3..2676dd1 100644 --- a/models/gold/defi/core__ez_dex_swaps.yml +++ b/models/gold/defi/core__ez_dex_swaps.yml @@ -3,7 +3,7 @@ version: 2 models: - name: core__ez_dex_swaps description: |- - This table records all the swap transactions occurring in NEAR. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.DEFI.EZ_DEX_SWAPS**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: BLOCK_ID diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql new file mode 100644 index 0000000..a39b2c9 --- /dev/null +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -0,0 +1,73 @@ +{{ config( + materialized = 'view', + secure = true, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'DEFI, SWAPS' + } + } + }, + tags = ['core'] +) }} + +WITH dex_swaps AS ( + SELECT + * + FROM + {{ ref('silver__dex_swaps_s3') }} +), +unique_swap_ids AS ( + SELECT + swap_id, + COUNT(1) AS swaps + FROM + {{ ref('silver__dex_swaps_s3') }} + GROUP BY + 1 + HAVING + swaps = 1 +), +FINAL AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + swap_id, + trader, + platform, + pool_id, + token_in_symbol AS token_in, + token_in AS token_in_contract, + amount_in, + token_out_symbol AS token_out, + token_out AS token_out_contract, + amount_out + FROM + dex_swaps + WHERE + swap_id IN ( + SELECT + swap_id + FROM + unique_swap_ids + ) + AND amount_in IS NOT NULL + AND amount_out IS NOT NULL +) +SELECT + block_id, + block_timestamp, + tx_hash, + swap_id, + trader, + platform, + pool_id, + token_in, + token_in_contract, + amount_in, + token_out, + token_out_contract, + amount_out +FROM + FINAL diff --git a/models/gold/defi/defi__ez_dex_swaps.yml b/models/gold/defi/defi__ez_dex_swaps.yml new file mode 100644 index 0000000..904bb54 --- /dev/null +++ b/models/gold/defi/defi__ez_dex_swaps.yml @@ -0,0 +1,109 @@ +version: 2 + +models: + - name: defi__ez_dex_swaps + description: |- + This table records all the swap transactions occurring in NEAR. + + columns: + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + + - name: TX_HASH + description: "{{ doc('tx_hash')}}" + + - name: SWAP_ID + description: "{{ doc('swap_id')}}" + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: PLATFORM + description: "{{ doc('platform')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TRADER + description: "{{ doc('trader')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: POOL_ID + description: "{{ doc('pool_id')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: TOKEN_IN_SYMBOL + description: "{{ doc('symbol')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TOKEN_IN + description: "{{ doc('token_in')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: AMOUNT_IN + description: "{{ doc('amount_in')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - DOUBLE + + - name: TOKEN_OUT + description: "{{ doc('token_out')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TOKEN_OUT_SYMBOL + description: "{{ doc('symbol')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: AMOUNT_OUT + description: "{{ doc('amount_out')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - DOUBLE diff --git a/models/gold/governance/core__dim_staking_pools.yml b/models/gold/governance/core__dim_staking_pools.yml index 92f3ca6..c454903 100644 --- a/models/gold/governance/core__dim_staking_pools.yml +++ b/models/gold/governance/core__dim_staking_pools.yml @@ -3,7 +3,7 @@ version: 2 models: - name: core__dim_staking_pools description: |- - This table contains registered staking pools with NEAR. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.GOV.DIM_STAKING_POOLS**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: TX_HASH @@ -60,4 +60,4 @@ models: - STRING - VARCHAR - dbt_expectations.expect_column_values_to_be_in_set: - value_set: ['Create', 'Update'] + value_set: ["Create", "Update"] diff --git a/models/gold/governance/core__fact_lockup_actions.yml b/models/gold/governance/core__fact_lockup_actions.yml index 508befd..e7c4e5e 100644 --- a/models/gold/governance/core__fact_lockup_actions.yml +++ b/models/gold/governance/core__fact_lockup_actions.yml @@ -3,7 +3,7 @@ version: 2 models: - name: core__fact_lockup_actions description: |- - This table records all disbursements by the contract lockup.near. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.GOV.FACT_LOCKUP_ACTIONS**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: TX_HASH @@ -17,7 +17,7 @@ models: - name: DEPOSIT description: "{{ doc('deposit')}}" - tests: + tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: @@ -27,17 +27,17 @@ models: - name: LOCKUP_ACCOUNT_ID description: "{{ doc('lockup_account_id')}}" - tests: + tests: - not_null - name: OWNER_ACCOUNT_ID description: "{{ doc('owner_account_id')}}" - tests: + tests: - not_null - name: LOCKUP_DURATION description: "{{ doc('lockup_duration')}}" - tests: + tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: @@ -45,14 +45,14 @@ models: - name: LOCKUP_TIMESTAMP description: "{{ doc('lockup_timestamp')}}" - tests: + tests: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - VARCHAR - name: LOCKUP_TIMESTAMP_NTZ description: "{{ doc('lockup_timestamp_ntz')}}" - tests: + tests: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ @@ -69,4 +69,3 @@ models: - name: TRANSFERS_INFORMATION description: "{{ doc('transfers_information')}}" - \ No newline at end of file diff --git a/models/gold/governance/core__fact_staking_actions.yml b/models/gold/governance/core__fact_staking_actions.yml index 69a55e3..9bd3003 100644 --- a/models/gold/governance/core__fact_staking_actions.yml +++ b/models/gold/governance/core__fact_staking_actions.yml @@ -3,9 +3,7 @@ version: 2 models: - name: core__fact_staking_actions description: |- - An updated version of the staking actions table which looks at all logs, instead of just the first receipt. - There are four actions taken when staking: staking->deposit->unstaking->withdraw. - Note - in this core view the amount is decimal adjusted by 10^24. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.GOV.FACT_STAKING_ACTIONS**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: TX_HASH diff --git a/models/gold/governance/core__fact_staking_pool_balances.yml b/models/gold/governance/core__fact_staking_pool_balances.yml index 46b431d..9bcbc44 100644 --- a/models/gold/governance/core__fact_staking_pool_balances.yml +++ b/models/gold/governance/core__fact_staking_pool_balances.yml @@ -3,9 +3,7 @@ version: 2 models: - name: core__fact_staking_pool_balances description: |- - Staking pool balances as extracted from receipt logs when an individual makes a staking action. - To calculate balance at a point in time, isolate a single record for each pool. This table is transactional-based, so balances are updated with every staking event by users. - Note - the amount in balance is decimal adjusted by 10^24. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.GOV.FACT_STAKING_POOL_BALANCES**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: TX_HASH diff --git a/models/gold/governance/core__fact_staking_pool_daily_balances.yml b/models/gold/governance/core__fact_staking_pool_daily_balances.yml index c613ee3..ba6a680 100644 --- a/models/gold/governance/core__fact_staking_pool_daily_balances.yml +++ b/models/gold/governance/core__fact_staking_pool_daily_balances.yml @@ -3,7 +3,7 @@ version: 2 models: - name: core__fact_staking_pool_daily_balances description: |- - Aggregates the balances of each pool for each day, taking the last balance reported for each pool. This always excludes the present date. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.GOV.FACT_STAKING_POOL_DAILY_BALANCES**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: DATE diff --git a/models/gold/governance/gov__dim_staking_pools.sql b/models/gold/governance/gov__dim_staking_pools.sql new file mode 100644 index 0000000..db99398 --- /dev/null +++ b/models/gold/governance/gov__dim_staking_pools.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view', + secure = true, + tags = ['core', 'governance'], + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING GOVERNANCE' + } + } + } +) }} + +WITH staking_pools AS ( + + SELECT + * + FROM + {{ ref('silver__staking_pools_s3') }} +) +SELECT + tx_hash, + block_timestamp, + owner, + address, + reward_fee_fraction, + tx_type +FROM + staking_pools diff --git a/models/gold/governance/gov__dim_staking_pools.yml b/models/gold/governance/gov__dim_staking_pools.yml new file mode 100644 index 0000000..f35ef18 --- /dev/null +++ b/models/gold/governance/gov__dim_staking_pools.yml @@ -0,0 +1,63 @@ +version: 2 + +models: + - name: gov__dim_staking_pools + description: |- + This table contains registered staking pools with NEAR. + + columns: + - name: TX_HASH + description: "{{ doc('tx_hash') }}" + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: OWNER + description: "{{ doc('staking_pool_owner')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: ADDRESS + description: "{{ doc('staking_pool_address')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: REWARD_FEE_FRACTION + description: "{{ doc('staking_pool_reward_fee_fraction')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: TX_TYPE + description: "{{ doc('staking_pool_tx_type') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ["Create", "Update"] diff --git a/models/gold/governance/gov__fact_lockup_actions.sql b/models/gold/governance/gov__fact_lockup_actions.sql new file mode 100644 index 0000000..7d776ad --- /dev/null +++ b/models/gold/governance/gov__fact_lockup_actions.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'view', + secure = true, + tags = ['core', 'governance'], + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING GOVERNANCE' + } + } + } +) }} + +WITH lockup_actions AS ( + + SELECT + tx_hash, + block_timestamp, + block_id, + deposit, + lockup_account_id, + owner_account_id, + lockup_duration, + lockup_timestamp, + lockup_timestamp_ntz, + release_duration, + vesting_schedule, + transfers_information + FROM + {{ ref('silver__lockup_actions') }} +) +SELECT + * +FROM + lockup_actions diff --git a/models/gold/governance/gov__fact_lockup_actions.yml b/models/gold/governance/gov__fact_lockup_actions.yml new file mode 100644 index 0000000..50edabb --- /dev/null +++ b/models/gold/governance/gov__fact_lockup_actions.yml @@ -0,0 +1,71 @@ +version: 2 + +models: + - name: gov__fact_lockup_actions + description: |- + This table records all disbursements by the contract lockup.near. + + columns: + - name: TX_HASH + description: "{{ doc('tx_hash')}}" + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + + - name: DEPOSIT + description: "{{ doc('deposit')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DOUBLE + - FLOAT + - NUMBER + + - name: LOCKUP_ACCOUNT_ID + description: "{{ doc('lockup_account_id')}}" + tests: + - not_null + + - name: OWNER_ACCOUNT_ID + description: "{{ doc('owner_account_id')}}" + tests: + - not_null + + - name: LOCKUP_DURATION + description: "{{ doc('lockup_duration')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + + - name: LOCKUP_TIMESTAMP + description: "{{ doc('lockup_timestamp')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + + - name: LOCKUP_TIMESTAMP_NTZ + description: "{{ doc('lockup_timestamp_ntz')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: RELEASE_DURATION + description: "{{ doc('release_duration')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + + - name: VESTING_SCHEDULE + description: "{{ doc('vesting_schedule')}}" + + - name: TRANSFERS_INFORMATION + description: "{{ doc('transfers_information')}}" diff --git a/models/gold/governance/gov__fact_staking_actions.sql b/models/gold/governance/gov__fact_staking_actions.sql new file mode 100644 index 0000000..5fe0696 --- /dev/null +++ b/models/gold/governance/gov__fact_staking_actions.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view', + secure = true, + tags = ['core', 'governance'], + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING GOVERNANCE' + } + } + } +) }} + +WITH staking_actions AS ( + + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id AS address, + signer_id, + action, + amount_adj AS amount + FROM + {{ ref('silver__staking_actions_v2') }} +) +SELECT + * +FROM + staking_actions diff --git a/models/gold/governance/gov__fact_staking_actions.yml b/models/gold/governance/gov__fact_staking_actions.yml new file mode 100644 index 0000000..8839840 --- /dev/null +++ b/models/gold/governance/gov__fact_staking_actions.yml @@ -0,0 +1,33 @@ +version: 2 + +models: + - name: gov__fact_staking_actions + description: |- + An updated version of the staking actions table which looks at all logs, instead of just the first receipt. + There are four actions taken when staking: staking->deposit->unstaking->withdraw. + Note - in this core view the amount is decimal adjusted by 10^24. + + columns: + - name: TX_HASH + description: "{{ doc('tx_hash') }}" + + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + + - name: RECEIPT_OBJECT_ID + description: "{{ doc('receipt_object_id') }}" + + - name: ADDRESS + description: "{{ doc('pool_address') }}" + + - name: SIGNER_ID + description: "{{ doc('signer_id') }}" + + - name: ACTION + description: "{{ doc('staking_action') }}" + + - name: AMOUNT + description: "{{ doc('amount') }}" diff --git a/models/gold/governance/gov__fact_staking_pool_balances.sql b/models/gold/governance/gov__fact_staking_pool_balances.sql new file mode 100644 index 0000000..6ee7803 --- /dev/null +++ b/models/gold/governance/gov__fact_staking_pool_balances.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view', + secure = true, + tags = ['core', 'governance'], + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING GOVERNANCE' + } + } + } +) }} + +WITH balance_changes AS ( + + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + receiver_id AS address, + amount_adj AS balance + FROM + {{ ref('silver__pool_balances') }} +) +SELECT + * +FROM + balance_changes diff --git a/models/gold/governance/gov__fact_staking_pool_balances.yml b/models/gold/governance/gov__fact_staking_pool_balances.yml new file mode 100644 index 0000000..e746826 --- /dev/null +++ b/models/gold/governance/gov__fact_staking_pool_balances.yml @@ -0,0 +1,27 @@ +version: 2 + +models: + - name: gov__fact_staking_pool_balances + description: |- + Staking pool balances as extracted from receipt logs when an individual makes a staking action. + To calculate balance at a point in time, isolate a single record for each pool. This table is transactional-based, so balances are updated with every staking event by users. + Note - the amount in balance is decimal adjusted by 10^24. + + columns: + - name: TX_HASH + description: "{{ doc('tx_hash') }}" + + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + + - name: RECEIPT_OBJECT_ID + description: "{{ doc('receipt_object_id') }}" + + - name: ADDRESS + description: "{{ doc('pool_address') }}" + + - name: BALANCE + description: "{{ doc('balance') }}" diff --git a/models/gold/governance/gov__fact_staking_pool_daily_balances.sql b/models/gold/governance/gov__fact_staking_pool_daily_balances.sql new file mode 100644 index 0000000..0383be9 --- /dev/null +++ b/models/gold/governance/gov__fact_staking_pool_daily_balances.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view', + secure = true, + tags = ['core', 'governance'], + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING GOVERNANCE' + } + } + } +) }} + +WITH daily_balance AS ( + + SELECT + date_day as date, + address, + balance + FROM + {{ ref('silver__pool_balance_daily') }} +) +SELECT + * +FROM + daily_balance diff --git a/models/gold/governance/gov__fact_staking_pool_daily_balances.yml b/models/gold/governance/gov__fact_staking_pool_daily_balances.yml new file mode 100644 index 0000000..5620995 --- /dev/null +++ b/models/gold/governance/gov__fact_staking_pool_daily_balances.yml @@ -0,0 +1,16 @@ +version: 2 + +models: + - name: gov__fact_staking_pool_daily_balances + description: |- + Aggregates the balances of each pool for each day, taking the last balance reported for each pool. This always excludes the present date. + + columns: + - name: DATE + description: "{{ doc('date') }}" + + - name: ADDRESS + description: "{{ doc('pool_address') }}" + + - name: BALANCE + description: "{{ doc('balance') }}" diff --git a/models/gold/nft/core__fact_nft_mints.yml b/models/gold/nft/core__fact_nft_mints.yml index 1bcec81..ce3e06f 100644 --- a/models/gold/nft/core__fact_nft_mints.yml +++ b/models/gold/nft/core__fact_nft_mints.yml @@ -3,7 +3,7 @@ version: 2 models: - name: core__fact_nft_mints description: |- - This table records all the NFT Mints per the NEP171 standard, across various methods. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.NFT.FACT_NFT_MINTS**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: RECEIPT_OBJECT_ID @@ -17,7 +17,7 @@ models: - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp')}}" - + - name: TOKEN_ID description: "{{ doc('nft_token_id') }}" tests: @@ -66,7 +66,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - FLOAT + - FLOAT - name: MINT_PER_TX description: "{{ doc('mint_per_tx') }}" @@ -75,7 +75,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - FLOAT + - FLOAT - name: GAS_BURNT description: "{{ doc('gas_burnt') }}" @@ -84,7 +84,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - FLOAT + - FLOAT - name: TRANSACTION_FEE description: "{{ doc('transaction_fee') }}" @@ -92,7 +92,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - FLOAT + - FLOAT - name: TX_STATUS description: "{{ doc('tx_status') }}" @@ -100,4 +100,4 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - - VARCHAR \ No newline at end of file + - VARCHAR diff --git a/models/gold/nft/nft__fact_nft_mints.sql b/models/gold/nft/nft__fact_nft_mints.sql new file mode 100644 index 0000000..ece0807 --- /dev/null +++ b/models/gold/nft/nft__fact_nft_mints.sql @@ -0,0 +1,41 @@ +{{ config( + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'NFT' + } + } + }, + tags = ['core', 'nft'] +) }} + +WITH nft_mints AS ( + + SELECT + receipt_object_id, + tx_hash, + block_id, + block_timestamp, + token_id, + method_name, + args, + memo, + deposit, + tx_receiver, + receiver_id, + signer_id, + owner_id, + owner_per_tx, + mint_per_tx, + gas_burnt, + transaction_fee, + implied_price, + tx_status + FROM + {{ ref('silver__standard_nft_mint_s3') }} +) +SELECT + * +FROM + nft_mints diff --git a/models/gold/nft/nft__fact_nft_mints.yml b/models/gold/nft/nft__fact_nft_mints.yml new file mode 100644 index 0000000..6c6f0ab --- /dev/null +++ b/models/gold/nft/nft__fact_nft_mints.yml @@ -0,0 +1,103 @@ +version: 2 + +models: + - name: nft__fact_nft_mints + description: |- + This table records all the NFT Mints per the NEP171 standard, across various methods. + + columns: + - name: RECEIPT_OBJECT_ID + description: "{{ doc('receipt_object_id') }}" + + - name: TX_HASH + description: "{{ doc('tx_hash') }}" + + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + + - name: TOKEN_ID + description: "{{ doc('nft_token_id') }}" + tests: + - not_null: + where: signer_id != 'decentland.near' + + - name: METHOD_NAME + description: "{{ doc('method_name') }}" + + - name: ARGS + description: "{{ doc('args') }}" + + - name: MEMO + description: "{{ doc('memo') }}" + + - name: DEPOSIT + description: "{{ doc('deposit') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: TX_SIGNER + description: "{{ doc('tx_signer') }}" + + - name: RECEIVER_ID + description: "{{ doc('receiver_id')}}" + + - name: SIGNER_ID + description: "{{ doc('signer_id')}}" + + - name: OWNER_ID + description: "{{ doc('owner_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: OWNER_PER_TX + description: "{{ doc('owner_per_tx') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: MINT_PER_TX + description: "{{ doc('mint_per_tx') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: GAS_BURNT + description: "{{ doc('gas_burnt') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: TRANSACTION_FEE + description: "{{ doc('transaction_fee') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: TX_STATUS + description: "{{ doc('tx_status') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR diff --git a/models/gold/prices/core__fact_prices.yml b/models/gold/prices/core__fact_prices.yml index 7978d99..e84abfb 100644 --- a/models/gold/prices/core__fact_prices.yml +++ b/models/gold/prices/core__fact_prices.yml @@ -3,7 +3,7 @@ version: 2 models: - name: core__fact_prices description: |- - This table presents asset prices for the NEAR blockchain from various sources. Presently, only the on-chain oracle is provide. + Deprecating soon: This is a notice that we're replacing this table with **NEAR.PRICE.FACT_PRICES**. Please migrate queries to the new table by **9/14/23**. There will be no column changes or any other changes of any kind. columns: - name: TIMESTAMP diff --git a/models/gold/prices/price__fact_prices.sql b/models/gold/prices/price__fact_prices.sql new file mode 100644 index 0000000..fce4dad --- /dev/null +++ b/models/gold/prices/price__fact_prices.sql @@ -0,0 +1,36 @@ +{{ config( + materialized = 'view', + secure = true, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'PRICE' + } + } + }, + tags = ['core', 'price'] +) }} + +WITH oracle_prices AS ( + + SELECT + block_timestamp AS TIMESTAMP, + token, + symbol, + token_contract, + raw_price, + price_usd, + source + FROM + {{ ref('silver__prices_oracle_s3') }} +), +FINAL AS ( + SELECT + * + FROM + oracle_prices +) +SELECT + * +FROM + FINAL diff --git a/models/gold/prices/price__fact_prices.yml b/models/gold/prices/price__fact_prices.yml new file mode 100644 index 0000000..f856afe --- /dev/null +++ b/models/gold/prices/price__fact_prices.yml @@ -0,0 +1,71 @@ +version: 2 + +models: + - name: price__fact_prices + description: |- + This table presents asset prices for the NEAR blockchain from various sources. Presently, only the on-chain oracle is provide. + + columns: + - name: TIMESTAMP + description: "{{ doc('timestamp')}}" + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: TOKEN + description: "{{ doc('token')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: SYMBOL + description: "{{ doc('symbol')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TOKEN_CONTRACT + description: "{{ doc('token_contract')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: RAW_PRICE + description: "{{ doc('price_usd')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - DOUBLE + + - name: PRICE_USD + description: "{{ doc('price_usd')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - DOUBLE + + - name: SOURCE + description: "{{ doc('source')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR diff --git a/models/sources.yml b/models/sources.yml index 31f8d4a..8a9158e 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -18,5 +18,5 @@ sources: schema: core database: crosschain tables: - - name: address_labels + - name: dim_labels - name: dim_dates diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ec44b06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +dbt-snowflake>=1.4,<1.5 \ No newline at end of file