From 8d19961ce605c22d3b82f3320c9d83c5ecdf8bbd Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Mon, 17 Jun 2024 10:46:30 -0700 Subject: [PATCH] Cleanup/decrease test time and docs update (#584) * update docs * update test range * revert test --- README.md | 2 +- models/descriptions/__overview__.md | 17 +++++++++++++++++ models/gold/defi/defi__fact_swaps.yml | 7 +++++-- models/silver/silver___inner_instructions.yml | 16 +++++++++------- .../tokens/silver__token_mint_actions.yml | 19 +++++++++++-------- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ca6e0352..175b3c65 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ solana: ``` ### Notes -- `block_id` column in our data models matches what [Solcan](https://solscan.io/) classifies as `Block`. This value is same as what is referred to as `Slot`. +- `block_id` column in our data models matches what [Solscan](https://solscan.io/) classifies as `Block`. This value is same as what is referred to as `Slot`. ### Resources: - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 3c32b61c..cdaf99fe 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -21,6 +21,7 @@ There is more information on how to use dbt docs in the last section of this doc **Core Dimension Tables:** - [dim_labels](#!/model/model.solana_models.core__dim_labels) +- [dim_idls](#!/model/model.solana_models.core__dim_idls) **Core Fact Tables:** - [fact_blocks](#!/model/model.solana_models.core__fact_blocks) @@ -28,17 +29,23 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_transactions](#!/model/model.solana_models.core__fact_transactions) - [fact_transfers](#!/model/model.solana_models.core__fact_transfers) - [fact_decoded_instructions](#!/model/model.solana_models.core__fact_decoded_instructions) +- [fact_token_account_owners](#!/model/model.solana_models.core__fact_token_account_owners) **Core Convenience Tables:** - [ez_signers](#!/model/model.solana_models.core__ez_signers) - [ez_events_decoded](#!/model/model.solana_models.core__ez_events_decoded) ### DeFi Tables (`Solana`.`DEFI`.``) +- [ez_dex_swaps](#!/model/model.solana_models.defi__ez_dex_swaps) - [fact_liquidity_pool_actions](#!/model/model.solana_models.defi__fact_liquidity_pool_actions) - [fact_stake_pool_actions](#!/model/model.solana_models.defi__fact_stake_pool_actions) - [fact_swaps](#!/model/model.solana_models.defi__fact_swaps) +- [fact_bridge_activity](#!/model/model.solana_models.defi__fact_bridge_activity) +- [fact_token_burn_actions](#!/model/model.solana_models.defi__fact_token_burn_actions) +- [fact_token_mint_actions](#!/model/model.solana_models.defi__fact_token_mint_actions) ### Governance Tables (`Solana`.`GOV`.``) +- [dim_epoch](#!/model/model.solana_models.gov__dim_epoch) - [ez_staking_lp_actions](#!/model/model.solana_models.gov__ez_staking_lp_actions) - [fact_gauges_creates](#!/model/model.solana_models.gov__fact_gauges_create) - [fact_gauges_votes](#!/model/model.solana_models.gov__fact_gauges_votes) @@ -51,11 +58,17 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_block_production](#!/model/model.solana_models.gov__fact_block_production) - [fact_validators](#!/model/model.solana_models.gov__fact_validators) - [fact_votes_agg_block](#!/model/model.solana_models.gov__fact_votes_agg_block) +- [fact_rewards_fees](#!/model/model.solana_models.gov__fact_rewards_fees) +- [fact_rewards_rent](#!/model/model.solana_models.gov__fact_rewards_rent) +- [fact_rewards_staking](#!/model/model.solana_models.gov__fact_rewards_staking) +- [fact_rewards_voting](#!/model/model.solana_models.gov__fact_rewards_voting) ### NFT Tables (`Solana`.`NFT`.``) - [dim_nft_metadata](#!/model/model.solana_models.nft__dim_nft_metadata) - [fact_nft_mints](#!/model/model.solana_models.nft__fact_nft_mints) - [fact_nft_sales](#!/model/model.solana_models.nft__fact_nft_sales) +- [fact_nft_burn_actions](#!/model/model.solana_models.nft__fact_nft_burn_actions) +- [fact_nft_mint_actions](#!/model/model.solana_models.nft__fact_nft_mint_actions) ### Price Tables (`Solana`.`PRICE`.``) - [dim_asset_metadata](#!/model/model.solana_models.price__dim_asset_metadata) @@ -63,6 +76,10 @@ There is more information on how to use dbt docs in the last section of this doc - [ez_asset_metadata](#!/model/model.solana_models.price__ez_asset_metadata) - [ez_prices_hourly](#!/model/model.solana_models.price__ez_prices_hourly) +### Stats Tables (`Solana`.`STATS`.``) +- [ez_core_metrics](#!/model/model.solana_models.stats__ez_core_metrics) + + ## **Data Model Overview** The Solana models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold.** diff --git a/models/gold/defi/defi__fact_swaps.yml b/models/gold/defi/defi__fact_swaps.yml index 4bea7ea6..d3d7b4a1 100644 --- a/models/gold/defi/defi__fact_swaps.yml +++ b/models/gold/defi/defi__fact_swaps.yml @@ -2,6 +2,9 @@ version: 2 models: - name: defi__fact_swaps description: This table contains swaps performed on Jupiter, Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. For Phoenix, we are not capturing swaps where there are separate transactions for placing the order and filling the order. + recent_date_filter: &recent_date_filter + config: + where: inserted_timestamp >= current_date - 7 columns: - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" @@ -46,11 +49,11 @@ models: - name: SWAP_PROGRAM description: name of decentralized exchange used to perform the swap tests: - - not_null + - not_null: *recent_date_filter - name: _LOG_ID description: "Combination of TX_ID and event index" tests: - - not_null + - not_null: *recent_date_filter - name: FACT_SWAPS_ID description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP diff --git a/models/silver/silver___inner_instructions.yml b/models/silver/silver___inner_instructions.yml index 2551e8d8..e0075253 100644 --- a/models/silver/silver___inner_instructions.yml +++ b/models/silver/silver___inner_instructions.yml @@ -1,35 +1,37 @@ version: 2 models: - name: silver___inner_instructions + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 columns: - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" tests: - - not_null: - where: block_id > 39824213 + - not_null: *recent_date_filter - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day interval: 1 - name: BLOCK_ID description: "{{ doc('block_id') }}" tests: - - not_null + - not_null: *recent_date_filter - name: TX_ID description: "{{ doc('tx_id') }}" tests: - - not_null + - not_null: *recent_date_filter - name: INDEX description: Location of the inner_instruction within the inner_instructions of a transaction tests: - - not_null + - not_null: *recent_date_filter - name: MAPPED_INSTRUCTION_INDEX description: Specifies the instruction which this inner_instruction applies tests: - - not_null + - not_null: *recent_date_filter - name: VALUE description: json object that contains the inner instruction tests: - - not_null + - not_null: *recent_date_filter - name: _INSERTED_TIMESTAMP description: "{{ doc('_inserted_timestamp') }}" tests: diff --git a/models/silver/tokens/silver__token_mint_actions.yml b/models/silver/tokens/silver__token_mint_actions.yml index b1758021..65bae120 100644 --- a/models/silver/tokens/silver__token_mint_actions.yml +++ b/models/silver/tokens/silver__token_mint_actions.yml @@ -9,6 +9,9 @@ models: - INNER_INDEX - MINT where: block_timestamp::date > current_date - 30 + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 columns: - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" @@ -21,11 +24,11 @@ models: - name: BLOCK_ID description: "{{ doc('block_id') }}" tests: - - not_null + - not_null: *recent_date_filter - name: TX_ID description: "{{ doc('tx_id') }}" tests: - - not_null + - not_null: *recent_date_filter - name: SUCCEEDED description: "{{ doc('tx_succeeded') }}" tests: @@ -33,26 +36,26 @@ models: - name: INDEX description: "Location of the event within the instructions of a transaction" tests: - - not_null + - not_null: *recent_date_filter - name: INNER_INDEX description: "Location of the event within the inner instructions of a transaction" - name: EVENT_TYPE description: "{{ doc('event_type') }}" tests: - - not_null + - not_null: *recent_date_filter - name: MINT description: "{{ doc('mint') }}" tests: - - not_null + - not_null: *recent_date_filter - name: MINT_AMOUNT description: Number of tokens burned tests: - - not_null: + - not_null: *recent_date_filter where: event_type in ('mintToChecked', 'mintTo') - name: MINT_AUTHORITY description: Account address authorizing burn tests: - - not_null + - not_null: *recent_date_filter - name: TOKEN_ACCOUNT description: The account address where tokens are minted to - name: SIGNERS @@ -60,7 +63,7 @@ models: - name: DECIMAL description: "{{ doc('decimal') }}" tests: - - not_null + - not_null: *recent_date_filter - name: MINT_STANDARD_TYPE description: "{{ doc('mint_standard_type') }}" - name: _INSERTED_TIMESTAMP