diff --git a/dbt_project.yml b/dbt_project.yml index adc7d46..ffa5829 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -42,11 +42,12 @@ models: deploy: core: materialized: ephemeral - + data_tests: ton_models: # replace with the name of the chain +store_failures: true # all tests + +where: "modified_timestamp::DATE > dateadd(hour, -{{ var('TEST_HOURS_THRESHOLD', 36) }}, sysdate())" on-run-start: - '{{create_sps()}}' diff --git a/models/gold/core/gold_core.yml b/models/gold/core/gold_core.yml index c71e65f..b98d27a 100644 --- a/models/gold/core/gold_core.yml +++ b/models/gold/core/gold_core.yml @@ -1,8 +1,445 @@ version: 2 models: + - name: core__fact_account_states + description: "Contains information about TON account states, tracking changes in account balances, statuses, and transaction histories. Based on the account_states table from TON-ETL." + columns: + - name: block_timestamp + description: "Timestamp of the block when the account state was recorded, converted from Unix timestamp." + - name: account + description: "TON account address in base64 format." + - name: timestamp + description: "Unix timestamp when the account state was recorded." + - name: last_trans_lt + description: "Logical time of the last transaction associated with this account." + - name: last_tx_hash + description: "Hash of the last transaction associated with this account." + - name: account_status + description: "Current status of the account (active, uninitialized, or frozen)." + - name: balance + description: "Current balance of the account in nanoTONs." + - name: frozen_hash + description: "Hash of the frozen state if the account is frozen, otherwise null." + - name: account_state_hash + description: "Hash of the account state." + - name: fact_account_states_id + description: "Unique identifier for the account state record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + + - name: core__fact_balances + description: "Contains historical balance information for both native TON and Jetton balances. Tracks changes in balances over time." + columns: + - name: block_timestamp + description: "Timestamp of the block when the balance was recorded, converted from Unix timestamp." + - name: address + description: "Address of the asset owner." + - name: asset + description: "Asset type, 'TON' for native TON or jetton address for Jetton balance." + - name: amount + description: "Balance amount. For TON, this is in nanoTONs. For Jettons, this is the raw amount before dividing by 10^decimals." + - name: mintless_claimed + description: "Boolean flag indicating if the mintless jetton was claimed (only applicable for mintless jetton wallets)." + - name: lt + description: "Logical time of the balance update." + - name: fact_balances_history_id + description: "Unique identifier for the balance history record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + - name: core__fact_blocks - - config: - event_time: block_date - \ No newline at end of file + description: "Contains information about blocks in the TON blockchain (masterchain and workchains). Represents the blockchain structure." + columns: + - name: block_timestamp + description: "Timestamp of the block, converted from gen_utime." + - name: gen_utime + description: "Unix timestamp when the block was generated." + - name: workchain + description: "Workchain ID where the block was created (usually -1 for masterchain, 0 for base workchain)." + - name: version + description: "Version of the block." + - name: shard + description: "Shard identifier for the block." + - name: seqno + description: "Block sequence number within its workchain." + - name: vert_seqno + description: "Vertical sequence number for the block." + - name: start_lt + description: "Starting logical time of the block." + - name: end_lt + description: "Ending logical time of the block." + - name: mc_block_seqno + description: "Masterchain block sequence number referenced by this block." + - name: mc_block_shard + description: "Masterchain block shard referenced by this block." + - name: tx_count + description: "Number of transactions in the block." + - name: global_id + description: "Global identifier for the TON network." + - name: created_by + description: "Validator that created the block." + - name: want_merge + description: "Flag indicating if the block wants to be merged." + - name: root_hash + description: "Root hash of the block." + - name: key_block + description: "Boolean flag indicating if this is a key block." + - name: vert_seqno_incr + description: "Increment of vertical sequence number." + - name: validator_list_hash_short + description: "Short hash of the validator list." + - name: after_merge + description: "Flag indicating if the block was created after a merge." + - name: want_split + description: "Flag indicating if the block wants to be split." + - name: after_split + description: "Flag indicating if the block was created after a split." + - name: master_ref_seqno + description: "Masterchain reference sequence number." + - name: mc_block_workchain + description: "Masterchain block workchain referenced by this block." + - name: file_hash + description: "Hash of the block file." + - name: prev_key_block_seqno + description: "Sequence number of the previous key block." + - name: flags + description: "Flags associated with the block." + - name: rand_seed + description: "Random seed used in block generation." + - name: gen_catchain_seqno + description: "Catchain sequence number used in block generation." + - name: min_ref_mc_seqno + description: "Minimum referenced masterchain sequence number." + - name: before_split + description: "Flag indicating if the block was created before a split." + - name: fact_blocks_id + description: "Unique identifier for the block record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + + - name: core__fact_jetton_events + description: "Contains information about Jetton events such as transfers, burns, and mints according to TEP-74 standard." + columns: + - name: block_timestamp + description: "Timestamp of the block when the jetton event occurred, converted from utime." + - name: tx_hash + description: "Transaction hash of the event." + - name: tx_succeeded + description: "Boolean flag indicating if the transaction succeeded. Set to FALSE if tx_aborted is TRUE." + - name: type + description: "Event type: 'transfer' for TEP-74 transfer, 'burn' for TEP-74 burn, or 'mint' for jetton minting." + - name: source + description: "Source address of the tokens. Null for mint events." + - name: destination + description: "Destination address for the tokens. Null for burn events." + - name: forward_ton_amount + description: "Amount of TON forwarded with the transaction, stored as a raw decimal value." + - name: amount + description: "Amount of jettons involved in the event, stored as a raw value without dividing by 10^decimals." + - name: jetton + description: "Address of the jetton master contract." + - name: comment + description: "Optional comment attached to the transaction." + - name: query_id + description: "Query ID of the transaction, stored as a raw decimal value." + - name: tx_lt + description: "Logical time of the transaction." + - name: utime + description: "Unix timestamp when the event occurred." + - name: tx_aborted + description: "Boolean flag indicating if the transaction was aborted. If true, the event should be discarded." + - name: fact_jetton_events_id + description: "Unique identifier for the jetton event record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + + - name: core__fact_jetton_metadata + description: "Contains metadata information for Jettons (TEP-64 standard), including both on-chain and off-chain data." + columns: + - name: address + description: "Jetton master address." + - name: update_timestamp_onchain + description: "Timestamp of the last on-chain update, converted from update_time_onchain." + - name: symbol + description: "TEP-64 jetton symbol from on-chain, off-chain, or tonapi source." + - name: decimals + description: "TEP-64 jetton decimals from on-chain, off-chain, or tonapi source. If not present, defaults to 9." + - name: name + description: "TEP-64 jetton name from on-chain, off-chain, or tonapi source." + - name: description + description: "TEP-64 jetton description from on-chain, off-chain, or tonapi source." + - name: tonapi_image_url + description: "Cached image URL from tonapi." + - name: image_data + description: "TEP-64 jetton image data from on-chain, off-chain, or tonapi source." + - name: image + description: "TEP-64 jetton image URL from on-chain, off-chain, or tonapi source." + - name: admin_address + description: "Admin address from on-chain data." + - name: mintable + description: "Boolean flag indicating if the jetton is mintable based on on-chain data." + - name: jetton_content_onchain + description: "JSON serialized string with on-chain jetton content." + - name: jetton_wallet_code_hash + description: "Jetton wallet code hash from on-chain data." + - name: code_hash + description: "Jetton code hash from on-chain data." + - name: adding_at + description: "Date when the output file was created." + - name: sources + description: "Record with sources of jetton metadata fields (symbol, name, description, image, image_data, decimals). Values can be 'onchain', 'offchain', 'tonapi', or '' (not set)." + - name: metadata_status + description: "Off-chain metadata update status: 0 - no off-chain metadata, 1 - success, -1 - error." + - name: update_time_onchain + description: "Unix timestamp of on-chain update, such as admin address transfer." + - name: update_time_metadata + description: "Unix timestamp of off-chain metadata update." + - name: fact_jetton_metadata_id + description: "Unique identifier for the jetton metadata record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: update_timestamp_onchain + interval: 30 + severity: error + tags: ['test_recency'] + + - name: core__fact_messages + description: "Contains messages from transactions, including both internal and external messages with their body data." + columns: + - name: block_timestamp + description: "Timestamp of the block when the message was processed, converted from tx_now." + - name: tx_hash + description: "Transaction hash associated with the message." + - name: msg_hash + description: "Hash of the message." + - name: body_hash + description: "Hash of the message body." + - name: trace_id + description: "Trace ID for the message within the transaction." + - name: direction + description: "Direction of the message: 'in' for messages that initiated a transaction, 'out' for messages resulting from a transaction." + - name: source + description: "Source address of the message." + - name: destination + description: "Destination address of the message." + - name: value + description: "Amount of TON attached to the message." + - name: opcode + description: "Operation code extracted from the message body." + - name: created_at + description: "Timestamp when the message was created." + - name: tx_now + description: "Unix timestamp of the transaction processing." + - name: ihr_fee + description: "Instant hypercube routing fee." + - name: import_fee + description: "Import fee for the message." + - name: fwd_fee + description: "Forward fee for the message." + - name: ihr_disabled + description: "Boolean flag indicating if instant hypercube routing is disabled for this message." + - name: bounced + description: "Boolean flag indicating if the message was bounced." + - name: bounce + description: "Boolean flag indicating if the message can bounce if delivery fails." + - name: comment + description: "Text comment extracted from the message body if available." + - name: tx_lt + description: "Logical time of the transaction that processed this message." + - name: created_lt + description: "Logical time when the message was created." + - name: init_state_hash + description: "Hash of the initial state if present." + - name: init_state_boc + description: "Raw initial state BOC (Bag of Cells) if present." + - name: body_boc + description: "Raw body BOC (Bag of Cells) of the message." + - name: fact_messages_id + description: "Unique identifier for the message record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + + - name: core__fact_transactions + description: "Contains information about transactions in the TON blockchain, tracking account changes, fees, and computation results." + columns: + - name: block_timestamp + description: "Timestamp of the block when the transaction was processed, converted from now." + - name: tx_hash + description: "Hash of the transaction." + - name: prev_tx_hash + description: "Hash of the previous transaction for the same account." + - name: tx_succeeded + description: "Boolean flag indicating if the transaction succeeded. Set to FALSE if aborted is TRUE." + - name: aborted + description: "Boolean flag indicating if the transaction was aborted." + - name: account + description: "Account address associated with the transaction." + - name: orig_status + description: "Original status of the account before the transaction." + - name: end_status + description: "End status of the account after the transaction." + - name: compute_success + description: "Boolean flag indicating if the computation was successful." + - name: compute_skipped + description: "Boolean flag indicating if the computation was skipped." + - name: compute_gas_fees + description: "Gas fees for the computation." + - name: action_result_code + description: "Result code for the action phase." + - name: action_success + description: "Boolean flag indicating if the action phase was successful." + - name: action_spec_actions + description: "Number of special actions performed." + - name: action_result_arg + description: "Argument for the action result." + - name: action_skipped_actions + description: "Number of actions that were skipped." + - name: action_valid + description: "Boolean flag indicating if the action was valid." + - name: action_tot_actions + description: "Total number of actions." + - name: action_no_funds + description: "Boolean flag indicating if there were insufficient funds for the action." + - name: action_status_change + description: "Status change during the action phase." + - name: compute_msg_state_used + description: "Boolean flag indicating if the message state was used during computation." + - name: descr + description: "Description or type of the transaction." + - name: block_workchain + description: "Workchain ID of the block containing this transaction." + - name: block_seqno + description: "Sequence number of the block containing this transaction." + - name: block_shard + description: "Shard ID of the block containing this transaction." + - name: mc_block_seqno + description: "Masterchain block sequence number referenced by this transaction." + - name: total_fees + description: "Total fees for the transaction." + - name: storage_fees_collected + description: "Storage fees collected during the transaction." + - name: credit_due_fees_collected + description: "Credit fees collected during the transaction." + - name: action_total_fwd_fees + description: "Total forward fees for actions." + - name: storage_fees_due + description: "Storage fees due for the transaction." + - name: action_total_action_fees + description: "Total fees for all actions." + - name: account_state_balance_before + description: "Account balance before the transaction." + - name: account_state_balance_after + description: "Account balance after the transaction." + - name: account_state_hash_before + description: "Hash of the account state before the transaction." + - name: account_state_hash_after + description: "Hash of the account state after the transaction." + - name: account_state_code_hash_before + description: "Hash of the account code before the transaction." + - name: account_state_code_hash_after + description: "Hash of the account code after the transaction." + - name: installed + description: "Boolean flag indicating if code was installed during the transaction." + - name: destroyed + description: "Boolean flag indicating if the account was destroyed during the transaction." + - name: is_tock + description: "Boolean flag indicating if this is a tick-tock transaction." + - name: credit_first + description: "Boolean flag indicating if credit was processed first." + - name: compute_account_activated + description: "Boolean flag indicating if the account was activated during computation." + - name: compute_vm_steps + description: "Number of VM steps executed during computation." + - name: compute_exit_arg + description: "Exit argument from the computation." + - name: compute_gas_credit + description: "Gas credit for the computation." + - name: compute_gas_limit + description: "Gas limit for the computation." + - name: compute_gas_used + description: "Gas used during the computation." + - name: compute_vm_init_state_hash + description: "Hash of the VM's initial state during computation." + - name: compute_vm_final_state_hash + description: "Hash of the VM's final state after computation." + - name: skipped_reason + description: "Reason for skipping the transaction if it was skipped." + - name: compute_exit_code + description: "Exit code from the computation." + - name: storage_status_change + description: "Storage status change during the transaction." + - name: compute_mode + description: "Mode of computation." + - name: credit + description: "Credit amount for the transaction." + - name: trace_id + description: "Trace ID for the transaction." + - name: lt + description: "Logical time of the transaction." + - name: prev_trans_lt + description: "Logical time of the previous transaction for the same account." + - name: now + description: "Unix timestamp when the transaction was processed." + - name: fact_transactions_id + description: "Unique identifier for the transaction record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] \ No newline at end of file diff --git a/models/gold/defi/gold_defi.yml b/models/gold/defi/gold_defi.yml new file mode 100644 index 0000000..02d4fd5 --- /dev/null +++ b/models/gold/defi/gold_defi.yml @@ -0,0 +1,112 @@ +version: 2 + +models: + - name: defi__fact_dex_pools + description: "Contains history of DEX pools states including static information about pools, reserves, and estimated TVL values. Each state includes information about the jettons in the pool, fees, and liquidity metrics." + columns: + - name: block_timestamp_last_updated + description: "Timestamp of when the pool was last updated, converted from last_updated Unix timestamp." + - name: block_timestamp_discovered_at + description: "Timestamp of when the pool was first discovered (first swap), converted from discovered_at Unix timestamp." + - name: project + description: "Project name of the DEX (e.g., ston.fi, dedust.io, megaton.fi, tonco)." + - name: pool + description: "Address of the pool." + - name: version + description: "Version of the project protocol." + - name: is_liquid + description: "Boolean flag indicating if the pool is liquid. A pool is considered liquid if it has TON, LSD, or a stable coin." + - name: reserves_right + description: "Raw amount of the right jetton in the pool." + - name: reserves_left + description: "Raw amount of the left jetton in the pool." + - name: jetton_right + description: "Address of the right jetton in the pool (fixed for each pool address, cannot be changed)." + - name: jetton_left + description: "Address of the left jetton in the pool (fixed for each pool address, cannot be changed)." + - name: total_supply + description: "Total supply of the pool LP-jetton. For TONCO, represents the number of active NFT positions." + - name: protocol_fee + description: "Fee percentage allocated to the protocol. Total fee equals lp_fee + protocol_fee + referral_fee (when referral address is present)." + - name: referral_fee + description: "Fee percentage allocated to referrals when a referral address is specified during a swap. For ston.fi v2, this value is always null." + - name: lp_fee + description: "Fee percentage allocated to liquidity providers." + - name: tvl_ton + description: "Total Value Locked in the pool denominated in TON. Null for pools with is_liquid=false." + - name: tvl_usd + description: "Total Value Locked in the pool denominated in USD. Null for pools with is_liquid=false." + - name: last_updated + description: "Unix timestamp of the pool's last update (swap or pool LP-jetton mint/burn)." + - name: discovered_at + description: "Unix timestamp of when the pool was first discovered through a swap." + - name: fact_dex_pools_id + description: "Unique identifier for the DEX pool record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp_last_updated + interval: 30 + severity: error + tags: ['test_recency'] + + - name: defi__fact_dex_trades + description: "Contains DEX and launchpad trading data, capturing swap events across different decentralized exchanges and launchpad platforms in the TON ecosystem." + columns: + - name: block_timestamp + description: "Timestamp of the block when the trade event occurred, converted from event_time." + - name: event_type + description: "Type of the event: 'trade' for normal DEX swaps or 'launch' for when liquidity is collected from a bonding curve and sent to DEX." + - name: tx_hash + description: "Transaction hash of the trade." + - name: project_type + description: "Type of project: 'dex' for classical AMM DEXs or 'launchpad' for bonding curve launchpads." + - name: project + description: "Name of the project (e.g., ston.fi, dedust.io, megaton.fi, tonco, ton.fun, gaspump)." + - name: pool_address + description: "Address of the pool where the trade occurred. Null if the pool is not known." + - name: version + description: "Version of the project protocol." + - name: trader_address + description: "Address of the trader who executed the swap." + - name: token_bought_address + description: "Address of the token that was bought. May include special wrapped TON aliases." + - name: token_sold_address + description: "Address of the token that was sold. May include special wrapped TON aliases." + - name: amount_bought_raw + description: "Raw amount of the token bought without dividing by 10^decimals. To get decimals, use the jetton_metadata table." + - name: amount_sold_raw + description: "Raw amount of the token sold without dividing by 10^decimals. To get decimals, use the jetton_metadata table." + - name: router_address + description: "Address of the router used for the trade. Null if the router is not used by the project." + - name: volume_ton + description: "Volume of the trade in TON. Calculated based on the amount of tokens traded if the trade involves TON, wrapped TON, USDT, USDC, or LSD (stTON, tsTON, hTON)." + - name: volume_usd + description: "Volume of the trade in USD. Calculated based on the amount of tokens traded if the trade involves TON, wrapped TON, USDT, USDC, or LSD." + - name: referral_address + description: "Referral address if specified. Null if the referral is not specified or not supported by the project." + - name: platform_tag + description: "Platform address. Null if the platform is not specified or not supported by the project." + - name: trace_id + description: "Trace ID for the transaction." + - name: query_id + description: "Query ID of the transaction. Null if query ID is not supported by the project." + - name: event_time + description: "Unix timestamp when the trade event occurred." + - name: fact_dex_trades_id + description: "Unique identifier for the DEX trade record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] \ No newline at end of file diff --git a/models/gold/nft/gold_nft.yml b/models/gold/nft/gold_nft.yml new file mode 100644 index 0000000..6cf4658 --- /dev/null +++ b/models/gold/nft/gold_nft.yml @@ -0,0 +1,282 @@ + +version: 2 + +models: + - name: nft__fact_nft_events + description: "Comprehensive data mart containing all NFT-related events including mints, transfers, sales, auctions, etc. Built on top of nft_items, nft_transfers, messages, and nft_sales tables." + columns: + - name: block_timestamp + description: "Timestamp of the block when the NFT event occurred, converted from timestamp." + - name: tx_hash + description: "Transaction hash associated with the NFT event, if applicable." + - name: type + description: "Type of NFT event: 'mint' (NFT deployment), 'put_on_sale' (NFT listed for sale), 'cancel_sale' (sale canceled), 'sale' (NFT sold), 'transfer' (direct NFT transfer), or 'bid' (auction bid)." + - name: sale_type + description: "Type of the sale: 'sale' for fixed price sales or 'auction' for auctions with bids." + - name: nft_item_address + description: "Address of the NFT item involved in the event." + - name: nft_item_index + description: "Index of the NFT item within its collection." + - name: payment_asset + description: "Asset type used for payment. Currently only 'TON' is supported." + - name: sale_price + description: "Price of the NFT in a sale event." + - name: forward_amount + description: "Amount of TON forwarded in transfer message, if related to the event." + - name: royalty_amount + description: "Amount of royalty in a sale event." + - name: marketplace_fee + description: "Fee charged by the marketplace in a sale event." + - name: auction_max_bid + description: "Maximum bid allowed in an auction." + - name: auction_min_bid + description: "Minimum bid required in an auction." + - name: sale_contract + description: "Address of the sale contract if related to the event." + - name: royalty_address + description: "Address receiving the royalty payment." + - name: marketplace_address + description: "Address of the marketplace facilitating the sale." + - name: marketplace_fee_address + description: "Address receiving the marketplace fee." + - name: owner_address + description: "Current owner address of the NFT." + - name: collection_address + description: "Address of the NFT collection. May be null for standalone NFTs." + - name: content_onchain + description: "NFT metadata extracted from on-chain data in JSON format." + - name: trace_id + description: "Trace ID for the transaction if applicable." + - name: query_id + description: "Query ID associated with the event if it exists." + - name: is_init + description: "Boolean flag indicating if the NFT is initialized." + - name: custom_payload + description: "Custom payload from the transfer message if related to the event." + - name: comment + description: "Text comment extracted from forward_payload." + - name: sale_end_time + description: "End time of the sale if applicable." + - name: forward_payload + description: "Payload of the forward message from transfer message if related to the event." + - name: auction_min_step + description: "Minimum step for bids in an auction." + - name: prev_owner + description: "Previous owner address. For mint events, this is the deployer address. For transfers, it's the previous owner. For sales, it's the seller. For bids, it's the bidder. Null for put_on_sale and cancel_sale events." + - name: timestamp + description: "Unix timestamp of the NFT event." + - name: lt + description: "Logical time of the NFT event." + - name: fact_nft_events_id + description: "Unique identifier for the NFT event record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + + - name: nft__fact_nft_items + description: "Contains full history of NFT item states according to TEP-62 NFT standard. Includes details about ownership and on-chain metadata." + columns: + - name: block_timestamp + description: "Timestamp of the block when the NFT state was recorded, converted from timestamp." + - name: address + description: "Address of the NFT item." + - name: index + description: "Index of the NFT item within its collection." + - name: collection_address + description: "Address of the NFT collection. May be null for standalone NFTs." + - name: owner_address + description: "Current owner address of the NFT item." + - name: content_onchain + description: "NFT metadata extracted from on-chain data in JSON format." + - name: is_init + description: "Boolean flag indicating if the NFT is initialized." + - name: lt + description: "Logical time of the NFT state update." + - name: timestamp + description: "Unix timestamp of the NFT state update." + - name: fact_nft_items_id + description: "Unique identifier for the NFT item record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + + - name: nft__fact_nft_metadata + description: "Contains NFT metadata for both items and collections, including on-chain and off-chain data according to TEP-64 standard." + columns: + - name: address + description: "NFT item or collection address." + - name: parent_address + description: "Collection owner address if type is 'collection', otherwise collection address for the item." + - name: update_timestamp_onchain + description: "Timestamp of the last on-chain update, converted from update_time_onchain." + - name: type + description: "Type of NFT entity: 'item' for individual NFTs or 'collection' for NFT collections." + - name: name + description: "TEP-64 NFT name from on-chain, off-chain, or tonapi source." + - name: description + description: "TEP-64 NFT description from on-chain, off-chain, or tonapi source." + - name: metadata_status + description: "Off-chain metadata update status: 0 - no off-chain metadata, 1 - success, -1 - error." + - name: attributes + description: "TEP-64 NFT attributes from on-chain, off-chain, or tonapi source. Only available for NFT items, not collections." + - name: image + description: "TEP-64 NFT image URL from on-chain, off-chain, or tonapi source." + - name: image_data + description: "TEP-64 NFT image data from on-chain, off-chain, or tonapi source." + - name: tonapi_image_url + description: "Cached image URL from tonapi." + - name: content_onchain + description: "JSON serialized string with on-chain metadata content." + - name: sources + description: "Record with sources of NFT metadata fields (name, description, image, image_data, attributes). Values can be 'onchain', 'offchain', 'tonapi', or '' (not set)." + - name: update_time_onchain + description: "Unix timestamp of on-chain update, such as admin address transfer." + - name: update_time_metadata + description: "Unix timestamp of off-chain metadata update." + - name: adding_at + description: "Date when the output file was created." + - name: fact_nft_metadata_id + description: "Unique identifier for the NFT metadata record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: update_timestamp_onchain + interval: 30 + severity: error + tags: ['test_recency'] + + - name: nft__fact_nft_sales + description: "Contains history of NFT sales contracts, including both fixed-price sales and auctions." + columns: + - name: block_timestamp + description: "Timestamp of the block when the NFT sale state was recorded, converted from timestamp." + - name: address + description: "Address of the NFT sales contract." + - name: nft_address + description: "Address of the NFT item being sold." + - name: type + description: "Type of sale: 'sale' for fixed price sales or 'auction' for auctions with bids." + - name: asset + description: "Asset type used for payment, currently only 'TON' is supported." + - name: price + description: "Price of the NFT or current bid for auction." + - name: marketplace_fee + description: "Fee amount charged by the marketplace." + - name: royalty_amount + description: "Amount of royalty to be paid to the royalty address." + - name: nft_owner_address + description: "Address of the NFT owner who created the sale." + - name: marketplace_address + description: "Address of the marketplace facilitating the sale." + - name: marketplace_fee_address + description: "Address receiving the marketplace fee." + - name: royalty_address + description: "Address receiving the royalty payment." + - name: is_complete + description: "Boolean flag indicating if the sale is complete." + - name: is_canceled + description: "Boolean flag indicating if the sale is canceled (only for auctions)." + - name: min_bid + description: "Minimum bid amount for an auction." + - name: max_bid + description: "Maximum bid amount for an auction." + - name: min_step + description: "Minimum step for bids in an auction." + - name: end_time + description: "Time of expiration of the sale." + - name: last_bid_at + description: "Timestamp of the last bid for an auction." + - name: last_member + description: "Address of the last bidder for an auction." + - name: created_at + description: "Timestamp of the NFT sales contract creation." + - name: timestamp + description: "Unix timestamp of the sale state update." + - name: lt + description: "Logical time of the sale state update." + - name: fact_nft_sales_id + description: "Unique identifier for the NFT sale record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] + + - name: nft__fact_nft_transfers + description: "Contains history of NFT transfers according to TEP-62 standard." + columns: + - name: block_timestamp + description: "Timestamp of the block when the NFT transfer occurred, converted from tx_now." + - name: tx_hash + description: "Transaction hash of the transfer." + - name: tx_succeeded + description: "Boolean flag indicating if the transfer transaction succeeded. Set to FALSE if tx_aborted is TRUE." + - name: tx_aborted + description: "Boolean flag indicating if the transfer transaction was aborted." + - name: old_owner + description: "Address of the previous owner of the NFT." + - name: new_owner + description: "Address of the new owner of the NFT." + - name: nft_collection_address + description: "Address of the NFT collection. May be null for standalone NFTs." + - name: nft_item_address + description: "Address of the NFT item being transferred." + - name: nft_item_index + description: "Index of the NFT item within its collection." + - name: response_destination + description: "Destination address for the response message. See TEP-62 standard." + - name: forward_amount + description: "Amount of TON forwarded with the transfer message." + - name: custom_payload + description: "Custom payload included in the transfer message. See TEP-62 standard." + - name: forward_payload + description: "Payload of the forward message. See TEP-62 standard." + - name: comment + description: "Text comment extracted from forward_payload." + - name: trace_id + description: "Trace ID from the transaction." + - name: query_id + description: "Query ID of the transfer." + - name: tx_now + description: "Unix timestamp of the transaction block." + - name: tx_lt + description: "Logical time of the transaction." + - name: fact_nft_transfers_id + description: "Unique identifier for the NFT transfer record." + - name: inserted_timestamp + description: "Timestamp when the record was inserted into the data warehouse." + - name: modified_timestamp + description: "Timestamp when the record was last modified in the data warehouse." + tests: + - dbt_utils.recency: + datepart: hour + field: block_timestamp + interval: 30 + severity: error + tags: ['test_recency'] diff --git a/models/silver/core/silver_core.yml b/models/silver/core/silver_core.yml index e6c0182..7dd5aa8 100644 --- a/models/silver/core/silver_core.yml +++ b/models/silver/core/silver_core.yml @@ -1,7 +1,541 @@ version: 2 models: + - name: silver__account_states + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: account + data_type: VARCHAR + + - name: silver__balances_history + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: lt + data_type: NUMBER + - name: silver__blocks config: - event_time: block_date - \ No newline at end of file + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: version + data_type: NUMBER + - name: created_by + data_type: VARCHAR + - name: end_lt + data_type: NUMBER + - name: want_merge + data_type: BOOLEAN + - name: gen_utime + data_type: TIMESTAMP_NTZ + - name: tx_count + data_type: NUMBER + - name: global_id + data_type: NUMBER + - name: root_hash + data_type: VARCHAR + - name: key_block + data_type: BOOLEAN + - name: mc_block_seqno + data_type: NUMBER + - name: vert_seqno_incr + data_type: NUMBER + - name: validator_list_hash_short + data_type: VARCHAR + - name: after_merge + data_type: BOOLEAN + - name: want_split + data_type: BOOLEAN + - name: after_split + data_type: BOOLEAN + - name: master_ref_seqno + data_type: NUMBER + - name: mc_block_workchain + data_type: NUMBER + - name: file_hash + data_type: VARCHAR + - name: prev_key_block_seqno + data_type: NUMBER + - name: shard + data_type: VARCHAR + - name: seqno + data_type: NUMBER + - name: vert_seqno + data_type: NUMBER + - name: flags + data_type: NUMBER + - name: rand_seed + data_type: VARCHAR + - name: gen_catchain_seqno + data_type: NUMBER + - name: min_ref_mc_seqno + data_type: NUMBER + - name: start_lt + data_type: NUMBER + - name: mc_block_shard + data_type: VARCHAR + - name: before_split + data_type: BOOLEAN + - name: workchain + data_type: NUMBER + - name: blocks_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__dex_pools + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: reserves_right + data_type: NUMBER + + - name: silver__dex_trades + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: volume_ton + data_type: NUMBER + + - name: silver__jetton_events + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: tx_hash + data_type: VARCHAR + + - name: silver__jetton_metadata + config: + contract: + enforced: true + columns: + - name: adding_date + data_type: DATE + - name: tonapi_image_url + data_type: VARCHAR + + + - name: silver__messages_with_data + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: body_boc + data_type: VARIANT + + - name: silver__nft_events + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: sale_price + data_type: NUMBER + - name: royalty_address + data_type: VARCHAR + - name: payment_asset + data_type: VARCHAR + - name: marketplace_fee_address + data_type: VARCHAR + - name: owner_address + data_type: VARCHAR + - name: collection_address + data_type: VARCHAR + - name: content_onchain + data_type: VARIANT + - name: trace_id + data_type: VARCHAR + - name: sale_contract + data_type: VARCHAR + - name: forward_amount + data_type: NUMBER + - name: nft_item_index + data_type: NUMBER + - name: query_id + data_type: VARCHAR + - name: is_init + data_type: BOOLEAN + - name: timestamp + data_type: TIMESTAMP_NTZ + - name: nft_item_address + data_type: VARCHAR + - name: custom_payload + data_type: VARIANT + - name: comment + data_type: VARCHAR + - name: sale_end_time + data_type: TIMESTAMP_NTZ + - name: sale_type + data_type: VARCHAR + - name: auction_max_bid + data_type: NUMBER + - name: auction_min_bid + data_type: NUMBER + - name: marketplace_address + data_type: VARCHAR + - name: forward_payload + data_type: VARIANT + - name: royalty_amount + data_type: NUMBER + - name: auction_min_step + data_type: NUMBER + - name: type + data_type: VARCHAR + - name: prev_owner + data_type: VARCHAR + - name: tx_hash + data_type: VARCHAR + - name: marketplace_fee + data_type: NUMBER + - name: lt + data_type: NUMBER + - name: nft_events_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__nft_items + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: collection_address + data_type: VARCHAR + - name: is_init + data_type: BOOLEAN + - name: lt + data_type: NUMBER + - name: timestamp + data_type: TIMESTAMP_NTZ + - name: address + data_type: VARCHAR + - name: owner_address + data_type: VARCHAR + - name: index + data_type: NUMBER + - name: content_onchain + data_type: VARIANT + - name: nft_items_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__nft_metadata + config: + contract: + enforced: true + columns: + - name: adding_date + data_type: DATE + - name: description + data_type: VARCHAR + - name: image + data_type: VARCHAR + - name: metadata_status + data_type: VARCHAR + - name: parent_address + data_type: VARCHAR + - name: update_time_metadata + data_type: TIMESTAMP_NTZ + - name: adding_at + data_type: TIMESTAMP_NTZ + - name: update_time_onchain + data_type: TIMESTAMP_NTZ + - name: address + data_type: VARCHAR + - name: tonapi_image_url + data_type: VARCHAR + - name: content_onchain + data_type: VARIANT + - name: type + data_type: VARCHAR + - name: attributes + data_type: VARIANT + - name: name + data_type: VARCHAR + - name: sources + data_type: VARIANT + - name: image_data + data_type: VARIANT + - name: nft_metadata_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__nft_sales + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: is_canceled + data_type: BOOLEAN + - name: marketplace_fee_address + data_type: VARCHAR + - name: end_time + data_type: TIMESTAMP_NTZ + - name: is_complete + data_type: BOOLEAN + - name: last_member + data_type: VARCHAR + - name: marketplace_address + data_type: VARCHAR + - name: royalty_amount + data_type: NUMBER + - name: created_at + data_type: TIMESTAMP_NTZ + - name: nft_address + data_type: VARCHAR + - name: marketplace_fee + data_type: NUMBER + - name: asset + data_type: VARCHAR + - name: price + data_type: NUMBER + - name: nft_owner_address + data_type: VARCHAR + - name: address + data_type: VARCHAR + - name: min_bid + data_type: NUMBER + - name: timestamp + data_type: TIMESTAMP_NTZ + - name: royalty_address + data_type: VARCHAR + - name: min_step + data_type: NUMBER + - name: max_bid + data_type: NUMBER + - name: last_bid_at + data_type: TIMESTAMP_NTZ + - name: lt + data_type: NUMBER + - name: type + data_type: VARCHAR + - name: nft_sales_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__nft_transfers + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: trace_id + data_type: VARCHAR + - name: tx_now + data_type: TIMESTAMP_NTZ + - name: custom_payload + data_type: VARIANT + - name: new_owner + data_type: VARCHAR + - name: forward_payload + data_type: VARIANT + - name: comment + data_type: VARCHAR + - name: old_owner + data_type: VARCHAR + - name: tx_aborted + data_type: BOOLEAN + - name: query_id + data_type: VARCHAR + - name: tx_hash + data_type: VARCHAR + - name: tx_lt + data_type: NUMBER + - name: response_destination + data_type: VARCHAR + - name: nft_collection_address + data_type: VARCHAR + - name: forward_amount + data_type: NUMBER + - name: nft_item_address + data_type: VARCHAR + - name: nft_item_index + data_type: NUMBER + - name: nft_transfers_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR + + - name: silver__transactions + config: + contract: + enforced: true + columns: + - name: block_date + data_type: DATE + - name: compute_success + data_type: BOOLEAN + - name: total_fees + data_type: NUMBER + - name: orig_status + data_type: VARCHAR + - name: compute_skipped + data_type: BOOLEAN + - name: compute_gas_fees + data_type: NUMBER + - name: action_result_code + data_type: NUMBER + - name: destroyed + data_type: BOOLEAN + - name: action_success + data_type: BOOLEAN + - name: compute_msg_state_used + data_type: BOOLEAN + - name: is_tock + data_type: BOOLEAN + - name: account_state_hash_after + data_type: VARCHAR + - name: action_spec_actions + data_type: NUMBER + - name: descr + data_type: VARCHAR + - name: account_state_balance_before + data_type: NUMBER + - name: hash + data_type: VARCHAR + - name: action_result_arg + data_type: NUMBER + - name: aborted + data_type: BOOLEAN + - name: mc_block_seqno + data_type: NUMBER + - name: compute_account_activated + data_type: BOOLEAN + - name: action_skipped_actions + data_type: NUMBER + - name: now + data_type: TIMESTAMP_NTZ + - name: credit_due_fees_collected + data_type: NUMBER + - name: block_shard + data_type: VARCHAR + - name: end_status + data_type: VARCHAR + - name: credit_first + data_type: BOOLEAN + - name: prev_trans_hash + data_type: VARCHAR + - name: block_workchain + data_type: NUMBER + - name: account + data_type: VARCHAR + - name: compute_vm_steps + data_type: NUMBER + - name: storage_fees_collected + data_type: NUMBER + - name: compute_exit_arg + data_type: NUMBER + - name: action_valid + data_type: BOOLEAN + - name: action_status_change + data_type: VARCHAR + - name: installed + data_type: BOOLEAN + - name: prev_trans_lt + data_type: NUMBER + - name: compute_gas_credit + data_type: NUMBER + - name: compute_gas_limit + data_type: NUMBER + - name: skipped_reason + data_type: VARCHAR + - name: action_total_fwd_fees + data_type: NUMBER + - name: account_state_code_hash_before + data_type: VARCHAR + - name: account_state_hash_before + data_type: VARCHAR + - name: compute_exit_code + data_type: NUMBER + - name: trace_id + data_type: VARCHAR + - name: block_seqno + data_type: NUMBER + - name: storage_status_change + data_type: VARCHAR + - name: lt + data_type: NUMBER + - name: compute_mode + data_type: VARCHAR + - name: credit + data_type: NUMBER + - name: storage_fees_due + data_type: NUMBER + - name: compute_gas_used + data_type: NUMBER + - name: account_state_code_hash_after + data_type: VARCHAR + - name: action_total_action_fees + data_type: NUMBER + - name: compute_vm_init_state_hash + data_type: VARCHAR + - name: account_state_balance_after + data_type: NUMBER + - name: action_tot_actions + data_type: NUMBER + - name: compute_vm_final_state_hash + data_type: VARCHAR + - name: action_no_funds + data_type: BOOLEAN + - name: transactions_id + data_type: VARCHAR + - name: inserted_timestamp + data_type: TIMESTAMP_NTZ + - name: modified_timestamp + data_type: TIMESTAMP_NTZ + - name: _invocation_id + data_type: VARCHAR diff --git a/package-lock.yml b/package-lock.yml index 10b641c..6c3daa5 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -2,7 +2,7 @@ packages: - package: calogica/dbt_expectations version: 0.8.5 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: d3cf679e079f0cf06142de9386f215e55fe26b3b + revision: 042a1bcb4cd9781a54ea3c6fc1669963e4d279c7 - package: get-select/dbt_snowflake_query_tags version: 2.5.0 - package: dbt-labs/dbt_external_tables @@ -11,6 +11,6 @@ packages: version: 1.0.0 - package: calogica/dbt_date version: 0.7.2 - - git: https://github.com/FlipsideCrypto/livequery-models.git - revision: b024188be4e9c6bc00ed77797ebdc92d351d620e -sha1_hash: 733d4f1fb94f4356106bab2f9af580d0898b3b50 + - git: https://github.com/FlipsideCrypto/livequery-base.git + revision: d60977ef9abf27435cd61146a3681f8fab0d3a3a +sha1_hash: d4e0215e9c307ca861d34b896318e3ef07f08f58 diff --git a/packages.yml b/packages.yml index 2ab7d3b..0ba6989 100644 --- a/packages.yml +++ b/packages.yml @@ -2,7 +2,7 @@ packages: - package: calogica/dbt_expectations version: [">=0.4.0", "<0.9.0"] - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.32.0 + revision: v1.34.1 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] - package: dbt-labs/dbt_external_tables