From 27683afe1c660abf113131f74ab7152ecdb298bc Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Tue, 15 Apr 2025 16:46:44 -0400 Subject: [PATCH] docs update wf, tests --- .github/workflows/dbt_test_daily.yml | 47 +++++ models/descriptions/__overview__.md | 77 ++++++++ models/gold/core/gold_core.yml | 261 +++++++++++++++++++++++++-- models/gold/defi/gold_defi.yml | 56 ++++++ models/gold/nft/gold_nft.yml | 145 ++++++++++++++- 5 files changed, 569 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/dbt_test_daily.yml create mode 100644 models/descriptions/__overview__.md diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml new file mode 100644 index 0000000..57adcb9 --- /dev/null +++ b/.github/workflows/dbt_test_daily.yml @@ -0,0 +1,47 @@ +name: dbt_test_daily +run-name: dbt_test_daily + +on: + workflow_dispatch: + branches: + - "main" + schedule: + # an hour after the daily TON incremental job kicks off + - cron: "35 4 * * *" + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m tag:test_recency tag:test_quality --vars '{"TEST_HOURS_THRESHOLD":30}' diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md new file mode 100644 index 0000000..5b99a5e --- /dev/null +++ b/models/descriptions/__overview__.md @@ -0,0 +1,77 @@ +{% docs __overview__ %} + +# Welcome to the Flipside Crypto TON Models Documentation + +## **What does this documentation cover?** +The documentation included here details the design of the TON blockchain tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/ton-models/) + +## **How do I use these docs?** +The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. + +If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them. + +There is more information on how to use dbt docs in the last section of this document. + +## **Quick Links to Table Documentation** + +**Click on the links below to jump to the documentation for each schema.** + +### Core Fact Tables (`ton`.`CORE`.``) +- [fact_account_states](#!/model/model.ton_models.core__fact_account_states) +- [fact_balances](#!/model/model.ton_models.core__fact_balances) +- [fact_blocks](#!/model/model.ton_models.core__fact_blocks) +- [fact_jetton_events](#!/model/model.ton_models.core__fact_jetton_events) +- [fact_jetton_metadata](#!/model/model.ton_models.core__fact_jetton_metadata) +- [fact_messages](#!/model/model.ton_models.core__fact_messages) +- [fact_transactions](#!/model/model.ton_models.core__fact_transactions) + +### DeFi Tables (`ton`.`DEFI`.``) +- [fact_dex_pools](#!/model/model.ton_models.defi__fact_dex_pools) +- [fact_dex_trades](#!/model/model.ton_models.defi__fact_dex_trades) + +### NFT Tables (`ton`.`NFT`.``) +- [fact_nft_events](#!/model/model.ton_models.nft__fact_nft_events) +- [fact_nft_items](#!/model/model.ton_models.nft__fact_nft_items) +- [fact_nft_metadata](#!/model/model.ton_models.nft__fact_nft_metadata) +- [fact_nft_sales](#!/model/model.ton_models.nft__fact_nft_sales) +- [fact_nft_transfers](#!/model/model.ton_models.nft__fact_nft_transfers) + + +The TON models are built using three layers of SQL models: **bronze, silver, and gold (or core/defi/nft).** + +- Bronze: Data is loaded in from the source as a view +- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here +- Gold (or core/defi/nft): Final views and tables that are available publicly + +The dimension tables are sourced from a variety of on-chain and off-chain sources. + +Convenience views (denoted ez_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. + +## **Using dbt docs** +### Navigation + +You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project. + +### Database Tab + +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database. + +### Graph Exploration + +You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models. + +On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. + +Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). + +Note that you can also right-click on models to interactively filter and explore the graph. + + +### **More information** +- [Flipside](https://flipsidecrypto.xyz/) +- [Studio](https://flipsidecrypto.xyz/studio/) +- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) +- [Github](https://github.com/FlipsideCrypto/ton-models) +- [What is dbt?](https://docs.getdbt.com/docs/introduction) + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/gold_core.yml b/models/gold/core/gold_core.yml index 4048836..bfed48a 100644 --- a/models/gold/core/gold_core.yml +++ b/models/gold/core/gold_core.yml @@ -6,28 +6,43 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the account state was recorded, converted from Unix timestamp." + data_type: timestamp_ntz - name: account description: "TON account address in base64 format." + data_type: varchar - name: timestamp description: "Unix timestamp when the account state was recorded." + data_type: number(38,0) - name: last_trans_lt description: "Logical time of the last transaction associated with this account." + data_type: number(38,0) - name: last_tx_hash description: "Hash of the last transaction associated with this account." + data_type: varchar - name: account_status description: "Current status of the account (active, uninitialized, or frozen)." + data_type: varchar - name: balance description: "Current balance of the account in nanoTONs." + data_type: number(38,0) - name: frozen_hash description: "Hash of the frozen state if the account is frozen, otherwise null." + data_type: varchar - name: account_state_hash description: "Hash of the account state." + data_type: varchar - name: fact_account_states_id description: "Unique identifier for the account state record." + data_type: varchar - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -41,22 +56,37 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the balance was recorded, converted from Unix timestamp." + data_type: timestamp_ntz - name: address description: "Address of the asset owner." + data_type: varchar - name: asset description: "Asset type, 'TON' for native TON or jetton address for Jetton balance." + data_type: varchar - name: amount description: "Balance amount. For TON, this is in nanoTONs. For Jettons, this is the raw amount before dividing by 10^decimals." + data_type: number(38,0) - name: mintless_claimed description: "Boolean flag indicating if the mintless jetton was claimed (only applicable for mintless jetton wallets)." + data_type: boolean - name: lt description: "Logical time of the balance update." + data_type: number(38,0) - name: fact_balances_history_id description: "Unique identifier for the balance history record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -70,72 +100,112 @@ models: columns: - name: block_timestamp description: "Timestamp of the block, converted from gen_utime." + data_type: timestamp_ntz - name: gen_utime description: "Unix timestamp when the block was generated." + data_type: number(38,0) - name: workchain description: "Workchain ID where the block was created (usually -1 for masterchain, 0 for base workchain)." + data_type: number(38,0) - name: version description: "Version of the block." + data_type: number(38,0) - name: shard description: "Shard identifier for the block." + data_type: NUMBER - name: seqno description: "Block sequence number within its workchain." + data_type: number(38,0) - name: vert_seqno description: "Vertical sequence number for the block." + data_type: number(38,0) - name: start_lt description: "Starting logical time of the block." + data_type: number(38,0) - name: end_lt description: "Ending logical time of the block." + data_type: number(38,0) - name: mc_block_seqno description: "Masterchain block sequence number referenced by this block." + data_type: number(38,0) - name: mc_block_shard description: "Masterchain block shard referenced by this block." + data_type: NUMBER - name: tx_count description: "Number of transactions in the block." + data_type: number(38,0) - name: global_id description: "Global identifier for the TON network." + data_type: number(38,0) - name: created_by description: "Validator that created the block." + data_type: varchar - name: want_merge description: "Flag indicating if the block wants to be merged." + data_type: boolean - name: root_hash description: "Root hash of the block." + data_type: varchar - name: key_block description: "Boolean flag indicating if this is a key block." + data_type: boolean - name: vert_seqno_incr description: "Increment of vertical sequence number." + data_type: BOOLEAN - name: validator_list_hash_short description: "Short hash of the validator list." + data_type: number(38,0) - name: after_merge description: "Flag indicating if the block was created after a merge." + data_type: boolean - name: want_split description: "Flag indicating if the block wants to be split." + data_type: boolean - name: after_split description: "Flag indicating if the block was created after a split." + data_type: boolean - name: master_ref_seqno description: "Masterchain reference sequence number." + data_type: number(38,0) - name: mc_block_workchain description: "Masterchain block workchain referenced by this block." + data_type: number(38,0) - name: file_hash description: "Hash of the block file." + data_type: varchar - name: prev_key_block_seqno description: "Sequence number of the previous key block." + data_type: number(38,0) - name: flags description: "Flags associated with the block." + data_type: number(38,0) - name: rand_seed description: "Random seed used in block generation." + data_type: varchar - name: gen_catchain_seqno description: "Catchain sequence number used in block generation." + data_type: number(38,0) - name: min_ref_mc_seqno description: "Minimum referenced masterchain sequence number." + data_type: number(38,0) - name: before_split description: "Flag indicating if the block was created before a split." + data_type: boolean - name: fact_blocks_id description: "Unique identifier for the block record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -143,58 +213,90 @@ models: interval: 30 severity: error tags: ['test_recency'] - - dbt_utils.sequential_values: - column_name: seqno - interval: 1 - config: - severity: error - error_if: ">1" - tags: ['test_recency'] - - dbt_utils.sequential_values: - column_name: mc_block_seqno - interval: 1 - config: - severity: error - error_if: ">1" - tags: ['test_recency'] + # - dbt_utils.sequential_values: + # column_name: seqno + # interval: 1 + # group_by_columns: + # - workchain + # - shard + # where: workchain = 0 + # config: + # severity: error + # error_if: ">1" + # tags: ['test_recency'] + # - dbt_utils.sequential_values: + # column_name: mc_block_seqno + # interval: 1 + # group_by_columns: + # - mc_block_seqno + # config: + # severity: error + # error_if: ">1" + # tags: ['test_recency'] - name: core__fact_jetton_events description: "Contains information about Jetton events such as transfers, burns, and mints according to TEP-74 standard." columns: - name: block_timestamp description: "Timestamp of the block when the jetton event occurred, converted from utime." + data_type: timestamp_ntz - name: tx_hash description: "Transaction hash of the event." + data_type: varchar - name: tx_succeeded description: "Boolean flag indicating if the transaction succeeded. Set to FALSE if tx_aborted is TRUE." + data_type: boolean - name: type description: "Event type: 'transfer' for TEP-74 transfer, 'burn' for TEP-74 burn, or 'mint' for jetton minting." + data_type: varchar - name: source description: "Source address of the tokens. Null for mint events." + data_type: varchar - name: destination description: "Destination address for the tokens. Null for burn events." + data_type: varchar - name: forward_ton_amount description: "Amount of TON forwarded with the transaction, stored as a raw decimal value." + data_type: number(38,0) - name: amount description: "Amount of jettons involved in the event, stored as a raw value without dividing by 10^decimals." - - name: jetton + data_type: number(38,0) + - name: jetton_master description: "Address of the jetton master contract." + data_type: varchar + - name: jetton_wallet + description: "Address of the jetton wallet contract." + data_type: varchar - name: comment description: "Optional comment attached to the transaction." + data_type: varchar - name: query_id description: "Query ID of the transaction, stored as a raw decimal value." + data_type: number(38,0) - name: tx_lt description: "Logical time of the transaction." + data_type: number(38,0) - name: utime description: "Unix timestamp when the event occurred." + data_type: number(38,0) - name: tx_aborted description: "Boolean flag indicating if the transaction was aborted. If true, the event should be discarded." + data_type: boolean - name: fact_jetton_events_id description: "Unique identifier for the jetton event record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -208,48 +310,76 @@ models: columns: - name: address description: "Jetton master address." + data_type: varchar - name: update_timestamp_onchain description: "Timestamp of the last on-chain update, converted from update_time_onchain." + data_type: timestamp_ntz - name: symbol description: "TEP-64 jetton symbol from on-chain, off-chain, or tonapi source." + data_type: varchar - name: decimals description: "TEP-64 jetton decimals from on-chain, off-chain, or tonapi source. If not present, defaults to 9." + data_type: number(38,0) - name: name description: "TEP-64 jetton name from on-chain, off-chain, or tonapi source." + data_type: varchar - name: description description: "TEP-64 jetton description from on-chain, off-chain, or tonapi source." + data_type: varchar - name: tonapi_image_url description: "Cached image URL from tonapi." + data_type: varchar - name: image_data description: "TEP-64 jetton image data from on-chain, off-chain, or tonapi source." + data_type: varchar - name: image description: "TEP-64 jetton image URL from on-chain, off-chain, or tonapi source." + data_type: varchar - name: admin_address description: "Admin address from on-chain data." + data_type: varchar - name: mintable description: "Boolean flag indicating if the jetton is mintable based on on-chain data." + data_type: boolean - name: jetton_content_onchain description: "JSON serialized string with on-chain jetton content." + data_type: varchar - name: jetton_wallet_code_hash description: "Jetton wallet code hash from on-chain data." + data_type: varchar - name: code_hash description: "Jetton code hash from on-chain data." + data_type: varchar - name: adding_at description: "Date when the output file was created." + data_type: number - name: sources description: "Record with sources of jetton metadata fields (symbol, name, description, image, image_data, decimals). Values can be 'onchain', 'offchain', 'tonapi', or '' (not set)." + data_type: variant - name: metadata_status description: "Off-chain metadata update status: 0 - no off-chain metadata, 1 - success, -1 - error." + data_type: number(38,0) - name: update_time_onchain description: "Unix timestamp of on-chain update, such as admin address transfer." + data_type: number(38,0) - name: update_time_metadata description: "Unix timestamp of off-chain metadata update." + data_type: number(38,0) - name: fact_jetton_metadata_id description: "Unique identifier for the jetton metadata record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -263,58 +393,91 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the message was processed, converted from tx_now." + data_type: timestamp_ntz - name: tx_hash description: "Transaction hash associated with the message." + data_type: varchar - name: msg_hash description: "Hash of the message." + data_type: varchar - name: body_hash description: "Hash of the message body." + data_type: varchar - name: trace_id description: "Trace ID for the message within the transaction." + data_type: varchar - name: direction description: "Direction of the message: 'in' for messages that initiated a transaction, 'out' for messages resulting from a transaction." + data_type: varchar - name: source description: "Source address of the message." + data_type: varchar - name: destination description: "Destination address of the message." + data_type: varchar - name: value description: "Amount of TON attached to the message." + data_type: number(38,0) - name: opcode description: "Operation code extracted from the message body." + data_type: number(38,0) - name: created_at description: "Timestamp when the message was created." + data_type: number - name: tx_now description: "Unix timestamp of the transaction processing." + data_type: number(38,0) - name: ihr_fee description: "Instant hypercube routing fee." + data_type: number(38,0) - name: import_fee description: "Import fee for the message." + data_type: number(38,0) - name: fwd_fee description: "Forward fee for the message." + data_type: number(38,0) - name: ihr_disabled description: "Boolean flag indicating if instant hypercube routing is disabled for this message." + data_type: boolean - name: bounced description: "Boolean flag indicating if the message was bounced." + data_type: boolean - name: bounce description: "Boolean flag indicating if the message can bounce if delivery fails." + data_type: boolean - name: comment description: "Text comment extracted from the message body if available." + data_type: varchar - name: tx_lt description: "Logical time of the transaction that processed this message." + data_type: number(38,0) - name: created_lt description: "Logical time when the message was created." + data_type: number(38,0) - name: init_state_hash description: "Hash of the initial state if present." + data_type: varchar - name: init_state_boc description: "Raw initial state BOC (Bag of Cells) if present." + data_type: binary - name: body_boc description: "Raw body BOC (Bag of Cells) of the message." + data_type: binary - name: fact_messages_id description: "Unique identifier for the message record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -328,128 +491,196 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the transaction was processed, converted from now." + data_type: timestamp_ntz - name: tx_hash description: "Hash of the transaction." + data_type: varchar - name: prev_tx_hash description: "Hash of the previous transaction for the same account." + data_type: varchar - name: tx_succeeded description: "Boolean flag indicating if the transaction succeeded. Set to FALSE if aborted is TRUE." + data_type: boolean - name: aborted description: "Boolean flag indicating if the transaction was aborted." + data_type: boolean - name: account description: "Account address associated with the transaction." + data_type: varchar - name: orig_status description: "Original status of the account before the transaction." + data_type: varchar - name: end_status description: "End status of the account after the transaction." + data_type: varchar - name: compute_success description: "Boolean flag indicating if the computation was successful." + data_type: boolean - name: compute_skipped description: "Boolean flag indicating if the computation was skipped." + data_type: boolean - name: compute_gas_fees description: "Gas fees for the computation." + data_type: number(38,0) - name: action_result_code description: "Result code for the action phase." + data_type: number(38,0) - name: action_success description: "Boolean flag indicating if the action phase was successful." + data_type: boolean - name: action_spec_actions description: "Number of special actions performed." + data_type: number(38,0) - name: action_result_arg description: "Argument for the action result." + data_type: number(38,0) - name: action_skipped_actions description: "Number of actions that were skipped." + data_type: number(38,0) - name: action_valid description: "Boolean flag indicating if the action was valid." + data_type: boolean - name: action_tot_actions description: "Total number of actions." + data_type: number(38,0) - name: action_no_funds description: "Boolean flag indicating if there were insufficient funds for the action." + data_type: boolean - name: action_status_change description: "Status change during the action phase." + data_type: varchar - name: compute_msg_state_used description: "Boolean flag indicating if the message state was used during computation." + data_type: boolean - name: descr description: "Description or type of the transaction." + data_type: varchar - name: block_workchain description: "Workchain ID of the block containing this transaction." + data_type: number(38,0) - name: block_seqno description: "Sequence number of the block containing this transaction." + data_type: number(38,0) - name: block_shard description: "Shard ID of the block containing this transaction." + data_type: number - name: mc_block_seqno description: "Masterchain block sequence number referenced by this transaction." + data_type: number(38,0) - name: total_fees description: "Total fees for the transaction." + data_type: number(38,0) - name: storage_fees_collected description: "Storage fees collected during the transaction." + data_type: number(38,0) - name: credit_due_fees_collected description: "Credit fees collected during the transaction." + data_type: number(38,0) - name: action_total_fwd_fees description: "Total forward fees for actions." + data_type: number(38,0) - name: storage_fees_due description: "Storage fees due for the transaction." + data_type: number(38,0) - name: action_total_action_fees description: "Total fees for all actions." + data_type: number(38,0) - name: account_state_balance_before description: "Account balance before the transaction." + data_type: number(38,0) - name: account_state_balance_after description: "Account balance after the transaction." + data_type: number(38,0) - name: account_state_hash_before description: "Hash of the account state before the transaction." + data_type: varchar - name: account_state_hash_after description: "Hash of the account state after the transaction." + data_type: varchar - name: account_state_code_hash_before description: "Hash of the account code before the transaction." + data_type: varchar - name: account_state_code_hash_after description: "Hash of the account code after the transaction." + data_type: varchar - name: installed description: "Boolean flag indicating if code was installed during the transaction." + data_type: boolean - name: destroyed description: "Boolean flag indicating if the account was destroyed during the transaction." + data_type: boolean - name: is_tock description: "Boolean flag indicating if this is a tick-tock transaction." + data_type: boolean - name: credit_first description: "Boolean flag indicating if credit was processed first." + data_type: boolean - name: compute_account_activated description: "Boolean flag indicating if the account was activated during computation." + data_type: boolean - name: compute_vm_steps description: "Number of VM steps executed during computation." + data_type: number(38,0) - name: compute_exit_arg description: "Exit argument from the computation." + data_type: number(38,0) - name: compute_gas_credit description: "Gas credit for the computation." + data_type: number(38,0) - name: compute_gas_limit description: "Gas limit for the computation." + data_type: number(38,0) - name: compute_gas_used description: "Gas used during the computation." + data_type: number(38,0) - name: compute_vm_init_state_hash description: "Hash of the VM's initial state during computation." + data_type: varchar - name: compute_vm_final_state_hash description: "Hash of the VM's final state after computation." + data_type: varchar - name: skipped_reason description: "Reason for skipping the transaction if it was skipped." + data_type: varchar - name: compute_exit_code description: "Exit code from the computation." + data_type: number(38,0) - name: storage_status_change description: "Storage status change during the transaction." + data_type: varchar - name: compute_mode description: "Mode of computation." + data_type: number(38,0) - name: credit description: "Credit amount for the transaction." + data_type: number(38,0) - name: trace_id description: "Trace ID for the transaction." + data_type: varchar - name: lt description: "Logical time of the transaction." + data_type: number(38,0) - name: prev_trans_lt description: "Logical time of the previous transaction for the same account." + data_type: number(38,0) - name: now description: "Unix timestamp when the transaction was processed." + data_type: number(38,0) - name: fact_transactions_id description: "Unique identifier for the transaction record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour diff --git a/models/gold/defi/gold_defi.yml b/models/gold/defi/gold_defi.yml index 02d4fd5..4b33392 100644 --- a/models/gold/defi/gold_defi.yml +++ b/models/gold/defi/gold_defi.yml @@ -6,46 +6,73 @@ models: columns: - name: block_timestamp_last_updated description: "Timestamp of when the pool was last updated, converted from last_updated Unix timestamp." + data_type: timestamp_ntz - name: block_timestamp_discovered_at description: "Timestamp of when the pool was first discovered (first swap), converted from discovered_at Unix timestamp." + data_type: timestamp_ntz - name: project description: "Project name of the DEX (e.g., ston.fi, dedust.io, megaton.fi, tonco)." + data_type: varchar - name: pool description: "Address of the pool." + data_type: varchar - name: version description: "Version of the project protocol." + data_type: INT - name: is_liquid description: "Boolean flag indicating if the pool is liquid. A pool is considered liquid if it has TON, LSD, or a stable coin." + data_type: boolean - name: reserves_right description: "Raw amount of the right jetton in the pool." + data_type: number - name: reserves_left description: "Raw amount of the left jetton in the pool." + data_type: number - name: jetton_right description: "Address of the right jetton in the pool (fixed for each pool address, cannot be changed)." + data_type: varchar - name: jetton_left description: "Address of the left jetton in the pool (fixed for each pool address, cannot be changed)." + data_type: varchar - name: total_supply description: "Total supply of the pool LP-jetton. For TONCO, represents the number of active NFT positions." + data_type: number - name: protocol_fee description: "Fee percentage allocated to the protocol. Total fee equals lp_fee + protocol_fee + referral_fee (when referral address is present)." + data_type: BIGINT - name: referral_fee description: "Fee percentage allocated to referrals when a referral address is specified during a swap. For ston.fi v2, this value is always null." + data_type: BIGINT - name: lp_fee description: "Fee percentage allocated to liquidity providers." + data_type: BIGINT - name: tvl_ton description: "Total Value Locked in the pool denominated in TON. Null for pools with is_liquid=false." + data_type: BIGINT - name: tvl_usd description: "Total Value Locked in the pool denominated in USD. Null for pools with is_liquid=false." + data_type: BIGINT - name: last_updated description: "Unix timestamp of the pool's last update (swap or pool LP-jetton mint/burn)." + data_type: number - name: discovered_at description: "Unix timestamp of when the pool was first discovered through a swap." + data_type: number - name: fact_dex_pools_id description: "Unique identifier for the DEX pool record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -59,50 +86,79 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the trade event occurred, converted from event_time." + data_type: timestamp_ntz - name: event_type description: "Type of the event: 'trade' for normal DEX swaps or 'launch' for when liquidity is collected from a bonding curve and sent to DEX." + data_type: varchar - name: tx_hash description: "Transaction hash of the trade." + data_type: varchar - name: project_type description: "Type of project: 'dex' for classical AMM DEXs or 'launchpad' for bonding curve launchpads." + data_type: varchar - name: project description: "Name of the project (e.g., ston.fi, dedust.io, megaton.fi, tonco, ton.fun, gaspump)." + data_type: varchar - name: pool_address description: "Address of the pool where the trade occurred. Null if the pool is not known." + data_type: varchar - name: version description: "Version of the project protocol." + data_type: INT - name: trader_address description: "Address of the trader who executed the swap." + data_type: varchar - name: token_bought_address description: "Address of the token that was bought. May include special wrapped TON aliases." + data_type: varchar - name: token_sold_address description: "Address of the token that was sold. May include special wrapped TON aliases." + data_type: varchar - name: amount_bought_raw description: "Raw amount of the token bought without dividing by 10^decimals. To get decimals, use the jetton_metadata table." + data_type: number - name: amount_sold_raw description: "Raw amount of the token sold without dividing by 10^decimals. To get decimals, use the jetton_metadata table." + data_type: number - name: router_address description: "Address of the router used for the trade. Null if the router is not used by the project." + data_type: varchar - name: volume_ton description: "Volume of the trade in TON. Calculated based on the amount of tokens traded if the trade involves TON, wrapped TON, USDT, USDC, or LSD (stTON, tsTON, hTON)." + data_type: BIGINT - name: volume_usd description: "Volume of the trade in USD. Calculated based on the amount of tokens traded if the trade involves TON, wrapped TON, USDT, USDC, or LSD." + data_type: BIGINT - name: referral_address description: "Referral address if specified. Null if the referral is not specified or not supported by the project." + data_type: varchar - name: platform_tag description: "Platform address. Null if the platform is not specified or not supported by the project." + data_type: varchar - name: trace_id description: "Trace ID for the transaction." + data_type: varchar - name: query_id description: "Query ID of the transaction. Null if query ID is not supported by the project." + data_type: BIGINT - name: event_time description: "Unix timestamp when the trade event occurred." + data_type: number - name: fact_dex_trades_id description: "Unique identifier for the DEX trade record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour diff --git a/models/gold/nft/gold_nft.yml b/models/gold/nft/gold_nft.yml index 6cf4658..b1771cc 100644 --- a/models/gold/nft/gold_nft.yml +++ b/models/gold/nft/gold_nft.yml @@ -1,4 +1,3 @@ - version: 2 models: @@ -7,72 +6,112 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the NFT event occurred, converted from timestamp." + data_type: timestamp_ntz - name: tx_hash description: "Transaction hash associated with the NFT event, if applicable." + data_type: varchar - name: type description: "Type of NFT event: 'mint' (NFT deployment), 'put_on_sale' (NFT listed for sale), 'cancel_sale' (sale canceled), 'sale' (NFT sold), 'transfer' (direct NFT transfer), or 'bid' (auction bid)." + data_type: varchar - name: sale_type description: "Type of the sale: 'sale' for fixed price sales or 'auction' for auctions with bids." + data_type: varchar - name: nft_item_address description: "Address of the NFT item involved in the event." + data_type: varchar - name: nft_item_index description: "Index of the NFT item within its collection." + data_type: text - name: payment_asset description: "Asset type used for payment. Currently only 'TON' is supported." + data_type: varchar - name: sale_price description: "Price of the NFT in a sale event." + data_type: number - name: forward_amount description: "Amount of TON forwarded in transfer message, if related to the event." + data_type: number - name: royalty_amount description: "Amount of royalty in a sale event." + data_type: number - name: marketplace_fee description: "Fee charged by the marketplace in a sale event." + data_type: number - name: auction_max_bid description: "Maximum bid allowed in an auction." + data_type: number - name: auction_min_bid description: "Minimum bid required in an auction." + data_type: number - name: sale_contract description: "Address of the sale contract if related to the event." + data_type: varchar - name: royalty_address description: "Address receiving the royalty payment." + data_type: varchar - name: marketplace_address description: "Address of the marketplace facilitating the sale." + data_type: varchar - name: marketplace_fee_address description: "Address receiving the marketplace fee." + data_type: varchar - name: owner_address description: "Current owner address of the NFT." + data_type: varchar - name: collection_address description: "Address of the NFT collection. May be null for standalone NFTs." + data_type: varchar - name: content_onchain description: "NFT metadata extracted from on-chain data in JSON format." + data_type: text - name: trace_id description: "Trace ID for the transaction if applicable." + data_type: varchar - name: query_id description: "Query ID associated with the event if it exists." + data_type: number - name: is_init description: "Boolean flag indicating if the NFT is initialized." + data_type: boolean - name: custom_payload description: "Custom payload from the transfer message if related to the event." + data_type: binary - name: comment description: "Text comment extracted from forward_payload." + data_type: varchar - name: sale_end_time description: "End time of the sale if applicable." + data_type: number - name: forward_payload description: "Payload of the forward message from transfer message if related to the event." + data_type: binary - name: auction_min_step description: "Minimum step for bids in an auction." + data_type: number - name: prev_owner description: "Previous owner address. For mint events, this is the deployer address. For transfers, it's the previous owner. For sales, it's the seller. For bids, it's the bidder. Null for put_on_sale and cancel_sale events." + data_type: varchar - name: timestamp description: "Unix timestamp of the NFT event." + data_type: number - name: lt description: "Logical time of the NFT event." + data_type: number - name: fact_nft_events_id description: "Unique identifier for the NFT event record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -86,28 +125,46 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the NFT state was recorded, converted from timestamp." + data_type: timestamp_ntz - name: address description: "Address of the NFT item." + data_type: varchar - name: index description: "Index of the NFT item within its collection." + data_type: text - name: collection_address description: "Address of the NFT collection. May be null for standalone NFTs." + data_type: varchar - name: owner_address description: "Current owner address of the NFT item." + data_type: varchar - name: content_onchain description: "NFT metadata extracted from on-chain data in JSON format." + data_type: text - name: is_init description: "Boolean flag indicating if the NFT is initialized." + data_type: boolean - name: lt description: "Logical time of the NFT state update." + data_type: number - name: timestamp description: "Unix timestamp of the NFT state update." + data_type: number - name: fact_nft_items_id description: "Unique identifier for the NFT item record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -121,42 +178,67 @@ models: columns: - name: address description: "NFT item or collection address." + data_type: varchar - name: parent_address description: "Collection owner address if type is 'collection', otherwise collection address for the item." + data_type: varchar - name: update_timestamp_onchain description: "Timestamp of the last on-chain update, converted from update_time_onchain." + data_type: timestamp_ntz - name: type description: "Type of NFT entity: 'item' for individual NFTs or 'collection' for NFT collections." + data_type: varchar - name: name description: "TEP-64 NFT name from on-chain, off-chain, or tonapi source." + data_type: varchar - name: description description: "TEP-64 NFT description from on-chain, off-chain, or tonapi source." + data_type: varchar - name: metadata_status description: "Off-chain metadata update status: 0 - no off-chain metadata, 1 - success, -1 - error." + data_type: number - name: attributes description: "TEP-64 NFT attributes from on-chain, off-chain, or tonapi source. Only available for NFT items, not collections." + data_type: text - name: image description: "TEP-64 NFT image URL from on-chain, off-chain, or tonapi source." + data_type: varchar - name: image_data description: "TEP-64 NFT image data from on-chain, off-chain, or tonapi source." + data_type: varchar - name: tonapi_image_url description: "Cached image URL from tonapi." + data_type: varchar - name: content_onchain description: "JSON serialized string with on-chain metadata content." + data_type: text - name: sources description: "Record with sources of NFT metadata fields (name, description, image, image_data, attributes). Values can be 'onchain', 'offchain', 'tonapi', or '' (not set)." + data_type: VARIANT - name: update_time_onchain description: "Unix timestamp of on-chain update, such as admin address transfer." + data_type: number - name: update_time_metadata description: "Unix timestamp of off-chain metadata update." + data_type: number - name: adding_at description: "Date when the output file was created." + data_type: number - name: fact_nft_metadata_id description: "Unique identifier for the NFT metadata record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -170,56 +252,88 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the NFT sale state was recorded, converted from timestamp." + data_type: timestamp_ntz - name: address description: "Address of the NFT sales contract." + data_type: varchar - name: nft_address description: "Address of the NFT item being sold." + data_type: varchar - name: type description: "Type of sale: 'sale' for fixed price sales or 'auction' for auctions with bids." + data_type: varchar - name: asset description: "Asset type used for payment, currently only 'TON' is supported." + data_type: varchar - name: price description: "Price of the NFT or current bid for auction." + data_type: number - name: marketplace_fee description: "Fee amount charged by the marketplace." + data_type: number - name: royalty_amount description: "Amount of royalty to be paid to the royalty address." + data_type: number - name: nft_owner_address description: "Address of the NFT owner who created the sale." + data_type: varchar - name: marketplace_address description: "Address of the marketplace facilitating the sale." + data_type: varchar - name: marketplace_fee_address description: "Address receiving the marketplace fee." + data_type: varchar - name: royalty_address description: "Address receiving the royalty payment." + data_type: varchar - name: is_complete description: "Boolean flag indicating if the sale is complete." + data_type: boolean - name: is_canceled description: "Boolean flag indicating if the sale is canceled (only for auctions)." + data_type: boolean - name: min_bid description: "Minimum bid amount for an auction." + data_type: number - name: max_bid description: "Maximum bid amount for an auction." + data_type: number - name: min_step description: "Minimum step for bids in an auction." + data_type: number - name: end_time description: "Time of expiration of the sale." + data_type: number - name: last_bid_at description: "Timestamp of the last bid for an auction." + data_type: number - name: last_member description: "Address of the last bidder for an auction." + data_type: varchar - name: created_at description: "Timestamp of the NFT sales contract creation." + data_type: number - name: timestamp description: "Unix timestamp of the sale state update." + data_type: number - name: lt description: "Logical time of the sale state update." + data_type: number - name: fact_nft_sales_id description: "Unique identifier for the NFT sale record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour @@ -233,50 +347,77 @@ models: columns: - name: block_timestamp description: "Timestamp of the block when the NFT transfer occurred, converted from tx_now." + data_type: timestamp_ntz - name: tx_hash description: "Transaction hash of the transfer." + data_type: varchar - name: tx_succeeded description: "Boolean flag indicating if the transfer transaction succeeded. Set to FALSE if tx_aborted is TRUE." + data_type: boolean - name: tx_aborted description: "Boolean flag indicating if the transfer transaction was aborted." + data_type: boolean - name: old_owner description: "Address of the previous owner of the NFT." + data_type: varchar - name: new_owner description: "Address of the new owner of the NFT." + data_type: varchar - name: nft_collection_address description: "Address of the NFT collection. May be null for standalone NFTs." + data_type: varchar - name: nft_item_address description: "Address of the NFT item being transferred." + data_type: varchar - name: nft_item_index description: "Index of the NFT item within its collection." + data_type: text - name: response_destination description: "Destination address for the response message. See TEP-62 standard." + data_type: varchar - name: forward_amount description: "Amount of TON forwarded with the transfer message." + data_type: number - name: custom_payload description: "Custom payload included in the transfer message. See TEP-62 standard." + data_type: binary - name: forward_payload description: "Payload of the forward message. See TEP-62 standard." + data_type: binary - name: comment description: "Text comment extracted from forward_payload." + data_type: varchar - name: trace_id description: "Trace ID from the transaction." + data_type: varchar - name: query_id description: "Query ID of the transfer." + data_type: number - name: tx_now description: "Unix timestamp of the transaction block." + data_type: number - name: tx_lt description: "Logical time of the transaction." + data_type: number - name: fact_nft_transfers_id description: "Unique identifier for the NFT transfer record." + data_type: varchar + data_tests: + - unique: + tags: ['test_quality'] - name: inserted_timestamp description: "Timestamp when the record was inserted into the data warehouse." + data_type: timestamp_ntz - name: modified_timestamp description: "Timestamp when the record was last modified in the data warehouse." + data_type: timestamp_ntz + config: + contract: + enforced: true tests: - dbt_utils.recency: datepart: hour field: block_timestamp interval: 30 severity: error - tags: ['test_recency'] + tags: ['test_recency'] \ No newline at end of file