From cbeafae80d2fad17db9fae2e61d27986bc7ea397 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Tue, 29 Apr 2025 15:54:49 -0400 Subject: [PATCH] checks --- dbt_project.yml | 7 +++ .../tables/core__fact_transfers.md | 2 +- models/gold/core/core__dim_tokens.sql | 51 +++++++++++++++++-- models/gold/core/core__dim_tokens.yml | 3 ++ models/gold/core/core__fact_transfers.sql | 41 +++++++++++++-- models/gold/core/core__fact_transfers.yml | 2 + .../core/silver__transfers_fungible.sql | 4 +- 7 files changed, 97 insertions(+), 13 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 1f40e03..b07d1a5 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -101,6 +101,13 @@ vars: - AWS_LAMBDA_APTOS_API - INTERNAL_DEV prod: + API_INTEGRATION: aws_aptos_api_prod_v2 + EXTERNAL_FUNCTION_URI: mxus3semvi.execute-api.us-east-1.amazonaws.com/prod/ + ROLES: + - AWS_LAMBDA_APTOS_API + - DBT_CLOUD_APTOS + - INTERNAL_DEV + prod-2xl: API_INTEGRATION: aws_aptos_api_prod_v2 EXTERNAL_FUNCTION_URI: mxus3semvi.execute-api.us-east-1.amazonaws.com/prod/ ROLES: diff --git a/models/descriptions/tables/core__fact_transfers.md b/models/descriptions/tables/core__fact_transfers.md index 785cc7d..307130b 100644 --- a/models/descriptions/tables/core__fact_transfers.md +++ b/models/descriptions/tables/core__fact_transfers.md @@ -1,5 +1,5 @@ {% docs core__fact_transfers %} -This table contains Deposit and Withdraw events on the aptos blockchain. Note: transfers with a 0 amount are excluded. +This table contains Deposit and Withdraw events from the coin module as well as Deposit, Withdraw, DepositEvent, and WithdrawEvent from the fungible_asset module on the aptos blockchain. Note: transfers with a 0 amount are excluded. {% enddocs %} diff --git a/models/gold/core/core__dim_tokens.sql b/models/gold/core/core__dim_tokens.sql index 9fda51d..d115e44 100644 --- a/models/gold/core/core__dim_tokens.sql +++ b/models/gold/core/core__dim_tokens.sql @@ -1,6 +1,10 @@ {{ config( - materialized = 'view', - tags = ['noncore'] + materialized = 'incremental', + unique_key = ['token_address'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(token_address,symbol);", + tags = ['core','full_test'] ) }} SELECT @@ -12,10 +16,47 @@ SELECT creator_address, transaction_created_timestamp, transaction_version_created, - coin_info_id AS dim_token_id, - inserted_timestamp, - modified_timestamp + {{ dbt_utils.generate_surrogate_key( + ['token_address'] + ) }} AS dim_token_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, FROM {{ ref( 'silver__coin_info' ) }} + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION ALL +SELECT + token_address, + NAME, + symbol, + decimals, + NULL AS coin_type_hash, + NULL AS creator_address, + NULL AS transaction_created_timestamp, + NULL AS transaction_version_created, + {{ dbt_utils.generate_surrogate_key( + ['token_address'] + ) }} AS dim_token_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, +FROM + {{ ref('silver__fungible_asset_metadata') }} + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} diff --git a/models/gold/core/core__dim_tokens.yml b/models/gold/core/core__dim_tokens.yml index 06b6a3b..a2b8b40 100644 --- a/models/gold/core/core__dim_tokens.yml +++ b/models/gold/core/core__dim_tokens.yml @@ -6,6 +6,9 @@ models: columns: - name: TOKEN_ADDRESS description: '{{ doc("token_address") }}' + data_tests: + - not_null + - unique - name: NAME description: 'The full token name' - name: SYMBOL diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql index 9a4c172..742900c 100644 --- a/models/gold/core/core__fact_transfers.sql +++ b/models/gold/core/core__fact_transfers.sql @@ -20,13 +20,44 @@ SELECT account_address, amount, token_address, + FALSE AS is_fungible, + NULL :: STRING AS store_address, transfers_id AS fact_transfers_id, - inserted_timestamp, - modified_timestamp + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, FROM - {{ ref( - 'silver__transfers' - ) }} + {{ ref('silver__transfers') }} +WHERE + amount <> 0 + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +UNION ALL +SELECT + block_number, + block_timestamp, + tx_hash, + version, + success, + event_index, + NULL AS creation_number, + transfer_event, + owner_address AS account_address, + amount, + metadata_address AS token_address, + TRUE AS is_fungible, + store_address, + transfers_fungible_id AS fact_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, +FROM + {{ ref('silver__transfers_fungible') }} WHERE amount <> 0 diff --git a/models/gold/core/core__fact_transfers.yml b/models/gold/core/core__fact_transfers.yml index 9c3be55..1599c99 100644 --- a/models/gold/core/core__fact_transfers.yml +++ b/models/gold/core/core__fact_transfers.yml @@ -26,6 +26,8 @@ models: description: '{{ doc("amount") }}' - name: TOKEN_ADDRESS description: '{{ doc("token_address") }}' + - name: IS_FUNGIBLE + description: 'Boolean indicating if the transfer used the legacy coin transfer mechanism or the fungible_asset module' - name: FACT_TRANSFERS_ID description: '{{ doc("pk") }}' tests: diff --git a/models/silver/core/silver__transfers_fungible.sql b/models/silver/core/silver__transfers_fungible.sql index 0e7ac0c..7b71145 100644 --- a/models/silver/core/silver__transfers_fungible.sql +++ b/models/silver/core/silver__transfers_fungible.sql @@ -136,11 +136,11 @@ SELECT WHEN event_resource IN ( 'WithdrawEvent', 'Withdraw' - ) THEN 'Withdraw' + ) THEN 'WithdrawEvent' WHEN event_resource IN ( 'DepositEvent', 'Deposit' - ) THEN 'Deposit' + ) THEN 'DepositEvent' END AS transfer_event, e.store_address, COALESCE(