From 77416958bf8d4b5d0adcda01a1df7340bac58123 Mon Sep 17 00:00:00 2001 From: jacksan Date: Tue, 27 May 2025 15:33:25 -0600 Subject: [PATCH] move union from silver to fact. force amount_unadj to str --- models/descriptions/transfer_action.md | 15 +++++++++++++++ models/gold/core/core__ez_token_transfers.sql | 7 ++++--- .../core/core__fact_token_transfers.sql} | 16 +++++++++++++--- .../core/core__fact_token_transfers.yml} | 9 +++++++-- .../native/silver__token_transfer_deposit.sql | 2 +- ...silver__token_transfer_ft_transfers_event.sql | 2 +- ...ilver__token_transfer_ft_transfers_method.sql | 10 +++++++--- ...ilver__token_transfer_ft_transfers_method.yml | 7 +++++++ .../silver__token_transfer_liquidity.sql | 2 +- .../non_native/silver__token_transfer_mints.sql | 2 +- .../non_native/silver__token_transfer_orders.sql | 2 +- 11 files changed, 58 insertions(+), 16 deletions(-) create mode 100644 models/descriptions/transfer_action.md rename models/{silver/transfers/silver__token_transfers_complete.sql => gold/core/core__fact_token_transfers.sql} (92%) rename models/{silver/transfers/silver__token_transfers_complete.yml => gold/core/core__fact_token_transfers.yml} (92%) diff --git a/models/descriptions/transfer_action.md b/models/descriptions/transfer_action.md new file mode 100644 index 0000000..1aee4d1 --- /dev/null +++ b/models/descriptions/transfer_action.md @@ -0,0 +1,15 @@ +{% docs transfer_action %} + +The event that caused the transfer. + +- Transfer: A standard transfer Action of tokens from one address to another. +- Deposit: A deposit Action of tokens into a contract. +- ft_transfer: A transfer of tokens using the ft_transfer method. +- ft_transfer_call: A transfer of tokens using the ft_transfer_call method. +- ft_mint: A minting of tokens using the ft_mint method. +- order_added: An order being added to the orderbook. +- add_liquidity: Adding liquidity to a pool. +- near_deposit: A deposit of NEAR tokens into the wrap.near contract. +- near_withdraw: A withdrawal of NEAR tokens from the wrap.near contract. + +{% enddocs %} diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index 24168a8..e88b28c 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -15,7 +15,7 @@ SELECT MIN(DATE_TRUNC('day', block_timestamp)) AS block_timestamp_day FROM - {{ ref('silver__token_transfers_complete') }} + {{ ref('core__fact_token_transfers') }} WHERE modified_timestamp >= ( SELECT @@ -93,9 +93,10 @@ SELECT C.symbol AS symbol, price AS token_price, transfer_type, - transfers_complete_id AS ez_token_transfers_id + transfer_action, + fact_token_transfers_id AS ez_token_transfers_id FROM - {{ ref('silver__token_transfers_complete') }} + {{ ref('core__fact_token_transfers') }} t ASOF JOIN hourly_prices p MATCH_CONDITION (t.block_timestamp >= p.HOUR) diff --git a/models/silver/transfers/silver__token_transfers_complete.sql b/models/gold/core/core__fact_token_transfers.sql similarity index 92% rename from models/silver/transfers/silver__token_transfers_complete.sql rename to models/gold/core/core__fact_token_transfers.sql index d31c292..72901b9 100644 --- a/models/silver/transfers/silver__token_transfers_complete.sql +++ b/models/gold/core/core__fact_token_transfers.sql @@ -1,9 +1,10 @@ {{ config( materialized = 'incremental', + unique_key = 'fact_token_transfers_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - unique_key = 'transfers_complete_id', incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate_custom","block_timestamp::date"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,contract_address,from_address,to_address,fact_token_transfers_id);", tags = ['scheduled_non_core'] ) }} @@ -64,6 +65,7 @@ WITH native_transfers AS ( NULL AS memo, amount_unadj :: STRING AS amount_unadj, 'native' AS transfer_type, + 'Transfer' AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -89,6 +91,7 @@ native_deposits AS ( NULL AS memo, amount_unadj :: STRING AS amount_unadj, 'native' AS transfer_type, + 'Deposit' AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -114,6 +117,7 @@ ft_transfers_method AS ( memo, amount_unadj :: STRING AS amount_unadj, 'nep141' AS transfer_type, + method_name AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -138,6 +142,7 @@ ft_transfers_event AS ( memo, amount_unadj :: STRING AS amount_unadj, 'nep141' AS transfer_type, + 'ft_transfer' AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -162,6 +167,7 @@ mints AS ( memo, amount_unadj :: STRING AS amount_unadj, 'nep141' AS transfer_type, + 'ft_mint' AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -186,6 +192,7 @@ orders AS ( memo, amount_unadj :: STRING AS amount_unadj, 'nep141' AS transfer_type, + 'order_added' AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -210,6 +217,7 @@ liquidity AS ( memo, amount_unadj :: STRING AS amount_unadj, 'nep141' AS transfer_type, + 'add_liquidity' AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -234,6 +242,7 @@ wrapped_near AS ( memo, amount_unadj :: STRING AS amount_unadj, 'nep141' AS transfer_type, + method_name AS transfer_action, _partition_by_block_number, modified_timestamp FROM @@ -275,10 +284,11 @@ final_transfers AS ( memo, amount_unadj, transfer_type, + transfer_action, _partition_by_block_number, {{ dbt_utils.generate_surrogate_key( ['action_id', 'contract_address', 'amount_unadj', 'from_address', 'to_address', 'rn'] - ) }} AS transfers_complete_id, + ) }} AS fact_token_transfers_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id @@ -290,4 +300,4 @@ final_transfers AS ( ) SELECT * FROM final_transfers -QUALIFY(ROW_NUMBER() OVER (PARTITION BY transfers_complete_id ORDER BY modified_timestamp DESC)) = 1 +QUALIFY(ROW_NUMBER() OVER (PARTITION BY fact_token_transfers_id ORDER BY modified_timestamp DESC)) = 1 diff --git a/models/silver/transfers/silver__token_transfers_complete.yml b/models/gold/core/core__fact_token_transfers.yml similarity index 92% rename from models/silver/transfers/silver__token_transfers_complete.yml rename to models/gold/core/core__fact_token_transfers.yml index f2a9373..311d031 100644 --- a/models/silver/transfers/silver__token_transfers_complete.yml +++ b/models/gold/core/core__fact_token_transfers.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: silver__token_transfers_complete + - name: core__fact_token_transfers description: |- This table records all Native Token and Fungible Token (FT) transfers on the NEAR blockchain. It combines data from multiple sources including native transfers, deposits, NEP-141 method calls, events, mints, orders, and liquidity operations. @@ -59,10 +59,15 @@ models: - accepted_values: values: ['native', 'nep141'] + - name: TRANSFER_ACTION + description: "{{ doc('transfer_action')}}" + tests: + - not_null + - name: _PARTITION_BY_BLOCK_NUMBER description: "{{ doc('_partition_by_block_number')}}" - - name: TRANSFERS_COMPLETE_ID + - name: FACT_TOKEN_TRANSFERS_ID description: "{{doc('id')}}" tests: - not_null diff --git a/models/silver/transfers/native/silver__token_transfer_deposit.sql b/models/silver/transfers/native/silver__token_transfer_deposit.sql index 4bb6e25..a5d6ea6 100644 --- a/models/silver/transfers/native/silver__token_transfer_deposit.sql +++ b/models/silver/transfers/native/silver__token_transfer_deposit.sql @@ -18,7 +18,7 @@ WITH transfers AS ( receipt_predecessor_id AS predecessor_id, receipt_signer_id AS signer_id, receipt_receiver_id AS receiver_id, - action_data :deposit :: INT AS amount_unadj, + action_data :deposit :: STRING AS amount_unadj, receipt_succeeded, _partition_by_block_number FROM diff --git a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.sql b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.sql index e58cfa4..1ae165d 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.sql +++ b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_event.sql @@ -58,7 +58,7 @@ ft_transfers_final AS ( f.value :new_owner_id, f.value :owner_id ) :: STRING AS to_address, - f.value :amount :: variant AS amount_unadj, + f.value :amount :: STRING AS amount_unadj, f.value :memo :: STRING AS memo, log_index + f.index AS event_index, receipt_succeeded, diff --git a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql index 334500d..60be30c 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql +++ b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.sql @@ -74,6 +74,7 @@ WITH ft_transfer_actions AS ( receipt_predecessor_id AS predecessor_id, receipt_signer_id AS signer_id, receipt_succeeded, + action_data :method_name :: STRING AS method_name, _partition_by_block_number, modified_timestamp FROM @@ -126,7 +127,8 @@ ft_transfer_logs AS ( a.predecessor_id, a.signer_id, a.action_index, - a.receipt_succeeded + a.receipt_succeeded, + a.method_name FROM logs l INNER JOIN ft_transfer_actions a @@ -168,11 +170,12 @@ ft_transfers_final AS ( REGEXP_SUBSTR( log_value, '\\d+' - ) :: variant AS amount_unadj, + ) :: STRING AS amount_unadj, '' AS memo, log_index + action_index AS event_index, _partition_by_block_number, - receipt_succeeded + receipt_succeeded, + method_name FROM ft_transfer_logs WHERE @@ -194,6 +197,7 @@ SELECT amount_unadj, memo, event_index AS rn, + method_name, receipt_succeeded, _partition_by_block_number, {{ dbt_utils.generate_surrogate_key( diff --git a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.yml b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.yml index 451825d..1edf1e8 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.yml +++ b/models/silver/transfers/non_native/silver__token_transfer_ft_transfers_method.yml @@ -68,6 +68,13 @@ models: - dbt_expectations.expect_column_values_to_be_of_type: column_type: NUMBER + - name: METHOD_NAME + description: "{{doc('method_name')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + - name: RECEIPT_SUCCEEDED description: "{{doc('receipt_succeeded')}}" tests: diff --git a/models/silver/transfers/non_native/silver__token_transfer_liquidity.sql b/models/silver/transfers/non_native/silver__token_transfer_liquidity.sql index 6a21e27..253b764 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_liquidity.sql +++ b/models/silver/transfers/non_native/silver__token_transfer_liquidity.sql @@ -63,7 +63,7 @@ add_liquidity AS ( 1, 'e', 1 - ) :: variant AS amount_unadj, + ) :: STRING AS amount_unadj, 'add_liquidity' AS memo, log_index + INDEX AS event_index, predecessor_id, diff --git a/models/silver/transfers/non_native/silver__token_transfer_mints.sql b/models/silver/transfers/non_native/silver__token_transfer_mints.sql index 8e30a7e..9f45f9b 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_mints.sql +++ b/models/silver/transfers/non_native/silver__token_transfer_mints.sql @@ -57,7 +57,7 @@ ft_mints_final AS ( f.value :new_owner_id, f.value :owner_id ) :: STRING AS to_address, - f.value :amount :: variant AS amount_unadj, + f.value :amount :: STRING AS amount_unadj, f.value :memo :: STRING AS memo, log_index + f.index AS event_index, _partition_by_block_number, diff --git a/models/silver/transfers/non_native/silver__token_transfer_orders.sql b/models/silver/transfers/non_native/silver__token_transfer_orders.sql index c42d16e..64e2493 100644 --- a/models/silver/transfers/non_native/silver__token_transfer_orders.sql +++ b/models/silver/transfers/non_native/silver__token_transfer_orders.sql @@ -52,7 +52,7 @@ orders_final AS ( log_index, f.value :sell_token :: STRING AS contract_address, f.value :owner_id :: STRING AS from_address, - f.value :original_amount :: variant AS amount_unadj, + f.value :original_amount :: STRING AS amount_unadj, 'order' AS memo, log_index + f.index AS event_index, _partition_by_block_number,