From d336bff30e4a3efbc32b22363a7eb8429ebfea8f Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 28 Oct 2025 13:23:59 -0600 Subject: [PATCH] add ft_burns model to transfers --- .../gold/core/core__fact_token_transfers.sql | 29 ++++++ .../silver__token_transfer_burns.sql | 90 ++++++++++++++++++ .../silver__token_transfer_burns.yml | 95 +++++++++++++++++++ .../nearblocks/streamline__ft_tokenlist.sql | 18 ++++ 4 files changed, 232 insertions(+) create mode 100644 models/silver/transfers/non_native/silver__token_transfer_burns.sql create mode 100644 models/silver/transfers/non_native/silver__token_transfer_burns.yml diff --git a/models/gold/core/core__fact_token_transfers.sql b/models/gold/core/core__fact_token_transfers.sql index 37b79e2..06cf8b6 100644 --- a/models/gold/core/core__fact_token_transfers.sql +++ b/models/gold/core/core__fact_token_transfers.sql @@ -40,6 +40,8 @@ SELECT MIN(block_timestamp) block_timestamp FROM {{ ref('silver__token_transfer_liquidity') }} WHERE modified_timestamp >= '{{max_mod}}' UNION ALL SELECT MIN(block_timestamp) block_timestamp FROM {{ ref('silver__token_transfer_wrapped_near') }} WHERE modified_timestamp >= '{{max_mod}}' + UNION ALL + SELECT MIN(block_timestamp) block_timestamp FROM {{ ref('silver__token_transfer_burns') }} WHERE modified_timestamp >= '{{max_mod}}' ) {% endset %} @@ -177,6 +179,31 @@ mints AS ( WHERE block_timestamp::DATE >= '{{min_bd}}' {% endif %} ), +burns AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + receipt_id, + rn, + contract_address, + from_address, + to_address, + memo, + amount_unadj AS amount_unadj, + 'nep141' AS transfer_type, + 'ft_burn' AS transfer_action, + receipt_succeeded, + _partition_by_block_number, + modified_timestamp + FROM + {{ ref('silver__token_transfer_burns') }} + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} + {% elif is_incremental() %} + WHERE block_timestamp::DATE >= '{{min_bd}}' + {% endif %} +), orders AS ( SELECT block_id, @@ -263,6 +290,8 @@ all_transfers AS ( UNION ALL SELECT * FROM mints UNION ALL + SELECT * FROM burns + UNION ALL SELECT * FROM orders UNION ALL SELECT * FROM liquidity diff --git a/models/silver/transfers/non_native/silver__token_transfer_burns.sql b/models/silver/transfers/non_native/silver__token_transfer_burns.sql new file mode 100644 index 0000000..72363d6 --- /dev/null +++ b/models/silver/transfers/non_native/silver__token_transfer_burns.sql @@ -0,0 +1,90 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate_custom","block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE', 'modified_timestamp::DATE'], + unique_key = 'burn_id', + tags = ['scheduled_non_core'] +) }} + +WITH ft_burn_logs AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + receipt_id, + receiver_id AS contract_address, + predecessor_id, + signer_id, + log_index, + try_parse_json(clean_log) AS log_data, + receipt_succeeded, + _partition_by_block_number + FROM + {{ ref('silver__logs_s3') }} + WHERE + is_standard -- Only look at EVENT_JSON formatted logs + AND try_parse_json(clean_log) :event :: STRING = 'ft_burn' + + {% if var("MANUAL_FIX") %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + AND modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} +), +ft_burns_final AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + receipt_id, + contract_address, + predecessor_id, + signer_id, + NVL( + f.value :owner_id, + NULL + ) :: STRING AS from_address, + NULL AS to_address, + f.value :amount :: STRING AS amount_unadj, + f.value :memo :: STRING AS memo, + log_index + f.index AS event_index, + _partition_by_block_number, + receipt_succeeded + FROM + ft_burn_logs, + LATERAL FLATTEN( + input => log_data :data + ) f +) +SELECT + block_timestamp, + block_id, + tx_hash, + receipt_id AS action_id, + receipt_id, + contract_address, + from_address, + to_address, + amount_unadj, + memo, + event_index AS rn, + predecessor_id, + receipt_succeeded, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['receipt_id', 'contract_address', 'amount_unadj', 'to_address', 'rn'] + ) }} AS burn_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + ft_burns_final diff --git a/models/silver/transfers/non_native/silver__token_transfer_burns.yml b/models/silver/transfers/non_native/silver__token_transfer_burns.yml new file mode 100644 index 0000000..0ad84a8 --- /dev/null +++ b/models/silver/transfers/non_native/silver__token_transfer_burns.yml @@ -0,0 +1,95 @@ +version: 2 + +models: + - name: silver__token_transfer_burns + description: > + This model captures fungible token (FT) burn events from the NEAR blockchain. + It processes standard EVENT_JSON formatted logs from token contracts that emit + 'ft_burn' events. + + columns: + - name: BLOCK_TIMESTAMP + description: "{{doc('block_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: TIMESTAMP_NTZ + + - name: BLOCK_ID + description: "{{doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: TX_HASH + description: "{{doc('tx_hash')}}" + tests: + - not_null + + - name: RECEIPT_ID + description: "{{doc('receipt_id')}}" + + - name: CONTRACT_ADDRESS + description: "{{doc('contract_address')}}" + tests: + - not_null + + - name: FROM_ADDRESS + description: "{{doc('from_address')}}" + + - name: TO_ADDRESS + description: "{{doc('to_address')}}" + tests: + - not_null + + - name: AMOUNT_UNADJ + description: "{{doc('amount_unadj')}}" + tests: + - not_null` + + - name: MEMO + description: "{{doc('memo')}}" + tests: + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: RN + description: "{{doc('event_index')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: NUMBER + + - name: PREDECESSOR_ID + description: "{{doc('predecessor_id')}}" + + - name: RECEIPT_SUCCEEDED + description: "{{doc('receipt_succeeded')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: BOOLEAN + + - name: _PARTITION_BY_BLOCK_NUMBER + description: "{{doc('_partition_by_block_number')}}" + + - name: BURN_ID + description: "{{doc('id')}}" + tests: + - not_null + - unique + + - name: INSERTED_TIMESTAMP + description: "{{doc('inserted_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: TIMESTAMP_NTZ + + - name: MODIFIED_TIMESTAMP + description: "{{doc('modified_timestamp')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: TIMESTAMP_NTZ diff --git a/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql b/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql index 936467f..6a4b3c8 100644 --- a/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql +++ b/models/streamline/external/token_metadata/nearblocks/streamline__ft_tokenlist.sql @@ -38,11 +38,29 @@ AND modified_timestamp >= ( ) {% endif %} , +mints AS ( + SELECT + DISTINCT lower(contract_address) AS contract_address + FROM + {{ ref('silver__token_transfer_mints') }} +{% if is_incremental() %} +WHERE modified_timestamp >= ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01') + FROM + {{ this }}) + {% endif %} +), FINAL AS ( SELECT contract_address FROM ft_transfers + UNION + SELECT + contract_address + FROM + mints {% if not is_incremental() %} UNION