From a15c3a6828559c183b22c3026c8c33c9cfa183b7 Mon Sep 17 00:00:00 2001 From: eric-laurello <102970824+eric-laurello@users.noreply.github.com> Date: Thu, 3 Apr 2025 10:50:47 -0400 Subject: [PATCH] AN-5883 add trust lines (#11) * add trust lines * ov --- models/bronze/core/bronze__trust_lines.sql | 10 ++ models/bronze/core/bronze__trust_lines_FR.sql | 10 ++ models/descriptions/__overview__.md | 1 + models/descriptions/ledger_key.md | 3 + .../descriptions/tables/fact_trust_lines.md | 7 + models/descriptions/trust_line_limit.md | 3 + models/gold/core/core__fact_trust_lines.sql | 49 +++++++ models/gold/core/core__fact_trust_lines.yml | 64 +++++++++ models/silver/core/silver__trust_lines.sql | 122 ++++++++++++++++++ models/silver/core/silver__trust_lines.yml | 11 ++ models/sources.yml | 1 + 11 files changed, 281 insertions(+) create mode 100644 models/bronze/core/bronze__trust_lines.sql create mode 100644 models/bronze/core/bronze__trust_lines_FR.sql create mode 100644 models/descriptions/ledger_key.md create mode 100644 models/descriptions/tables/fact_trust_lines.md create mode 100644 models/descriptions/trust_line_limit.md create mode 100644 models/gold/core/core__fact_trust_lines.sql create mode 100644 models/gold/core/core__fact_trust_lines.yml create mode 100644 models/silver/core/silver__trust_lines.sql create mode 100644 models/silver/core/silver__trust_lines.yml diff --git a/models/bronze/core/bronze__trust_lines.sql b/models/bronze/core/bronze__trust_lines.sql new file mode 100644 index 0000000..e97bea4 --- /dev/null +++ b/models/bronze/core/bronze__trust_lines.sql @@ -0,0 +1,10 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query_v2( + model = "trust_lines", + partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')", + partition_name = "partition_gte_id", + unique_key = "metadata", + other_cols = "partition_id" +) }} diff --git a/models/bronze/core/bronze__trust_lines_FR.sql b/models/bronze/core/bronze__trust_lines_FR.sql new file mode 100644 index 0000000..5714397 --- /dev/null +++ b/models/bronze/core/bronze__trust_lines_FR.sql @@ -0,0 +1,10 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query_v2( + model = "trust_lines", + partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')", + partition_name = "partition_gte_id", + unique_key = "metadata", + other_cols = "partition_id" +) }} diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index a46aa04..d6f4017 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -28,6 +28,7 @@ There is more information on how to use dbt docs in the last section of this doc - [core.fact_ledgers](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_ledgers) - [core.fact_operations](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_operations) - [core.fact_transactions](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_transactions) +- [core.fact_trust_lines](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_trust_lines) **Convenience Views:** - [core.ez_operations](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__ez_operations) diff --git a/models/descriptions/ledger_key.md b/models/descriptions/ledger_key.md new file mode 100644 index 0000000..367c376 --- /dev/null +++ b/models/descriptions/ledger_key.md @@ -0,0 +1,3 @@ +{% docs ledger_key %} +The unique ledger key when the trust line state last changed +{% enddocs %} diff --git a/models/descriptions/tables/fact_trust_lines.md b/models/descriptions/tables/fact_trust_lines.md new file mode 100644 index 0000000..500ec69 --- /dev/null +++ b/models/descriptions/tables/fact_trust_lines.md @@ -0,0 +1,7 @@ +{% docs core__fact_trust_lines %} + +Trustlines track authorized and deleted lines of trust between an account and assets. This table can be viewed as a subentry to an account because all trustlines must be associated with a single account. The trust line also tracks the balance of the asset held by the account and any buying or selling liabilities on the orderbook for a given account and asset. You do not have to hold a balance of an asset to trust the asset. + +Learn more about Stellar trust lines: https://developers.stellar.org/docs/data/analytics/hubble/data-catalog/data-dictionary/trustlines + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/trust_line_limit.md b/models/descriptions/trust_line_limit.md new file mode 100644 index 0000000..43e39ef --- /dev/null +++ b/models/descriptions/trust_line_limit.md @@ -0,0 +1,3 @@ +{% docs trust_line_limit %} +The maximum amount of this asset that this account is willing to accept. The limit is specified when opening a trust line +{% enddocs %} diff --git a/models/gold/core/core__fact_trust_lines.sql b/models/gold/core/core__fact_trust_lines.sql new file mode 100644 index 0000000..90c75ff --- /dev/null +++ b/models/gold/core/core__fact_trust_lines.sql @@ -0,0 +1,49 @@ +-- depends_on: {{ ref('silver__transactions') }} +{{ config( + materialized = 'incremental', + unique_key = ["fact_trust_lines_id"], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','closed_at::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(account_id,asset_id,asset_code);", + tags = ['scheduled_core'] +) }} + +SELECT + account_id, + closed_at, + closed_at AS block_timestamp, + asset_id, + asset_type, + asset_issuer, + asset_code, + liquidity_pool_id, + balance, + trust_line_limit, + buying_liabilities, + selling_liabilities, + flags, + last_modified_ledger, + ledger_entry_change, + deleted, + ledger_sequence, + ledger_key, + sponsor, + batch_id, + batch_run_date, + batch_insert_ts, + trust_lines_id AS fact_trust_lines_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__trust_lines') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_trust_lines.yml b/models/gold/core/core__fact_trust_lines.yml new file mode 100644 index 0000000..099978e --- /dev/null +++ b/models/gold/core/core__fact_trust_lines.yml @@ -0,0 +1,64 @@ +version: 2 + +models: + - name: core__fact_trust_lines + description: "{{ doc('core__fact_trust_lines') }}" + columns: + + + - name: account_id + description: "{{ doc('account_id') }}" + - name: closed_at + description: "{{ doc('closed_at') }}" + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + - name: asset_id + description: "{{ doc('asset_id') }}" + - name: asset_type + description: "{{ doc('asset_type') }}" + - name: asset_issuer + description: "{{ doc('asset_issuer') }}" + - name: asset_code + description: "{{ doc('asset_code') }}" + - name: liquidity_pool_id + description: "{{ doc('liquidity_pool_id') }}" + - name: balance + description: "{{ doc('balance') }}" + - name: trust_line_limit + description: "{{ doc('trust_line_limit') }}" + - name: buying_liabilities + description: "{{ doc('buying_liabilities') }}" + - name: selling_liabilities + description: "{{ doc('selling_liabilities') }}" + - name: flags + description: "{{ doc('flags') }}" + - name: last_modified_ledger + description: "{{ doc('last_modified_ledger') }}" + - name: ledger_entry_change + description: "{{ doc('ledger_entry_change') }}" + - name: deleted + description: "{{ doc('deleted') }}" + - name: ledger_sequence + description: "{{ doc('ledger_sequence') }}" + - name: ledger_key + description: "{{ doc('ledger_key') }}" + - name: sponsor + description: "{{ doc('sponsor') }}" + - name: batch_id + description: "{{ doc('batch_id') }}" + - name: batch_run_date + description: "{{ doc('batch_run_date') }}" + - name: batch_insert_ts + description: "{{ doc('batch_insert_ts') }}" + - name: fact_trust_lines_id + description: "{{ doc('pk') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - unique: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + diff --git a/models/silver/core/silver__trust_lines.sql b/models/silver/core/silver__trust_lines.sql new file mode 100644 index 0000000..6a4b52e --- /dev/null +++ b/models/silver/core/silver__trust_lines.sql @@ -0,0 +1,122 @@ +-- depends_on: {{ ref('bronze__trust_lines') }} +{{ config( + materialized = 'incremental', + unique_key = "trust_lines_id", + incremental_predicates = ["dynamic_range_predicate", "partition_id::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['closed_at::DATE','partition_id','modified_timestamp::DATE'], + tags = ['scheduled_core'], +) }} + +{% if execute %} + +{% if is_incremental() %} +{% set max_is_query %} + +SELECT + MAX(_inserted_timestamp) AS _inserted_timestamp, + MAX(partition_gte_id) AS partition__gte_id +FROM + {{ this }} + + {% endset %} + {% set result = run_query(max_is_query) %} + {% set max_is = result [0] [0] %} + {% set max_part = result [0] [1] %} +{% endif %} +{% endif %} + +WITH pre_final AS ( + SELECT + partition_id, + partition_gte_id, + VALUE :account_id :: STRING AS account_id, + VALUE :asset_id :: bigint AS asset_id, + VALUE :asset_type :: STRING AS asset_type, + VALUE :asset_issuer :: STRING AS asset_issuer, + VALUE :asset_code :: STRING AS asset_code, + VALUE :liquidity_pool_id :: STRING AS liquidity_pool_id, + VALUE :balance :: FLOAT AS balance, + VALUE :trust_line_limit :: FLOAT AS trust_line_limit, + VALUE :buying_liabilities :: FLOAT AS buying_liabilities, + VALUE :selling_liabilities :: FLOAT AS selling_liabilities, + VALUE :flags :: STRING AS flags, + VALUE :last_modified_ledger :: bigint AS last_modified_ledger, + VALUE :ledger_entry_change :: INT AS ledger_entry_change, + VALUE :deleted :: BOOLEAN AS deleted, + TO_TIMESTAMP( + VALUE :closed_at :: INT, + 6 + ) AS closed_at, + VALUE :ledger_sequence :: bigint AS ledger_sequence, + VALUE :ledger_key :: STRING AS ledger_key, + VALUE :sponsor :: STRING AS sponsor, + VALUE :batch_id :: STRING AS batch_id, + TO_TIMESTAMP( + VALUE :batch_run_date :: INT, + 6 + ) AS batch_run_date, + TO_TIMESTAMP( + VALUE :batch_insert_ts :: INT, + 6 + ) AS batch_insert_ts, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__trust_lines') }} +{% else %} + {{ ref('bronze__trust_lines_FR') }} +{% endif %} + +{% if is_incremental() %} +WHERE + partition_gte_id >= '{{ max_part }}' + AND _inserted_timestamp > '{{ max_is }}' +{% endif %} + +qualify ROW_NUMBER() over ( + PARTITION BY account_id, + asset_type, + asset_issuer, + asset_code, + liquidity_pool_id, + closed_at + ORDER BY + batch_insert_ts DESC, + _inserted_timestamp DESC +) = 1 +) +SELECT + partition_id, + partition_gte_id, + account_id, + asset_id, + asset_type, + asset_issuer, + asset_code, + liquidity_pool_id, + balance, + trust_line_limit, + buying_liabilities, + selling_liabilities, + flags, + last_modified_ledger, + ledger_entry_change, + deleted, + closed_at, + ledger_sequence, + ledger_key, + sponsor, + batch_id, + batch_run_date, + batch_insert_ts, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['account_id', 'asset_type', 'asset_issuer', 'asset_code', 'liquidity_pool_id', 'closed_at'] + ) }} AS trust_lines_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + pre_final diff --git a/models/silver/core/silver__trust_lines.yml b/models/silver/core/silver__trust_lines.yml new file mode 100644 index 0000000..377aeba --- /dev/null +++ b/models/silver/core/silver__trust_lines.yml @@ -0,0 +1,11 @@ +version: 2 +models: + - name: silver__trust_lines + columns: + - name: trust_lines_id + description: "{{ doc('id') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - unique: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} diff --git a/models/sources.yml b/models/sources.yml index d231cc8..9d9c6fa 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -14,6 +14,7 @@ sources: - name: history_trades - name: history_transactions - name: liquidity_pools + - name: trust_lines - name: crosschain database: "{{ 'crosschain' if target.database == 'STELLAR' else 'crosschain_dev' }}" schema: core