AN-5883 add trust lines (#11)

* add trust lines

* ov

models/bronze/core/bronze__trust_lines.sql

@@ -0,0 +1,10 @@
+{{ config (
+    materialized = 'view'
+) }}
+{{ streamline_external_table_query_v2(
+    model = "trust_lines",
+    partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
+    partition_name = "partition_gte_id",
+    unique_key = "metadata",
+    other_cols = "partition_id"
+) }}

models/bronze/core/bronze__trust_lines_FR.sql

@@ -0,0 +1,10 @@
+{{ config (
+    materialized = 'view'
+) }}
+{{ streamline_external_table_FR_query_v2(
+    model = "trust_lines",
+    partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
+    partition_name = "partition_gte_id",
+    unique_key = "metadata",
+    other_cols = "partition_id"
+) }}

models/descriptions/__overview__.md

@@ -28,6 +28,7 @@ There is more information on how to use dbt docs in the last section of this doc
 - [core.fact_ledgers](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_ledgers)
 - [core.fact_operations](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_operations)
 - [core.fact_transactions](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_transactions)
+- [core.fact_trust_lines](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_trust_lines)
 
 **Convenience Views:**
 - [core.ez_operations](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__ez_operations)

models/descriptions/ledger_key.md

@@ -0,0 +1,3 @@
+{% docs ledger_key %}
+The unique ledger key when the trust line state last changed
+{% enddocs %}

models/descriptions/tables/fact_trust_lines.md

@@ -0,0 +1,7 @@
+{% docs core__fact_trust_lines %}
+
+Trustlines track authorized and deleted lines of trust between an account and assets. This table can be viewed as a subentry to an account because all trustlines must be associated with a single account. The trust line also tracks the balance of the asset held by the account and any buying or selling liabilities on the orderbook for a given account and asset. You do not have to hold a balance of an asset to trust the asset.
+
+Learn more about Stellar trust lines: https://developers.stellar.org/docs/data/analytics/hubble/data-catalog/data-dictionary/trustlines
+
+{% enddocs %}

models/descriptions/trust_line_limit.md

@@ -0,0 +1,3 @@
+{% docs trust_line_limit %}
+The maximum amount of this asset that this account is willing to accept. The limit is specified when opening a trust line
+{% enddocs %}

models/gold/core/core__fact_trust_lines.sql

@@ -0,0 +1,49 @@
+-- depends_on: {{ ref('silver__transactions') }}
+{{ config(
+    materialized = 'incremental',
+    unique_key = ["fact_trust_lines_id"],
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
+    merge_exclude_columns = ["inserted_timestamp"],
+    cluster_by = ['block_timestamp::DATE','closed_at::DATE'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(account_id,asset_id,asset_code);",
+    tags = ['scheduled_core']
+) }}
+
+SELECT
+    account_id,
+    closed_at,
+    closed_at AS block_timestamp,
+    asset_id,
+    asset_type,
+    asset_issuer,
+    asset_code,
+    liquidity_pool_id,
+    balance,
+    trust_line_limit,
+    buying_liabilities,
+    selling_liabilities,
+    flags,
+    last_modified_ledger,
+    ledger_entry_change,
+    deleted,
+    ledger_sequence,
+    ledger_key,
+    sponsor,
+    batch_id,
+    batch_run_date,
+    batch_insert_ts,
+    trust_lines_id AS fact_trust_lines_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp
+FROM
+    {{ ref('silver__trust_lines') }}
+
+{% if is_incremental() %}
+WHERE
+    modified_timestamp >= (
+        SELECT
+            MAX(modified_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/gold/core/core__fact_trust_lines.yml

@@ -0,0 +1,64 @@
+version: 2
+
+models:
+  - name: core__fact_trust_lines
+    description: "{{ doc('core__fact_trust_lines') }}"
+    columns:
+
+
+      - name: account_id
+        description: "{{ doc('account_id') }}"
+      - name: closed_at
+        description: "{{ doc('closed_at') }}"
+      - name: block_timestamp
+        description: "{{ doc('block_timestamp') }}"
+      - name: asset_id
+        description: "{{ doc('asset_id') }}"
+      - name: asset_type
+        description: "{{ doc('asset_type') }}"
+      - name: asset_issuer
+        description: "{{ doc('asset_issuer') }}"
+      - name: asset_code
+        description: "{{ doc('asset_code') }}"
+      - name: liquidity_pool_id
+        description: "{{ doc('liquidity_pool_id') }}"
+      - name: balance
+        description: "{{ doc('balance') }}"
+      - name: trust_line_limit
+        description: "{{ doc('trust_line_limit') }}"
+      - name: buying_liabilities
+        description: "{{ doc('buying_liabilities') }}"
+      - name: selling_liabilities
+        description: "{{ doc('selling_liabilities') }}"
+      - name: flags
+        description: "{{ doc('flags') }}"
+      - name: last_modified_ledger
+        description: "{{ doc('last_modified_ledger') }}"
+      - name: ledger_entry_change
+        description: "{{ doc('ledger_entry_change') }}"
+      - name: deleted
+        description: "{{ doc('deleted') }}"
+      - name: ledger_sequence
+        description: "{{ doc('ledger_sequence') }}"
+      - name: ledger_key
+        description: "{{ doc('ledger_key') }}"
+      - name: sponsor
+        description: "{{ doc('sponsor') }}"
+      - name: batch_id
+        description: "{{ doc('batch_id') }}"
+      - name: batch_run_date
+        description: "{{ doc('batch_run_date') }}"
+      - name: batch_insert_ts
+        description: "{{ doc('batch_insert_ts') }}"
+      - name: fact_trust_lines_id
+        description: "{{ doc('pk') }}"
+        tests:
+          - not_null:
+              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+          - unique:
+              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+      - name: inserted_timestamp
+        description: "{{ doc('inserted_timestamp') }}"
+      - name: modified_timestamp
+        description: "{{ doc('modified_timestamp') }}"
+

models/silver/core/silver__trust_lines.sql

@@ -0,0 +1,122 @@
+-- depends_on: {{ ref('bronze__trust_lines') }}
+{{ config(
+    materialized = 'incremental',
+    unique_key = "trust_lines_id",
+    incremental_predicates = ["dynamic_range_predicate", "partition_id::date"],
+    merge_exclude_columns = ["inserted_timestamp"],
+    cluster_by = ['closed_at::DATE','partition_id','modified_timestamp::DATE'],
+    tags = ['scheduled_core'],
+) }}
+
+{% if execute %}
+
+{% if is_incremental() %}
+{% set max_is_query %}
+
+SELECT
+    MAX(_inserted_timestamp) AS _inserted_timestamp,
+    MAX(partition_gte_id) AS partition__gte_id
+FROM
+    {{ this }}
+
+    {% endset %}
+    {% set result = run_query(max_is_query) %}
+    {% set max_is = result [0] [0] %}
+    {% set max_part = result [0] [1] %}
+{% endif %}
+{% endif %}
+
+WITH pre_final AS (
+    SELECT
+        partition_id,
+        partition_gte_id,
+        VALUE :account_id :: STRING AS account_id,
+        VALUE :asset_id :: bigint AS asset_id,
+        VALUE :asset_type :: STRING AS asset_type,
+        VALUE :asset_issuer :: STRING AS asset_issuer,
+        VALUE :asset_code :: STRING AS asset_code,
+        VALUE :liquidity_pool_id :: STRING AS liquidity_pool_id,
+        VALUE :balance :: FLOAT AS balance,
+        VALUE :trust_line_limit :: FLOAT AS trust_line_limit,
+        VALUE :buying_liabilities :: FLOAT AS buying_liabilities,
+        VALUE :selling_liabilities :: FLOAT AS selling_liabilities,
+        VALUE :flags :: STRING AS flags,
+        VALUE :last_modified_ledger :: bigint AS last_modified_ledger,
+        VALUE :ledger_entry_change :: INT AS ledger_entry_change,
+        VALUE :deleted :: BOOLEAN AS deleted,
+        TO_TIMESTAMP(
+            VALUE :closed_at :: INT,
+            6
+        ) AS closed_at,
+        VALUE :ledger_sequence :: bigint AS ledger_sequence,
+        VALUE :ledger_key :: STRING AS ledger_key,
+        VALUE :sponsor :: STRING AS sponsor,
+        VALUE :batch_id :: STRING AS batch_id,
+        TO_TIMESTAMP(
+            VALUE :batch_run_date :: INT,
+            6
+        ) AS batch_run_date,
+        TO_TIMESTAMP(
+            VALUE :batch_insert_ts :: INT,
+            6
+        ) AS batch_insert_ts,
+        _inserted_timestamp
+    FROM
+
+{% if is_incremental() %}
+{{ ref('bronze__trust_lines') }}
+{% else %}
+    {{ ref('bronze__trust_lines_FR') }}
+{% endif %}
+
+{% if is_incremental() %}
+WHERE
+    partition_gte_id >= '{{ max_part }}'
+    AND _inserted_timestamp > '{{ max_is }}'
+{% endif %}
+
+qualify ROW_NUMBER() over (
+    PARTITION BY account_id,
+    asset_type,
+    asset_issuer,
+    asset_code,
+    liquidity_pool_id,
+    closed_at
+    ORDER BY
+        batch_insert_ts DESC,
+        _inserted_timestamp DESC
+) = 1
+)
+SELECT
+    partition_id,
+    partition_gte_id,
+    account_id,
+    asset_id,
+    asset_type,
+    asset_issuer,
+    asset_code,
+    liquidity_pool_id,
+    balance,
+    trust_line_limit,
+    buying_liabilities,
+    selling_liabilities,
+    flags,
+    last_modified_ledger,
+    ledger_entry_change,
+    deleted,
+    closed_at,
+    ledger_sequence,
+    ledger_key,
+    sponsor,
+    batch_id,
+    batch_run_date,
+    batch_insert_ts,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['account_id', 'asset_type', 'asset_issuer', 'asset_code', 'liquidity_pool_id', 'closed_at']
+    ) }} AS trust_lines_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    pre_final

models/silver/core/silver__trust_lines.yml

@@ -0,0 +1,11 @@
+version: 2
+models:
+  - name: silver__trust_lines
+    columns:
+      - name: trust_lines_id
+        description: "{{ doc('id') }}"
+        tests:
+          - not_null:
+              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
+          - unique:
+              where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}

models/sources.yml

@@ -14,6 +14,7 @@ sources:
       - name: history_trades
       - name: history_transactions
       - name: liquidity_pools
+      - name: trust_lines
   - name: crosschain
     database: "{{ 'crosschain' if target.database == 'STELLAR' else 'crosschain_dev' }}"
     schema: core