moved tests to top level /gold dir; switched to FSC utils macros instead of custom; added liquidity pool actions pipelines and recency tests; edited registrations and swaps

dbt_project.yml

@@ -51,7 +51,6 @@ tests:
 on-run-start:
   - '{{create_sps()}}'
   - '{{create_udfs()}}'
-  - "{{create_decode_128_udf()}}"
 
 on-run-end:
   - '{{ apply_meta_as_tags(results) }}'

models/gold/defi/defi__fact_liquidity_pool_actions.sql

@@ -0,0 +1,66 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp::DATE >= (select min(block_timestamp::DATE) from ' ~ generate_tmp_view_name(this) ~ ')'],
+    unique_key = ['fact_liquidity_pool_actions_id'],
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ['inserted_timestamp'],
+    cluster_by = ['block_timestamp::DATE'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(liquidity_pool_protocol,liquidity_provider,root_action,liquidity_action,token1_id,token2_id,token1_name,token2_name,liquidity_pool_protocol);",
+    tags = ['noncore', 'full_test']
+) }}
+
+WITH arcane AS (
+    SELECT
+        block_timestamp,
+        block_id,
+        tx_id,
+        succeeded,
+        root_action,
+        liquidity_action,
+        liquidity_provider,
+        token1_id,
+        token2_id,
+        token1_name,
+        token2_name,
+        token1_amount,
+        token2_amount,
+        'Arcane Finance' as liquidity_pool_protocol
+    FROM 
+        {{ ref('silver__liquidity_pool_actions_arcane') }}
+
+    {% if is_incremental() %}
+    WHERE
+        modified_timestamp >= DATEADD(
+            'minute',
+            -5,
+            (
+                SELECT
+                    MAX(modified_timestamp)
+                FROM
+                    {{ this }}
+            )
+        )
+    {% endif %}
+)
+
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    root_action,
+    liquidity_action,
+    liquidity_provider,
+    token1_id,
+    token2_id,
+    token1_name,
+    token2_name,
+    token1_amount,
+    token2_amount,
+    liquidity_pool_protocol,
+    {{ dbt_utils.generate_surrogate_key(['tx_id', 'liquidity_pool_protocol', 'token1_id','token2_id']) }} AS fact_liquidity_pool_actions_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    arcane -- union more later

models/gold/defi/defi__fact_liquidity_pool_actions.yml

@@ -0,0 +1,109 @@
+version: 2
+models:
+  - name: defi__fact_liquidity_pool_actions
+    description: Records of all liquidity pool actions on Aleo, including adding and removing liquidity.
+    columns:
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - not_null
+
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: SUCCEEDED
+        description: "{{ doc('succeeded') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - accepted_values:
+              values: [true, false]
+          - not_null
+
+      - name: ROOT_ACTION
+        description: "The program and function that initiated the liquidity pool action"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: LIQUIDITY_ACTION
+        description: "The type of liquidity action performed (e.g., add, remove)"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: LIQUIDITY_PROVIDER
+        description: "The address that performed the liquidity pool action"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: TOKEN1_ID
+        description: "The token id of the first token in the liquidity pair"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: TOKEN2_ID
+        description: "The token id of the second token in the liquidity pair"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: TOKEN1_NAME
+        description: "The name of the first token in the liquidity pair"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: TOKEN2_NAME
+        description: "The name of the second token in the liquidity pair"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: TOKEN1_AMOUNT
+        description: "The amount of the first token added or removed from the pool"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: TOKEN2_AMOUNT
+        description: "The amount of the second token added or removed from the pool"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: LIQUIDITY_POOL_PROTOCOL
+        description: "The name of the protocol where the liquidity pool action occurred"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+          - not_null
+
+      - name: FACT_LIQUIDITY_POOL_ACTIONS_ID
+        description: '{{ doc("pk") }}'
+        tests:
+          - unique
+          - not_null
+
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'
+        tests:
+          - not_null
+
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}'
+        tests:
+          - not_null

models/gold/defi/defi__fact_swaps.sql

@@ -55,7 +55,7 @@ SELECT
     swap_to_id,
     root_action,
     swap_protocol,
-    {{ dbt_utils.generate_surrogate_key(['TX_ID','swapper']) }} AS fact_swaps_id,
+    {{ dbt_utils.generate_surrogate_key(['tx_id','swap_protocol', 'swapper']) }} AS fact_swaps_id,
     SYSDATE() AS inserted_timestamp,
     SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS _invocation_id

models/gold/tests/defi/test_defi__liquidity_actions_recent.sql

@@ -0,0 +1,27 @@
+{{ config (
+    materialized = 'view',
+    tags = ['recent_test']
+) }}
+
+WITH last_3_days AS (
+
+    SELECT
+        block_date
+    FROM
+        {{ ref("_max_block_by_date") }}
+        qualify ROW_NUMBER() over (
+            ORDER BY
+                block_date DESC
+        ) = 3
+)
+SELECT
+    *
+FROM
+    {{ ref('defi__fact_liquidity_pool_actions') }}
+WHERE
+    block_timestamp :: DATE >= (
+        SELECT
+            block_date
+        FROM
+            last_3_days
+    )

models/gold/tests/defi/test_defi__liquidity_actions_recent.yml

@@ -0,0 +1,20 @@
+version: 2
+
+models:
+  - name: test_defi__liquidity_actions_recent
+    description: Records of all liquidity pool actions on Aleo.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_ID
+            - LIQUIDITY_POOL_PROTOCOL
+            - LIQUIDITY_PROVIDER
+            - TOKEN1_ID
+            - TOKEN2_ID
+    columns:
+      - name: BLOCK_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: hour
+              interval: 3

models/silver/core/tokens/silver__token_registrations.sql

@@ -8,7 +8,7 @@ with base_data as (
         tx_id,
         block_id,
         block_timestamp,
-        inputs
+        INPUTS
     from aleo_dev.core.fact_transitions
     where program_id = 'token_registry.aleo' 
     and function = 'register_token'
@@ -24,7 +24,7 @@ flattened_inputs as (
         value:value::string as value,
         row_number() over (partition by tx_id order by index) - 1 as input_index
     from base_data,
-    lateral flatten(input => inputs)
+    lateral flatten(input => INPUTS)
 ),
 
 parsed_inputs as (
@@ -65,7 +65,7 @@ cleaned_strings as (
         tx_id,
         block_id,
         block_timestamp,
-        token_id_raw,
+        split_part(token_id_raw, 'field', 1) as token_id,
         split_part(name_raw, 'u', 1) as name_encoded,
         split_part(symbol_raw, 'u', 1) as symbol_encoded,
         split_part(decimals_raw, 'u', 1) as decimals,
@@ -79,9 +79,9 @@ select
     tx_id,
     block_id,
     block_timestamp,
-    token_id_raw as token_id,
-    utils.udf_hex_to_string(substr(utils.udf_int_to_hex(name_encoded), 3)) as token_name,
-    utils.udf_hex_to_string(substr(utils.udf_int_to_hex(symbol_encoded), 3)) as symbol,
+    token_id,
+    udf_hex_to_string(substr(utils.udf_int_to_hex(name_encoded), 3)) as token_name,
+    udf_hex_to_string(substr(udf_int_to_hex(symbol_encoded), 3)) as symbol,
     decimals,
     max_supply,
     external_auth_required,

models/silver/defi/silver__liquidity_pool_actions_arcane.sql

@@ -0,0 +1,100 @@
+{{ config(
+    materialized='incremental',
+    unique_key='liquidity_pool_actions_arcane_id',
+    incremental_strategy='merge',
+    merge_exclude_columns = ['inserted_timestamp'],
+    cluster_by = ['modified_timestamp::DATE'],
+    tags=['noncore', 'full_test']
+) }}
+
+-- depends on: {{ ref('core__fact_transitions') }}
+with 
+root_actions as (
+    select
+        tx_id,
+        program_id,
+        function,
+        program_id || '/' || function as root_action,
+        CASE 
+            WHEN function ILIKE '%add%' THEN 'Add'
+            WHEN function ILIKE '%remove%' THEN 'Remove'
+            ELSE 'OTHER'
+        END as liquidity_action
+    from
+        aleo.core.fact_transitions
+    where
+        program_id ilike 'arcn%'
+        and function ilike '%liq%'
+        and function not ilike '%credits%'
+),
+reports as (
+    select
+        block_timestamp,
+        block_id,
+        tx_id,
+        succeeded,
+        INPUTS
+    from
+        aleo.core.fact_transitions
+    where
+        program_id ilike 'arcn%'
+        and function = 'report'
+),
+parsed as (
+    select
+        block_timestamp,
+        block_id,
+        tx_id,
+        succeeded,
+        root_action,
+        liquidity_action,
+        inputs[1] :value :: string as liquidity_provider,
+        inputs[2] :value :: string as token1_id,
+        inputs[3] :value :: string as token2_id,
+        split_part(inputs[4] :value, 'u', 1) :: number as token1_amount,
+        split_part(inputs[5] :value, 'u', 1) :: number as token2_amount
+    from 
+        root_actions
+    join
+        reports using(tx_id)
+),
+tokens as (
+    select
+        token_id,
+        name_encoded,
+        token_name,
+        decimals
+    from
+        aleo_dev.silver.token_registrations
+)
+select 
+    p.block_timestamp,
+    p.block_id,
+    p.tx_id,
+    p.succeeded,
+    p.root_action,
+    p.liquidity_action,
+    p.liquidity_provider,
+    p.token1_id,
+    p.token2_id,
+    case
+        when p.token1_id = '3443843282313283355522573239085696902919850365217539366784739393210722344986field' then 'Aleo'
+        else t1.token_name
+    end as token1_name,
+    case
+        when p.token2_id = '3443843282313283355522573239085696902919850365217539366784739393210722344986field' then 'Aleo'
+        else t2.token_name
+    end as token2_name,
+    p.token1_amount / power(10, coalesce(t1.decimals, 6)) as token1_amount,
+    p.token2_amount / power(10, coalesce(t2.decimals, 6)) as token2_amount,
+    {{ dbt_utils.generate_surrogate_key(['p.tx_id','p.token1_id', 'p.token2_id']) }} AS liquidity_pool_actions_arcane_id,
+    SYSDATE() as inserted_timestamp,
+    SYSDATE() as modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+from
+    parsed p
+left join {{ ref('silver__token_registrations') }} t1 on p.token1_id = t1.token_id
+left join {{ ref('silver__token_registrations') }} t2 on p.token2_id = t2.token_id
+qualify(ROW_NUMBER() over(PARTITION BY p.tx_id
+ORDER BY
+    root_action) = 1)

models/silver/defi/silver__liquidity_pool_actions_arcane.yml

@@ -0,0 +1,69 @@
+version: 2
+
+models:
+  - name: silver__liquidity_pool_actions_arcane    
+    columns:
+      - name: liquidity_pool_actions_arcane_id
+        tests:
+          - unique
+          - not_null
+
+      - name: block_timestamp
+        tests:
+          - not_null
+
+      - name: block_id
+        tests:
+          - not_null
+
+      - name: tx_id
+        tests:
+          - not_null
+
+      - name: succeeded
+        tests:
+          - not_null
+
+      - name: root_action
+        tests:
+          - not_null
+
+      - name: liquidity_action
+        tests:
+          - not_null
+
+      - name: liquidity_provider
+        tests:
+          - not_null
+
+      - name: token1_id
+        tests:
+          - not_null
+
+      - name: token2_id
+        tests:
+          - not_null
+
+      - name: token1_name
+        tests:
+          - not_null
+
+      - name: token2_name
+        tests:
+          - not_null
+
+      - name: token1_amount
+        tests:
+          - not_null
+
+      - name: token2_amount
+        tests:
+          - not_null
+
+      - name: inserted_timestamp
+        tests:
+          - not_null
+
+      - name: modified_timestamp
+        tests:
+          - not_null

models/silver/defi/silver__swaps_arcane.sql

@@ -3,7 +3,7 @@
     unique_key='swaps_arcane_id',
     incremental_strategy='merge',
     merge_exclude_columns = ['inserted_timestamp'],
-    cluster_by = ['modified_timestamp::DATE', 'swap_from_name', 'swap_to_name'],
+    cluster_by = ['modified_timestamp::DATE', 'swapper','swap_from_name', 'swap_to_name'],
     tags=['noncore', 'full_test']
 ) }}