An 4787/liquidity pool actions meteora (#552)

* (wip) initial model * create base ctes, fix syntax * move combining specific event types downstream * revert temp full refresh range change * add tests and schema * add relationship test * fix predicate * remove unnecessary predicates * filter out add balances where no balance is added * add coalesce in cases where this arg is not provided * include metadata fields in schema, fix wrong col name
2026-02-06 13:56:50 +00:00 · 2024-05-09 08:39:23 -07:00 · 2024-05-09 08:39:23 -07:00 · 71c9a9f5c0
commit 71c9a9f5c0
parent cf06cfdaa3
4 changed files with 440 additions and 0 deletions
--- a/data/testing__liquidity_pool_actions_meteora.csv
+++ b/data/testing__liquidity_pool_actions_meteora.csv
@ -0,0 +1,14 @@
+tx_id, index, inner_index, program_id, action, mint, amount, liquidity_provider, liquidity_pool_address
+HQVXg5FRV93oJeGuqkXVe4KnoJvRW3bJA7fJwjw1obJLgWWFHr8mtEhnYnW6NaNthrM7vbvsSgwX5uvs3us79J3,2,1,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,removeBalanceLiquidity,So11111111111111111111111111111111111111112,0.547046388,FoNpjgd9qDDagvWcDGGuE1zTEkX42pX5cJQuKX2zXXBw,ERgpKaq59Nnfm9YRVAAhnq16cZhHxGcDoDWCzXbhiaNw
+HQVXg5FRV93oJeGuqkXVe4KnoJvRW3bJA7fJwjw1obJLgWWFHr8mtEhnYnW6NaNthrM7vbvsSgwX5uvs3us79J3,2,4,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,removeBalanceLiquidity,J1toso1uCk3RLmjorhTtrVwY9HJ7X8V9yYac6Y7kGCPn,1.102009352,FoNpjgd9qDDagvWcDGGuE1zTEkX42pX5cJQuKX2zXXBw,ERgpKaq59Nnfm9YRVAAhnq16cZhHxGcDoDWCzXbhiaNw
+HQVXg5FRV93oJeGuqkXVe4KnoJvRW3bJA7fJwjw1obJLgWWFHr8mtEhnYnW6NaNthrM7vbvsSgwX5uvs3us79J3,2,6,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,burn,8fL5oNPFD9WEpwf4k3H4YnPQtVUo9RAGpN36vKFQoHPi,1.660970995,FoNpjgd9qDDagvWcDGGuE1zTEkX42pX5cJQuKX2zXXBw,ERgpKaq59Nnfm9YRVAAhnq16cZhHxGcDoDWCzXbhiaNw
+5cuJ3DsDjCZtmADeusavqh1qsG52BBDthg82cJpaQkJRgHE1wCSYahuusCD6EoxNKPxd98847B5mo7g1X95Unfa3,5,1,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,bootstrapLiquidity,3bXU1UpDytkwzb78FGSgKKVMDbowAS66azVw5GpzUDbn,124227.422103,af3hv22geZv3qpgCrw1zWmkrXCuQYTNjhuwwvZsviGh,gb6gDU1deUeYzADVmJoDDTHDjuWA76drmDXHv6BoGHi
+5cuJ3DsDjCZtmADeusavqh1qsG52BBDthg82cJpaQkJRgHE1wCSYahuusCD6EoxNKPxd98847B5mo7g1X95Unfa3,5,4,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,bootstrapLiquidity,So11111111111111111111111111111111111111112,2.995732216,af3hv22geZv3qpgCrw1zWmkrXCuQYTNjhuwwvZsviGh,gb6gDU1deUeYzADVmJoDDTHDjuWA76drmDXHv6BoGHi
+5cuJ3DsDjCZtmADeusavqh1qsG52BBDthg82cJpaQkJRgHE1wCSYahuusCD6EoxNKPxd98847B5mo7g1X95Unfa3,5,6,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,mintTo,FJLaVQkCxXj8CsEqBAF4incZKAbFq1axDw2xmBLpNjuF,19.291243881,af3hv22geZv3qpgCrw1zWmkrXCuQYTNjhuwwvZsviGh,gb6gDU1deUeYzADVmJoDDTHDjuWA76drmDXHv6BoGHi
+3RTWUqPBYKzPS2QW6rxCsvXvFYCgmTbMhEbCTHn479RwMUHskdEzPuY4Sknu2qHNb27ek6BFU7XDkMQ2yJsgvkpT,5,1,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,addImbalanceLiquidity,mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So,0.099526948,EbgNuB56nEo8WRrUQ4GV1jfeawprbt1Mj7A85AxpSVR3,HcjZvfeSNJbNkfLD4eEcRBr96AD3w1GpmMppaeRZf7ur
+3RTWUqPBYKzPS2QW6rxCsvXvFYCgmTbMhEbCTHn479RwMUHskdEzPuY4Sknu2qHNb27ek6BFU7XDkMQ2yJsgvkpT,5,3,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,mintTo,B2uEs9zjnz222hfUaUuRgesryUEYwy3JGuWe31sE9gsG,0.108001799,EbgNuB56nEo8WRrUQ4GV1jfeawprbt1Mj7A85AxpSVR3,HcjZvfeSNJbNkfLD4eEcRBr96AD3w1GpmMppaeRZf7ur
+4TYQhGdkBVnE2nF8RcSvWNwQKEPaACw9suBQP6Jjb5oEdJjxt75KaQeXaEMynoHGHQsWuWNZhtfHD3m4Y8t3x8d8,4,1,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,addBalanceLiquidity,9xYeZDHEwyuqJmqrTourbFRaxN2qhkYesnz3iQ3FPz4r,3265.77839,86NwaLqKhqqJAhnsb2aQqcB6ug2qaDu8HBBNZqpWtRoQ,FyaRzjEuRG8ravL76Ury86SYJBCAFiaZW9rTCAfB3KXz
+4TYQhGdkBVnE2nF8RcSvWNwQKEPaACw9suBQP6Jjb5oEdJjxt75KaQeXaEMynoHGHQsWuWNZhtfHD3m4Y8t3x8d8,4,4,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,addBalanceLiquidity,So11111111111111111111111111111111111111112,0.006000555,86NwaLqKhqqJAhnsb2aQqcB6ug2qaDu8HBBNZqpWtRoQ,FyaRzjEuRG8ravL76Ury86SYJBCAFiaZW9rTCAfB3KXz
+4TYQhGdkBVnE2nF8RcSvWNwQKEPaACw9suBQP6Jjb5oEdJjxt75KaQeXaEMynoHGHQsWuWNZhtfHD3m4Y8t3x8d8,4,6,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,mintTo,3J749zStSiwEXzjPiNd74DRSLshQufrFvcioGhrKCLU9,0.043961628,86NwaLqKhqqJAhnsb2aQqcB6ug2qaDu8HBBNZqpWtRoQ,FyaRzjEuRG8ravL76Ury86SYJBCAFiaZW9rTCAfB3KXz
+CsVci6T9dkkKD1nZN1gJiw7Wc47wSoV6DsCyD5K2uUXNPnV1e5R7vssG3czmmXfNoYuk2S5AqiVQRALZjMZVcyF,1,1,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,removeLiquiditySingleSide,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,13.096548,5tMZhBeyFJH1pVPPpvENM9sCp5tSnwawXJSds5LTBZ8Y,32D4zRxNc1EssbJieVHfPhZM3rH6CzfUPrWUuWxD9prG
+CsVci6T9dkkKD1nZN1gJiw7Wc47wSoV6DsCyD5K2uUXNPnV1e5R7vssG3czmmXfNoYuk2S5AqiVQRALZjMZVcyF,1,3,Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB,burn,xLebAypjbaQ9tmxUKHV6DZU4mY8ATAAP2sfkNNQLXjf,12,5tMZhBeyFJH1pVPPpvENM9sCp5tSnwawXJSds5LTBZ8Y,32D4zRxNc1EssbJieVHfPhZM3rH6CzfUPrWUuWxD9prG
--- a/models/silver/liquidity_pool/silver__liquidity_pool_actions_meteora.sql
+++ b/models/silver/liquidity_pool/silver__liquidity_pool_actions_meteora.sql
@ -0,0 +1,307 @@
+-- depends_on: {{ ref('silver__decoded_instructions_combined') }}
+{{ config(
+    materialized = 'incremental',
+    unique_key = "liquidity_pool_actions_meteora_id",
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
+    merge_exclude_columns = ["inserted_timestamp"],
+    cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'],
+    tags = ['scheduled_non_core'],
+) }}
+
+{% if execute %}
+    {% set base_query %}
+        CREATE OR REPLACE TEMPORARY TABLE silver.liquidity_pool_actions_meteora__intermediate_tmp AS 
+        SELECT 
+            block_timestamp,
+            block_id,
+            tx_id,
+            succeeded,
+            INDEX,
+            inner_index,
+            program_id,
+            event_type,
+            decoded_instruction,
+            _inserted_timestamp
+        FROM 
+            {{ ref('silver__decoded_instructions_combined') }}
+        WHERE
+            program_id = 'Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB'
+            AND event_type IN (
+                'addBalanceLiquidity',
+                'addImbalanceLiquidity',
+                'bootstrapLiquidity',
+                'removeBalanceLiquidity',
+                'removeLiquiditySingleSide'
+            )
+            AND succeeded
+            {% if is_incremental() %}
+            AND _inserted_timestamp >= (
+                SELECT
+                    MAX(_inserted_timestamp) - INTERVAL '1 hour'
+                FROM
+                    {{ this }}
+            )
+            {% else %} 
+            AND _inserted_timestamp :: DATE >= '2024-02-15'
+            {% endif %}
+    {% endset %}
+    {% do run_query(base_query) %}
+    {% set between_stmts = fsc_utils.dynamic_range_predicate("silver.liquidity_pool_actions_meteora__intermediate_tmp","block_timestamp::date") %}
+{% endif %}
+
+WITH base AS (
+    SELECT 
+        *
+    FROM 
+        silver.liquidity_pool_actions_meteora__intermediate_tmp
+),
+base_transfers AS (
+    SELECT 
+        block_timestamp,
+        tx_id,
+        index,
+        mint,
+        amount,
+        dest_token_account,
+        source_token_account
+    FROM
+        {{ ref('silver__transfers') }}
+    WHERE
+        {{ between_stmts }}
+),
+base_mint_actions AS (
+    SELECT 
+        block_timestamp,
+        tx_id,
+        index,
+        inner_index,
+        event_type,
+        mint,
+        mint_amount,
+        decimal
+    FROM
+        {{ ref('silver__token_mint_actions') }}
+    WHERE
+        {{ between_stmts }}
+),
+base_burn_actions AS (
+    SELECT 
+        block_timestamp,
+        tx_id,
+        index,
+        inner_index,
+        event_type,
+        mint,
+        burn_amount,
+        decimal
+    FROM
+        {{ ref('silver__token_burn_actions') }}
+    WHERE
+        {{ between_stmts }}
+),
+decoded AS (
+    SELECT
+        block_timestamp,
+        block_id,
+        tx_id,
+        succeeded,
+        INDEX,
+        inner_index,
+        program_id,
+        event_type AS action,
+        silver.udf_get_account_pubkey_by_name(
+            'user',
+            decoded_instruction :accounts
+        ) AS liquidity_provider,
+        silver.udf_get_account_pubkey_by_name(
+            'pool',
+            decoded_instruction :accounts
+        ) AS liquidity_pool_address,
+        silver.udf_get_account_pubkey_by_name(
+            'lpMint',
+            decoded_instruction :accounts
+        ) AS liquidity_pool_mint,
+        silver.udf_get_account_pubkey_by_name(
+            'aTokenVault',
+            decoded_instruction :accounts
+        ) AS liquidity_a_token_vault,
+        silver.udf_get_account_pubkey_by_name(
+            'bTokenVault',
+            decoded_instruction :accounts
+        ) AS liquidity_b_token_vault,
+        _inserted_timestamp,
+    FROM
+        base
+),
+deposits AS (
+    SELECT
+        d.block_timestamp,
+        d.block_id,
+        d.tx_id,
+        d.succeeded,
+        d.index,
+        SPLIT_PART(
+            t.index,
+            '.',
+            2
+        ) :: INT AS inner_index,
+        d.program_id,
+        d.action,
+        t.mint,
+        t.amount,
+        d.liquidity_provider,
+        d.liquidity_pool_address,
+        d._inserted_timestamp
+    FROM
+        base_transfers t
+        INNER JOIN decoded d
+        ON t.block_timestamp :: DATE = d.block_timestamp :: DATE
+        AND t.tx_id = d.tx_id
+        AND SPLIT_PART(
+            t.index,
+            '.',
+            1
+        ) = d.index
+    WHERE
+        d.action IN (
+            'addBalanceLiquidity',
+            'addImbalanceLiquidity',
+            'bootstrapLiquidity'
+        )
+        AND dest_token_account IN (
+            liquidity_a_token_vault,
+            liquidity_b_token_vault
+        )
+),
+withdraws AS (
+    SELECT
+        d.block_timestamp,
+        d.block_id,
+        d.tx_id,
+        d.succeeded,
+        d.index,
+        SPLIT_PART(
+            t.index,
+            '.',
+            2
+        ) :: INT AS inner_index,
+        d.program_id,
+        d.action,
+        t.mint,
+        t.amount,
+        d.liquidity_provider,
+        d.liquidity_pool_address,
+        d._inserted_timestamp
+    FROM
+        base_transfers t
+        INNER JOIN decoded d
+        ON t.block_timestamp :: DATE = d.block_timestamp :: DATE
+        AND t.tx_id = d.tx_id
+        AND SPLIT_PART(
+            t.index,
+            '.',
+            1
+        ) = d.index
+    WHERE
+        d.action IN (
+            'removeBalanceLiquidity',
+            'removeLiquiditySingleSide'
+        )
+        AND source_token_account IN (
+            liquidity_a_token_vault,
+            liquidity_b_token_vault
+        )
+),
+mints AS (
+    SELECT
+        d.block_timestamp,
+        d.block_id,
+        d.tx_id,
+        d.succeeded,
+        m.index,
+        m.inner_index,
+        d.program_id,
+        m.event_type,
+        m.mint,
+        m.mint_amount * pow(
+            10,-m.decimal
+        ) AS amount,
+        d.liquidity_provider,
+        d.liquidity_pool_address,
+        d._inserted_timestamp
+    FROM
+        base_mint_actions m
+        INNER JOIN decoded d
+        ON m.block_timestamp :: DATE = d.block_timestamp :: DATE
+        AND m.tx_id = d.tx_id
+        AND m.index = d.index
+    WHERE
+        m.mint = d.liquidity_pool_mint
+),
+burns AS (
+    SELECT
+        d.block_timestamp,
+        d.block_id,
+        d.tx_id,
+        d.succeeded,
+        m.index,
+        m.inner_index,
+        d.program_id,
+        m.event_type,
+        m.mint,
+        m.burn_amount * pow(
+            10,-m.decimal
+        ) AS amount,
+        d.liquidity_provider,
+        d.liquidity_pool_address,
+        d._inserted_timestamp
+    FROM
+        base_burn_actions m
+        INNER JOIN decoded d
+        ON m.block_timestamp :: DATE = d.block_timestamp :: DATE
+        AND m.tx_id = d.tx_id
+        AND m.index = d.index
+    WHERE
+        m.mint = d.liquidity_pool_mint
+),
+pre_final AS (
+    SELECT
+        *
+    FROM
+        deposits
+    UNION ALL
+    SELECT
+        *
+    FROM
+        withdraws
+    UNION ALL
+    SELECT
+        *
+    FROM
+        mints
+    UNION ALL
+    SELECT
+        *
+    FROM
+        burns
+)
+SELECT 
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    index,
+    inner_index,
+    program_id,
+    action,
+    mint,
+    amount,
+    liquidity_provider,
+    liquidity_pool_address,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(['tx_id','index','inner_index']) }} as liquidity_pool_actions_meteora_id,
+    sysdate() as inserted_timestamp,
+    sysdate() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM 
+    pre_final
--- a/models/silver/liquidity_pool/silver__liquidity_pool_actions_meteora.yml
+++ b/models/silver/liquidity_pool/silver__liquidity_pool_actions_meteora.yml
@ -0,0 +1,109 @@
+version: 2
+models:
+  - name: silver__liquidity_pool_actions_meteora
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_ID
+            - TX_ID
+            - INDEX
+            - INNER_INDEX
+          where: block_timestamp::date > current_date - 30
+      - compare_model_subset:
+          name: silver__liquidity_pool_actions_meteora_business_logic_test
+          compare_model: ref('testing__liquidity_pool_actions_meteora')
+          compare_columns: 
+            - tx_id
+            - index
+            - inner_index
+            - program_id
+            - action
+            - mint
+            - round(amount,8)
+            - liquidity_provider
+            - liquidity_pool_address
+          model_condition: "where tx_id in ('HQVXg5FRV93oJeGuqkXVe4KnoJvRW3bJA7fJwjw1obJLgWWFHr8mtEhnYnW6NaNthrM7vbvsSgwX5uvs3us79J3',
+          '5cuJ3DsDjCZtmADeusavqh1qsG52BBDthg82cJpaQkJRgHE1wCSYahuusCD6EoxNKPxd98847B5mo7g1X95Unfa3',
+          '3RTWUqPBYKzPS2QW6rxCsvXvFYCgmTbMhEbCTHn479RwMUHskdEzPuY4Sknu2qHNb27ek6BFU7XDkMQ2yJsgvkpT',
+          '4TYQhGdkBVnE2nF8RcSvWNwQKEPaACw9suBQP6Jjb5oEdJjxt75KaQeXaEMynoHGHQsWuWNZhtfHD3m4Y8t3x8d8',
+          'CsVci6T9dkkKD1nZN1gJiw7Wc47wSoV6DsCyD5K2uUXNPnV1e5R7vssG3czmmXfNoYuk2S5AqiVQRALZjMZVcyF')"
+    recent_date_filter: &recent_date_filter
+      config:
+        where: _inserted_timestamp >= current_date - 7
+    columns:
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
+          - not_null: *recent_date_filter
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: INDEX
+        description: "{{ doc('index') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: INNER_INDEX
+        description: "Position of event within inner instructions"
+        tests:
+          - not_null: *recent_date_filter
+      - name: PROGRAM_ID
+        description: "{{ doc('program_id') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: ACTION
+        description: "{{ doc('action') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: MINT
+        description: "{{ doc('mint') }}"
+        tests: 
+          - not_null:
+              where: block_id > 69393037 AND _inserted_timestamp >= current_date - 7
+      - name: AMOUNT
+        description: "{{ doc('amount') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: LIQUIDITY_PROVIDER
+        description: "{{ doc('liquidity_provider') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: LIQUIDITY_POOL_ADDRESS
+        description: "{{ doc('liquidity_pool_address') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: LIQUIDITY_POOL_ACTIONS_METEORA_ID
+        description: '{{ doc("pk") }}'   
+        tests: 
+          - unique: *recent_date_filter
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+        tests: 
+          - not_null: *recent_date_filter
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
+        tests: 
+          - not_null: *recent_date_filter
+      - name: _INVOCATION_ID
+        description: '{{ doc("_invocation_id") }}' 
+        tests: 
+          - not_null: 
+              name: test_silver__not_null_liquidity_pool_actions_meteora__invocation_id
+              <<: *recent_date_filter
+              
+
--- a/models/silver/parser/silver__decoded_instructions_combined.yml
+++ b/models/silver/parser/silver__decoded_instructions_combined.yml
@ -94,6 +94,16 @@ models:
                AND succeeded
                AND _inserted_timestamp between current_date - 7 AND current_timestamp() - INTERVAL '4 HOUR'
              to_condition: "_inserted_timestamp >= current_date - 7"
+          - dbt_utils.relationships_where:
+              name: dbt_utils_relationships_where_silver__decoded_instructions_combined_meteora_liquidity_pool_actions_tx_id
+              to: ref('silver__liquidity_pool_actions_meteora')
+              field: tx_id
+              from_condition: >
+                program_id = 'Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB'
+                AND event_type IN ('addBalanceLiquidity','addImbalanceLiquidity','bootstrapLiquidity','removeBalanceLiquidity','removeLiquiditySingleSide')
+                AND succeeded
+                AND coalesce(decoded_instruction:args:poolTokenAmount::int,-1) <> 0
+                and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR'
      - name: SIGNERS
      - name: SUCCEEDED
      - name: INDEX