An 5080/pumpfun swaps (#637)

* pumpfun models * silver update and add to gold * wip * clean up * add relationship test * formatting, remove hardcoded refs * remove backfill date predicate for pumpfun --------- Co-authored-by: desmond-hui <desmond@flipsidecrypto.com>
2026-02-06 13:56:50 +00:00 · 2024-08-19 10:30:01 -05:00 · 2024-08-19 10:30:01 -05:00 · 0faab509a8
commit 0faab509a8
parent e3f9e8dbad
6 changed files with 371 additions and 1 deletions
--- a/data/testing__swaps_pumpfun.csv
+++ b/data/testing__swaps_pumpfun.csv
@ -0,0 +1,7 @@
+tx_id, index, inner_index, swapper, from_mint, from_amount, to_mint, to_amount
+25NeiLaNVjyx8R2fsHoebnWwF1SkU8C5pyYCSRH6ohY9s8uNyeW9Ddk1YesftiUsEioRKsdQaWEQA9q9W8zu8y51,3,3,5wXBY9oK3qqxLwwS5ogrVDbQ1JqRV8ShAZb76eEMgqAo,So11111111111111111111111111111111111111112,0.020008291,3dtoLRpbuHtViwmsbRMfLVx5jbSEFW3t8Qwa76Vjpump,276314.421934
+25NeiLaNVjyx8R2fsHoebnWwF1SkU8C5pyYCSRH6ohY9s8uNyeW9Ddk1YesftiUsEioRKsdQaWEQA9q9W8zu8y51,4,1,5wXBY9oK3qqxLwwS5ogrVDbQ1JqRV8ShAZb76eEMgqAo,3dtoLRpbuHtViwmsbRMfLVx5jbSEFW3t8Qwa76Vjpump,276314.421934,So11111111111111111111111111111111111111112,0.02000829
+3RD28NDFPfHo5R3cr9fTZkLM9L2igFs3QUbCddHhoansimBVPGMZZkdQZWYD3xNtYecemSmTP328sR9PBeDFyUDb,2,3,JCx39HVyE73Wn17vdfjWaCe7QDuumSmW3RCWCWuwuht4,So11111111111111111111111111111111111111112,0.029955309,3hyUAudtmpG79RyQZbaoo3XVUXsuxKrZbhLr52WVpump,387192.766698
+3RD28NDFPfHo5R3cr9fTZkLM9L2igFs3QUbCddHhoansimBVPGMZZkdQZWYD3xNtYecemSmTP328sR9PBeDFyUDb,3,1,JCx39HVyE73Wn17vdfjWaCe7QDuumSmW3RCWCWuwuht4,3hyUAudtmpG79RyQZbaoo3XVUXsuxKrZbhLr52WVpump,387192.766698,So11111111111111111111111111111111111111112,0.029955308
+4MVDH9w5qBWMyuxVhDAGjkGpty6gSDMFaEj1qu5ppC2jErfa1ZYoTZAhPKN31NEr7hM7c2zmX8iTqhtNqUYeFYqq,2,2,Ek7QT3eSe16Lyoor2eaW6bBFz3W6XFf8FR57dmZvRvEZ,FKewpeekk5VfHzT1sM3bN7N1r8qjPAjhSFTuFhWBpump,8523894.705061,So11111111111111111111111111111111111111112,1.470942915
+663XSK9YNgjjQC98aja3JMZWJNLkYijHnsV6UURSXWakgHvLAq6hnuSfXP3o2N1CYEMLLCGQd5sxNDaE9QMWr3cN,3,3,6G8Cu53PRgm5aPHxMaZRguYHJfaNxmnmgoR129cKMvJk,So11111111111111111111111111111111111111112,0.08,GCQB2FwtUfureEe8HrcNWfFJfucJguFmDVZ6zUa3pump,316020.059367
--- a/models/gold/defi/defi__fact_swaps.sql
+++ b/models/gold/defi/defi__fact_swaps.sql
@ -322,6 +322,28 @@ WHERE modified_timestamp >= '{{ max_modified_timestamp }}'
 WHERE
    modified_timestamp::date < '{{ backfill_to_date }}'
 {% endif %}
+UNION ALL
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    succeeded,
+    swapper,
+    from_amount AS swap_from_amount,
+    from_mint AS swap_from_mint,
+    to_amount AS swap_to_amount,
+    to_mint AS swap_to_mint,
+    program_id,
+    swap_index,
+    swaps_pumpfun_id as fact_swaps_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__swaps_pumpfun') }}
+{% if is_incremental() %}
+WHERE
+    modified_timestamp >= '{{ max_modified_timestamp }}'
+{% endif %}
 )

 select 
--- a/models/silver/parser/silver__decoded_logs.yml
+++ b/models/silver/parser/silver__decoded_logs.yml
@ -20,6 +20,16 @@ models:
        description: "{{ doc('tx_id') }}"
        tests: 
          - not_null: *recent_date_filter
+          - dbt_utils.relationships_where:
+              name: dbt_utils_relationships_where_silver__decoded_logs_swaps_pumpfun_tx_id
+              to: ref('silver__swaps_pumpfun')
+              field: tx_id
+              from_condition: >
+                program_id = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P'
+                AND event_type = 'TradeEvent'
+                AND succeeded
+                and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR'
+              to_condition: "_inserted_timestamp >= current_date - 7"
      - name: INDEX
        description: "{{ doc('event_index') }}"
        tests: 
--- a/models/silver/swaps/jupiter/v6/silver__swaps_inner_intermediate_jupiterv6.sql
+++ b/models/silver/swaps/jupiter/v6/silver__swaps_inner_intermediate_jupiterv6.sql
@ -205,7 +205,7 @@ distinct_missing_decimals AS (
 get_missing_decimals AS (
    SELECT
        mint,
-        solana.live.udf_api(
+        live.udf_api(
            'POST',
            '{service}/{Authentication}',
            OBJECT_CONSTRUCT(
--- a/models/silver/swaps/pumpfun/silver__swaps_pumpfun.sql
+++ b/models/silver/swaps/pumpfun/silver__swaps_pumpfun.sql
@ -0,0 +1,227 @@
+-- depends_on: {{ ref('silver__decoded_logs') }}
+
+{{ config(
+    materialized = 'incremental',
+    unique_key = "swaps_pumpfun_id",
+    incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
+    merge_exclude_columns = ["inserted_timestamp"],
+    cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'],
+    post_hook = enable_search_optimization(
+        '{{this.schema}}',
+        '{{this.identifier}}',
+        'ON EQUALITY(tx_id, swapper, from_mint, to_mint)'
+    ),
+    tags = ['scheduled_non_core'],
+) }}
+
+{% if execute %}
+    {% set base_query %}
+        CREATE OR REPLACE TEMPORARY TABLE silver.swaps_pumpfun__intermediate_tmp AS
+        SELECT
+            *
+        FROM
+            {{ ref('silver__decoded_logs') }}
+        WHERE
+            program_id = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P' 
+            AND event_type = 'TradeEvent'
+            AND succeeded
+
+        {% if is_incremental() %}
+        AND _inserted_timestamp >= (
+            SELECT
+                MAX(_inserted_timestamp) - INTERVAL '1 hour'
+            FROM
+                {{ this }}
+        )
+        {% else %}
+            AND _inserted_timestamp :: DATE >= '2024-08-13'
+        {% endif %}
+    {% endset %}
+
+    {% do run_query(base_query) %}
+    {% set between_stmts = fsc_utils.dynamic_range_predicate(
+        "silver.swaps_pumpfun__intermediate_tmp",
+        "block_timestamp::date"
+    ) %}
+{% endif %}
+
+WITH base AS (
+    SELECT 
+        *
+    FROM
+        silver.swaps_pumpfun__intermediate_tmp
+),
+token_decimals AS (
+    SELECT 
+        mint,
+        decimal
+    FROM
+        {{ ref('silver__decoded_metadata') }}
+    UNION ALL 
+    SELECT 
+        'So11111111111111111111111111111111111111112',
+        9
+    UNION ALL 
+    SELECT 
+        'GyD5AvrcZAhSP5rrhXXGPUHri6sbkRpq67xfG3x8ourT',
+        9
+),
+swaps as (
+    SELECT 
+        block_timestamp,
+        block_id,
+        tx_id,
+        index,
+        inner_index,
+        succeeded,
+        program_id,
+        decoded_log:args:isBuy::boolean AS is_buy,
+        decoded_log:args:user::string AS swapper,
+        decoded_log:args:mint::string AS mint,
+        CASE 
+            WHEN is_buy THEN
+                'So11111111111111111111111111111111111111112'
+            ELSE
+                decoded_log:args:mint
+        END AS from_mint,
+        CASE
+            WHEN is_buy THEN 
+                decoded_log:args:mint
+            ELSE 
+                'So11111111111111111111111111111111111111112'
+        END AS to_mint,
+        CASE 
+            WHEN is_buy THEN 
+                decoded_log:args:solAmount::string
+            ELSE 
+                decoded_log:args:tokenAmount::string
+        END AS from_amount,
+        CASE 
+            WHEN is_buy THEN 
+                decoded_log:args:tokenAmount::string
+            ELSE 
+                decoded_log:args:solAmount::string
+        END AS to_amount,
+        _inserted_timestamp
+    FROM
+        base
+),
+pre_final AS (
+    SELECT
+        a.block_timestamp,
+        a.block_id,
+        a.program_id,
+        a.tx_id,
+        a.succeeded,
+        row_number() OVER (
+            PARTITION BY a.tx_id
+            ORDER BY
+                a.INDEX,
+                a.inner_index
+        ) AS swap_index,
+        a.index,
+        a.inner_index,
+        a.swapper,
+        a.from_mint,
+        a.to_mint,
+        a.from_amount AS from_amount_int,
+        a.from_amount * pow(10,-d.decimal) AS from_amount,
+        a.to_amount AS to_amount_int,
+        a.to_amount * pow(10,-d2.decimal) AS to_amount,
+        a._inserted_timestamp,
+        {{ dbt_utils.generate_surrogate_key(['a.tx_id','a.index','a.inner_index']) }} AS swaps_pumpfun_id,
+        sysdate() AS inserted_timestamp,
+        sysdate() AS modified_timestamp,
+        '{{ invocation_id }}' AS _invocation_id
+    FROM
+        swaps a
+    LEFT OUTER JOIN
+        token_decimals d
+        ON a.from_mint = d.mint
+    LEFT OUTER JOIN
+        token_decimals d2
+        ON a.to_mint = d2.mint
+),
+distinct_missing_decimals AS (
+    SELECT DISTINCT
+        to_mint AS mint
+    FROM
+        pre_final
+    WHERE
+        to_amount IS NULL
+    UNION
+    SELECT DISTINCT
+        from_mint
+    FROM
+        pre_final
+    WHERE
+        from_amount IS NULL
+),
+get_missing_decimals AS (
+    SELECT
+        mint,
+        live.udf_api(
+            'POST',
+            '{service}/{Authentication}',
+            OBJECT_CONSTRUCT(
+                'Content-Type',
+                'application/json'
+            ),
+            OBJECT_CONSTRUCT(
+                'id',
+                1,
+                'jsonrpc',
+                '2.0',
+                'method',
+                'getAccountInfo',
+                'params',
+                ARRAY_CONSTRUCT(
+                    mint,
+                    OBJECT_CONSTRUCT(
+                        'encoding',
+                        'jsonParsed'
+                    )
+                )
+            ),
+            'Vault/prod/solana/quicknode/mainnet'
+        ):data:result:value:data:parsed:info:decimals::int AS decimal
+    FROM
+        distinct_missing_decimals
+)
+SELECT
+    block_timestamp,
+    block_id,
+    tx_id,
+    index,
+    inner_index,
+    swap_index,
+    succeeded,
+    program_id,
+    swapper,
+    from_mint,
+    CASE
+        WHEN from_amount IS NULL THEN
+            from_amount_int * pow(10, -d.decimal)
+        ELSE
+            from_amount
+    END AS from_amount,
+    to_mint,
+    CASE
+        WHEN to_amount IS NULL THEN
+            to_amount_int * pow(10, -d2.decimal)
+        ELSE
+            to_amount
+    END AS to_amount,
+    _inserted_timestamp,
+    swaps_pumpfun_id,
+    inserted_timestamp,
+    modified_timestamp,
+    _invocation_id
+FROM
+    pre_final
+LEFT OUTER JOIN
+    get_missing_decimals d
+    ON from_mint = d.mint
+LEFT OUTER JOIN
+    get_missing_decimals d2
+    ON to_mint = d2.mint
--- a/models/silver/swaps/pumpfun/silver__swaps_pumpfun.yml
+++ b/models/silver/swaps/pumpfun/silver__swaps_pumpfun.yml
@ -0,0 +1,104 @@
+version: 2
+models:
+  - name: silver__swaps_pumpfun
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_ID
+            - INDEX
+            - INNER_INDEX
+          where: block_timestamp::date > current_date - 30
+      - compare_model_subset:
+          name: silver__swaps_pumpfun_business_logic_test
+          compare_model: ref('testing__swaps_pumpfun')
+          compare_columns: 
+            - tx_id
+            - index
+            - inner_index
+            - swapper
+            - from_mint
+            - round(from_amount,2)
+            - to_mint
+            - round(to_amount,2)
+          model_condition: "where tx_id in ('25NeiLaNVjyx8R2fsHoebnWwF1SkU8C5pyYCSRH6ohY9s8uNyeW9Ddk1YesftiUsEioRKsdQaWEQA9q9W8zu8y51',
+            '3RD28NDFPfHo5R3cr9fTZkLM9L2igFs3QUbCddHhoansimBVPGMZZkdQZWYD3xNtYecemSmTP328sR9PBeDFyUDb',
+            '4MVDH9w5qBWMyuxVhDAGjkGpty6gSDMFaEj1qu5ppC2jErfa1ZYoTZAhPKN31NEr7hM7c2zmX8iTqhtNqUYeFYqq',
+            '663XSK9YNgjjQC98aja3JMZWJNLkYijHnsV6UURSXWakgHvLAq6hnuSfXP3o2N1CYEMLLCGQd5sxNDaE9QMWr3cN')"  
+    recent_date_filter: &recent_date_filter
+      config:
+        where: _inserted_timestamp >= current_date - 7
+    columns:
+      - name: BLOCK_TIMESTAMP
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null: *recent_date_filter
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2
+      - name: BLOCK_ID
+        description: "{{ doc('block_id') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: TX_ID
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null: *recent_date_filter
+      - name: INDEX
+        description: "{{ doc('event_index') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: INNER_INDEX
+        description: "{{ doc('inner_index') }}. This is the inner index of the log event listing the inner swap"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: SUCCEEDED
+        description: "{{ doc('tx_succeeded') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: PROGRAM_ID
+        description: "{{ doc('program_id') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: SWAPPER
+        description: "{{ doc('swaps_swapper') }}"
+        tests: 
+          - not_null:
+              where: _inserted_timestamp between current_date - 7 and current_date - 2 # allow it to be null during lookback buffer
+      - name: FROM_AMOUNT
+        description:  "{{ doc('swaps_from_amt') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: FROM_MINT
+        description:  "{{ doc('swaps_from_mint') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: TO_AMOUNT
+        description:  "{{ doc('swaps_to_amt') }}"
+        tests: 
+          - not_null: *recent_date_filter
+      - name: TO_MINT
+        description:  "{{ doc('swaps_to_mint') }}"
+        tests: 
+          - not_null: *recent_date_filter 
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests: 
+          - not_null
+      - name: SWAPS_PUMPFUN_ID
+        description: '{{ doc("pk") }}'   
+        tests: 
+          - unique: *recent_date_filter
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+        tests: 
+          - not_null: *recent_date_filter
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
+        tests: 
+          - not_null: *recent_date_filter
+      - name: _INVOCATION_ID
+        description: '{{ doc("_invocation_id") }}' 
+        tests: 
+          - not_null: 
+              name: test_silver__not_null_swaps_pumpfun__invocation_id
+              <<: *recent_date_filter