diff --git a/data/testing__swaps_pumpfun.csv b/data/testing__swaps_pumpfun.csv new file mode 100644 index 00000000..0cc968fc --- /dev/null +++ b/data/testing__swaps_pumpfun.csv @@ -0,0 +1,7 @@ +tx_id, index, inner_index, swapper, from_mint, from_amount, to_mint, to_amount +25NeiLaNVjyx8R2fsHoebnWwF1SkU8C5pyYCSRH6ohY9s8uNyeW9Ddk1YesftiUsEioRKsdQaWEQA9q9W8zu8y51,3,3,5wXBY9oK3qqxLwwS5ogrVDbQ1JqRV8ShAZb76eEMgqAo,So11111111111111111111111111111111111111112,0.020008291,3dtoLRpbuHtViwmsbRMfLVx5jbSEFW3t8Qwa76Vjpump,276314.421934 +25NeiLaNVjyx8R2fsHoebnWwF1SkU8C5pyYCSRH6ohY9s8uNyeW9Ddk1YesftiUsEioRKsdQaWEQA9q9W8zu8y51,4,1,5wXBY9oK3qqxLwwS5ogrVDbQ1JqRV8ShAZb76eEMgqAo,3dtoLRpbuHtViwmsbRMfLVx5jbSEFW3t8Qwa76Vjpump,276314.421934,So11111111111111111111111111111111111111112,0.02000829 +3RD28NDFPfHo5R3cr9fTZkLM9L2igFs3QUbCddHhoansimBVPGMZZkdQZWYD3xNtYecemSmTP328sR9PBeDFyUDb,2,3,JCx39HVyE73Wn17vdfjWaCe7QDuumSmW3RCWCWuwuht4,So11111111111111111111111111111111111111112,0.029955309,3hyUAudtmpG79RyQZbaoo3XVUXsuxKrZbhLr52WVpump,387192.766698 +3RD28NDFPfHo5R3cr9fTZkLM9L2igFs3QUbCddHhoansimBVPGMZZkdQZWYD3xNtYecemSmTP328sR9PBeDFyUDb,3,1,JCx39HVyE73Wn17vdfjWaCe7QDuumSmW3RCWCWuwuht4,3hyUAudtmpG79RyQZbaoo3XVUXsuxKrZbhLr52WVpump,387192.766698,So11111111111111111111111111111111111111112,0.029955308 +4MVDH9w5qBWMyuxVhDAGjkGpty6gSDMFaEj1qu5ppC2jErfa1ZYoTZAhPKN31NEr7hM7c2zmX8iTqhtNqUYeFYqq,2,2,Ek7QT3eSe16Lyoor2eaW6bBFz3W6XFf8FR57dmZvRvEZ,FKewpeekk5VfHzT1sM3bN7N1r8qjPAjhSFTuFhWBpump,8523894.705061,So11111111111111111111111111111111111111112,1.470942915 +663XSK9YNgjjQC98aja3JMZWJNLkYijHnsV6UURSXWakgHvLAq6hnuSfXP3o2N1CYEMLLCGQd5sxNDaE9QMWr3cN,3,3,6G8Cu53PRgm5aPHxMaZRguYHJfaNxmnmgoR129cKMvJk,So11111111111111111111111111111111111111112,0.08,GCQB2FwtUfureEe8HrcNWfFJfucJguFmDVZ6zUa3pump,316020.059367 diff --git a/models/gold/defi/defi__fact_swaps.sql b/models/gold/defi/defi__fact_swaps.sql index e88f212a..f3f807fd 100644 --- a/models/gold/defi/defi__fact_swaps.sql +++ b/models/gold/defi/defi__fact_swaps.sql @@ -322,6 +322,28 @@ WHERE modified_timestamp >= '{{ max_modified_timestamp }}' WHERE modified_timestamp::date < '{{ backfill_to_date }}' {% endif %} +UNION ALL +SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + swapper, + from_amount AS swap_from_amount, + from_mint AS swap_from_mint, + to_amount AS swap_to_amount, + to_mint AS swap_to_mint, + program_id, + swap_index, + swaps_pumpfun_id as fact_swaps_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__swaps_pumpfun') }} +{% if is_incremental() %} +WHERE + modified_timestamp >= '{{ max_modified_timestamp }}' +{% endif %} ) select diff --git a/models/silver/parser/silver__decoded_logs.yml b/models/silver/parser/silver__decoded_logs.yml index 8001b8ea..10e5f067 100644 --- a/models/silver/parser/silver__decoded_logs.yml +++ b/models/silver/parser/silver__decoded_logs.yml @@ -20,6 +20,16 @@ models: description: "{{ doc('tx_id') }}" tests: - not_null: *recent_date_filter + - dbt_utils.relationships_where: + name: dbt_utils_relationships_where_silver__decoded_logs_swaps_pumpfun_tx_id + to: ref('silver__swaps_pumpfun') + field: tx_id + from_condition: > + program_id = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P' + AND event_type = 'TradeEvent' + AND succeeded + and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' + to_condition: "_inserted_timestamp >= current_date - 7" - name: INDEX description: "{{ doc('event_index') }}" tests: diff --git a/models/silver/swaps/jupiter/v6/silver__swaps_inner_intermediate_jupiterv6.sql b/models/silver/swaps/jupiter/v6/silver__swaps_inner_intermediate_jupiterv6.sql index 918242e6..ea333aea 100644 --- a/models/silver/swaps/jupiter/v6/silver__swaps_inner_intermediate_jupiterv6.sql +++ b/models/silver/swaps/jupiter/v6/silver__swaps_inner_intermediate_jupiterv6.sql @@ -205,7 +205,7 @@ distinct_missing_decimals AS ( get_missing_decimals AS ( SELECT mint, - solana.live.udf_api( + live.udf_api( 'POST', '{service}/{Authentication}', OBJECT_CONSTRUCT( diff --git a/models/silver/swaps/pumpfun/silver__swaps_pumpfun.sql b/models/silver/swaps/pumpfun/silver__swaps_pumpfun.sql new file mode 100644 index 00000000..035eef68 --- /dev/null +++ b/models/silver/swaps/pumpfun/silver__swaps_pumpfun.sql @@ -0,0 +1,227 @@ +-- depends_on: {{ ref('silver__decoded_logs') }} + +{{ config( + materialized = 'incremental', + unique_key = "swaps_pumpfun_id", + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + post_hook = enable_search_optimization( + '{{this.schema}}', + '{{this.identifier}}', + 'ON EQUALITY(tx_id, swapper, from_mint, to_mint)' + ), + tags = ['scheduled_non_core'], +) }} + +{% if execute %} + {% set base_query %} + CREATE OR REPLACE TEMPORARY TABLE silver.swaps_pumpfun__intermediate_tmp AS + SELECT + * + FROM + {{ ref('silver__decoded_logs') }} + WHERE + program_id = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P' + AND event_type = 'TradeEvent' + AND succeeded + + {% if is_incremental() %} + AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} + ) + {% else %} + AND _inserted_timestamp :: DATE >= '2024-08-13' + {% endif %} + {% endset %} + + {% do run_query(base_query) %} + {% set between_stmts = fsc_utils.dynamic_range_predicate( + "silver.swaps_pumpfun__intermediate_tmp", + "block_timestamp::date" + ) %} +{% endif %} + +WITH base AS ( + SELECT + * + FROM + silver.swaps_pumpfun__intermediate_tmp +), +token_decimals AS ( + SELECT + mint, + decimal + FROM + {{ ref('silver__decoded_metadata') }} + UNION ALL + SELECT + 'So11111111111111111111111111111111111111112', + 9 + UNION ALL + SELECT + 'GyD5AvrcZAhSP5rrhXXGPUHri6sbkRpq67xfG3x8ourT', + 9 +), +swaps as ( + SELECT + block_timestamp, + block_id, + tx_id, + index, + inner_index, + succeeded, + program_id, + decoded_log:args:isBuy::boolean AS is_buy, + decoded_log:args:user::string AS swapper, + decoded_log:args:mint::string AS mint, + CASE + WHEN is_buy THEN + 'So11111111111111111111111111111111111111112' + ELSE + decoded_log:args:mint + END AS from_mint, + CASE + WHEN is_buy THEN + decoded_log:args:mint + ELSE + 'So11111111111111111111111111111111111111112' + END AS to_mint, + CASE + WHEN is_buy THEN + decoded_log:args:solAmount::string + ELSE + decoded_log:args:tokenAmount::string + END AS from_amount, + CASE + WHEN is_buy THEN + decoded_log:args:tokenAmount::string + ELSE + decoded_log:args:solAmount::string + END AS to_amount, + _inserted_timestamp + FROM + base +), +pre_final AS ( + SELECT + a.block_timestamp, + a.block_id, + a.program_id, + a.tx_id, + a.succeeded, + row_number() OVER ( + PARTITION BY a.tx_id + ORDER BY + a.INDEX, + a.inner_index + ) AS swap_index, + a.index, + a.inner_index, + a.swapper, + a.from_mint, + a.to_mint, + a.from_amount AS from_amount_int, + a.from_amount * pow(10,-d.decimal) AS from_amount, + a.to_amount AS to_amount_int, + a.to_amount * pow(10,-d2.decimal) AS to_amount, + a._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['a.tx_id','a.index','a.inner_index']) }} AS swaps_pumpfun_id, + sysdate() AS inserted_timestamp, + sysdate() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + swaps a + LEFT OUTER JOIN + token_decimals d + ON a.from_mint = d.mint + LEFT OUTER JOIN + token_decimals d2 + ON a.to_mint = d2.mint +), +distinct_missing_decimals AS ( + SELECT DISTINCT + to_mint AS mint + FROM + pre_final + WHERE + to_amount IS NULL + UNION + SELECT DISTINCT + from_mint + FROM + pre_final + WHERE + from_amount IS NULL +), +get_missing_decimals AS ( + SELECT + mint, + live.udf_api( + 'POST', + '{service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + 1, + 'jsonrpc', + '2.0', + 'method', + 'getAccountInfo', + 'params', + ARRAY_CONSTRUCT( + mint, + OBJECT_CONSTRUCT( + 'encoding', + 'jsonParsed' + ) + ) + ), + 'Vault/prod/solana/quicknode/mainnet' + ):data:result:value:data:parsed:info:decimals::int AS decimal + FROM + distinct_missing_decimals +) +SELECT + block_timestamp, + block_id, + tx_id, + index, + inner_index, + swap_index, + succeeded, + program_id, + swapper, + from_mint, + CASE + WHEN from_amount IS NULL THEN + from_amount_int * pow(10, -d.decimal) + ELSE + from_amount + END AS from_amount, + to_mint, + CASE + WHEN to_amount IS NULL THEN + to_amount_int * pow(10, -d2.decimal) + ELSE + to_amount + END AS to_amount, + _inserted_timestamp, + swaps_pumpfun_id, + inserted_timestamp, + modified_timestamp, + _invocation_id +FROM + pre_final +LEFT OUTER JOIN + get_missing_decimals d + ON from_mint = d.mint +LEFT OUTER JOIN + get_missing_decimals d2 + ON to_mint = d2.mint diff --git a/models/silver/swaps/pumpfun/silver__swaps_pumpfun.yml b/models/silver/swaps/pumpfun/silver__swaps_pumpfun.yml new file mode 100644 index 00000000..4b7498a7 --- /dev/null +++ b/models/silver/swaps/pumpfun/silver__swaps_pumpfun.yml @@ -0,0 +1,104 @@ +version: 2 +models: + - name: silver__swaps_pumpfun + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - INDEX + - INNER_INDEX + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_pumpfun_business_logic_test + compare_model: ref('testing__swaps_pumpfun') + compare_columns: + - tx_id + - index + - inner_index + - swapper + - from_mint + - round(from_amount,2) + - to_mint + - round(to_amount,2) + model_condition: "where tx_id in ('25NeiLaNVjyx8R2fsHoebnWwF1SkU8C5pyYCSRH6ohY9s8uNyeW9Ddk1YesftiUsEioRKsdQaWEQA9q9W8zu8y51', + '3RD28NDFPfHo5R3cr9fTZkLM9L2igFs3QUbCddHhoansimBVPGMZZkdQZWYD3xNtYecemSmTP328sR9PBeDFyUDb', + '4MVDH9w5qBWMyuxVhDAGjkGpty6gSDMFaEj1qu5ppC2jErfa1ZYoTZAhPKN31NEr7hM7c2zmX8iTqhtNqUYeFYqq', + '663XSK9YNgjjQC98aja3JMZWJNLkYijHnsV6UURSXWakgHvLAq6hnuSfXP3o2N1CYEMLLCGQd5sxNDaE9QMWr3cN')" + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: *recent_date_filter + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('event_index') }}" + tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_index') }}. This is the inner index of the log event listing the inner swap" + tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null: *recent_date_filter + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null: *recent_date_filter + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: _inserted_timestamp between current_date - 7 and current_date - 2 # allow it to be null during lookback buffer + - name: FROM_AMOUNT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null: *recent_date_filter + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null: *recent_date_filter + - name: TO_AMOUNT + description: "{{ doc('swaps_to_amt') }}" + tests: + - not_null: *recent_date_filter + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + tests: + - not_null: *recent_date_filter + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null + - name: SWAPS_PUMPFUN_ID + description: '{{ doc("pk") }}' + tests: + - unique: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + tests: + - not_null: + name: test_silver__not_null_swaps_pumpfun__invocation_id + <<: *recent_date_filter \ No newline at end of file