From 3cf4b17de58e6ad31a25cc4b6219876100fbe57e Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Thu, 30 May 2024 13:53:00 -0700 Subject: [PATCH] An 4825/raydium swaps (#565) * wip * wip * clean up * update date filter * update per pr comments * remove unnecessary model --- ...sting__swaps_intermediate_raydium_clmm.csv | 6 + ...ing__swaps_intermediate_raydium_stable.csv | 3 + ...ing__swaps_intermediate_raydium_v4_amm.csv | 6 + .../silver__decoded_instructions_combined.yml | 30 +++ ...ilver__swaps_intermediate_raydium_clmm.sql | 162 +++++++++++++ ...ilver__swaps_intermediate_raydium_clmm.yml | 105 ++++++++ ...ver__swaps_intermediate_raydium_stable.sql | 203 ++++++++++++++++ ...ver__swaps_intermediate_raydium_stable.yml | 104 ++++++++ ...ver__swaps_intermediate_raydium_v4_amm.sql | 228 ++++++++++++++++++ ...ver__swaps_intermediate_raydium_v4_amm.yml | 105 ++++++++ 10 files changed, 952 insertions(+) create mode 100644 data/testing__swaps_intermediate_raydium_clmm.csv create mode 100644 data/testing__swaps_intermediate_raydium_stable.csv create mode 100644 data/testing__swaps_intermediate_raydium_v4_amm.csv create mode 100644 models/silver/swaps/silver__swaps_intermediate_raydium_clmm.sql create mode 100644 models/silver/swaps/silver__swaps_intermediate_raydium_clmm.yml create mode 100644 models/silver/swaps/silver__swaps_intermediate_raydium_stable.sql create mode 100644 models/silver/swaps/silver__swaps_intermediate_raydium_stable.yml create mode 100644 models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.sql create mode 100644 models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.yml diff --git a/data/testing__swaps_intermediate_raydium_clmm.csv b/data/testing__swaps_intermediate_raydium_clmm.csv new file mode 100644 index 00000000..9d03d16b --- /dev/null +++ b/data/testing__swaps_intermediate_raydium_clmm.csv @@ -0,0 +1,6 @@ +tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index +25DbM3XCMsxZe7cUXG2eRmsbjUWD7KLrgpH7a8wtsRsnRg4magjPUFgqjGiwtZTFP4uXNCXit3Xb5PN7qHLkr28F,GGztQqQ6pCPaJQnNpXBgELr5cs3WwDakRbh1iEMzjgSJ,255.941051,Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB,1.332422904,So11111111111111111111111111111111111111112,1 +yvyupEuKFdCdHyjuUivXS9NbVbpf9rZYKFSt7LxGZTaTaqJdZmA8h5SKJ3GfPuHKz55ejEGgdyCiVULXMosTZCK,MfDuWeqSHEqTFVYZ7LoexgAK9dxk7cy4DFJWjWMGVWa,2.036985403,So11111111111111111111111111111111111111112,406.375663,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1 +22tk9BXRx3Tv7rBimVah6X6Yjfb7PMNA13g21o9VT8eMc1VKPUttq9sfBeiPmB7N4EqzLfb9rF2K9UWKrqy5kPRD,projjosVCPQH49d5em7VYS7fJZzaqKixqKtus7yk416,18.297738512,So11111111111111111111111111111111111111112,1427.91122,4k3Dyjzvzp8eMZWUXbBCjEvwSkkk59S5iCNLY3QrkX6R,1 +22tk9BXRx3Tv7rBimVah6X6Yjfb7PMNA13g21o9VT8eMc1VKPUttq9sfBeiPmB7N4EqzLfb9rF2K9UWKrqy5kPRD,projjosVCPQH49d5em7VYS7fJZzaqKixqKtus7yk416,0.00023084,3NZ9JMVBmGAqocybic2c7LQCJScmgsAZ6vQqTDzcqmJh,0.106570479,So11111111111111111111111111111111111111112,2 +22tk9BXRx3Tv7rBimVah6X6Yjfb7PMNA13g21o9VT8eMc1VKPUttq9sfBeiPmB7N4EqzLfb9rF2K9UWKrqy5kPRD,projjosVCPQH49d5em7VYS7fJZzaqKixqKtus7yk416,0.106570479,So11111111111111111111111111111111111111112,8.313137,4k3Dyjzvzp8eMZWUXbBCjEvwSkkk59S5iCNLY3QrkX6R,3 \ No newline at end of file diff --git a/data/testing__swaps_intermediate_raydium_stable.csv b/data/testing__swaps_intermediate_raydium_stable.csv new file mode 100644 index 00000000..991d4a0e --- /dev/null +++ b/data/testing__swaps_intermediate_raydium_stable.csv @@ -0,0 +1,3 @@ +tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index +31JsUv3c9J5FMFJgxrBTeXghAVK3qnAKEjCBrZTXXjXvgxh6LUCc1yRuoNDXXDyywxwvcKxKGMaQh3GCsGLnAFAo,D7fbzwdBxdn3yNqTDZs4PJLnXnBDMcQT6RmmgBNdYVaW,3.659271,USDH1SM1ojwWUga67PGrgFWUHibbjqMvuMaDkRJTgkX,3.509682,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,1 +4hCaNMB42Gg1YedK91GzZKpJS25sGpgB2sri8vatLn6S4WUU3MKCANTXepsBPjPewzjhefeTUFKG36BTr8y7hweh,HU58ZPNjL7TZUhbAgz7zWxdUxh7zG9yUgvdeENaovuFP,254.532412,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,254.535243,Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB,1 \ No newline at end of file diff --git a/data/testing__swaps_intermediate_raydium_v4_amm.csv b/data/testing__swaps_intermediate_raydium_v4_amm.csv new file mode 100644 index 00000000..58f2f3b6 --- /dev/null +++ b/data/testing__swaps_intermediate_raydium_v4_amm.csv @@ -0,0 +1,6 @@ +tx_id,swapper,from_amt,from_mint,to_amt,to_mint,swap_index +3PmcuNbasp87STwtHkVXK8wZpqkfTdBa9p3LYT3Sp3G2BUjvfLWoqniQn222dW8d1ZhUWi2VrURTAwNWhBvGRQRH,5xMziPA43bsT5j97c8QqwUAKLBPg1ZN95bf6p5pQdRKV,90.473330516,FU1q8vJpZNUrmqsciSjp8bAKKidGsLmouB8CBdf8TKQv,0.291926008,So11111111111111111111111111111111111111112,1 +3x3HNjr34Vxzbsz29DAAm2Ngnrv7p9FXYGG5yw3MWiE6W4mt79Poe4axw2Lx4dXn3hMCDeDNVyiffE6kb3MhGGzp,AmcHQCkULiWhWxVB2qa98rksYjkzk8DJogvXbnjYRZHN,19934.810615,BxXmDhM8sTF3QG4foaVM2v1EUvg9DLSVUsDRTjR8tMyS,0.079963971,So11111111111111111111111111111111111111112,1 +3x3HNjr34Vxzbsz29DAAm2Ngnrv7p9FXYGG5yw3MWiE6W4mt79Poe4axw2Lx4dXn3hMCDeDNVyiffE6kb3MhGGzp,AmcHQCkULiWhWxVB2qa98rksYjkzk8DJogvXbnjYRZHN,0.079963971,So11111111111111111111111111111111111111112,19835.300448,BxXmDhM8sTF3QG4foaVM2v1EUvg9DLSVUsDRTjR8tMyS,2 +3z249coLZuL2jViqt3LfNpDhi12dweanrv7VUJYVtGJ9t79GpN87R2HTF9x4dyF544jX6AxEMquQVKNxB9NehoDf,3b3zfYBQ61sEicY4bhQMKUvRo9bAxPYAz32uyAP9Cj9u,0.042146155,So11111111111111111111111111111111111111112,8811.925102,9tzZzEHsKnwFL1A3DyFJwj36KnZj3gZ7g4srWp9YTEoh,1 +3z249coLZuL2jViqt3LfNpDhi12dweanrv7VUJYVtGJ9t79GpN87R2HTF9x4dyF544jX6AxEMquQVKNxB9NehoDf,3b3zfYBQ61sEicY4bhQMKUvRo9bAxPYAz32uyAP9Cj9u,8811.925151,9tzZzEHsKnwFL1A3DyFJwj36KnZj3gZ7g4srWp9YTEoh,6.431207,EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v,2 \ No newline at end of file diff --git a/models/silver/parser/silver__decoded_instructions_combined.yml b/models/silver/parser/silver__decoded_instructions_combined.yml index 209be3a5..1bbc733e 100644 --- a/models/silver/parser/silver__decoded_instructions_combined.yml +++ b/models/silver/parser/silver__decoded_instructions_combined.yml @@ -126,6 +126,36 @@ models: AND succeeded and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' to_condition: "_inserted_timestamp >= current_date - 7" + - dbt_utils.relationships_where: + name: dbt_utils_relationships_where_silver__decoded_instructions_combined_swaps_intermediate_raydium_clmm_tx_id + to: ref('silver__swaps_intermediate_raydium_clmm') + field: tx_id + from_condition: > + program_id = 'CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK' + AND event_type IN ('swapV2','swap') + AND succeeded + and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' + to_condition: "_inserted_timestamp >= current_date - 7" + - dbt_utils.relationships_where: + name: dbt_utils_relationships_where_silver__decoded_instructions_combined_swaps_intermediate_raydium_v4_amm_tx_id + to: ref('silver__swaps_intermediate_raydium_v4_amm') + field: tx_id + from_condition: > + program_id = '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8' + AND event_type IN ('swapBaseIn','swapBaseOut') + AND succeeded + and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' + to_condition: "_inserted_timestamp >= current_date - 7" + - dbt_utils.relationships_where: + name: dbt_utils_relationships_where_silver__decoded_instructions_combined_swaps_intermediate_raydium_stable_tx_id + to: ref('silver__swaps_intermediate_raydium_stable') + field: tx_id + from_condition: > + program_id = '5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h' + AND event_type IN ('swapBaseIn') + AND succeeded + and _inserted_timestamp between current_date - 7 and current_timestamp() - INTERVAL '4 HOUR' + to_condition: "_inserted_timestamp >= current_date - 7" - name: SIGNERS - name: SUCCEEDED - name: INDEX diff --git a/models/silver/swaps/silver__swaps_intermediate_raydium_clmm.sql b/models/silver/swaps/silver__swaps_intermediate_raydium_clmm.sql new file mode 100644 index 00000000..6f4a0fe1 --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_raydium_clmm.sql @@ -0,0 +1,162 @@ +-- depends_on: {{ ref('silver__decoded_instructions_combined') }} + +{{ config( + materialized = 'incremental', + unique_key = ['swaps_intermediate_raydium_clmm_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% set base_query %} + CREATE OR REPLACE TEMPORARY TABLE silver.swaps_intermediate_raydium_clmm__intermediate_tmp AS + SELECT + block_timestamp, + block_id, + tx_id, + succeeded, + INDEX, + inner_index, + program_id, + event_type, + decoded_instruction, + _inserted_timestamp + FROM + {{ ref('silver__decoded_instructions_combined') }} + WHERE + program_id = 'CAMMCzo5YL8w4VFF8KVHrK22GGUsp5VTaW7grrKgrWqK' + AND event_type in ('swapV2','swap') + + {% if is_incremental() %} + AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} + ) + {% else %} + AND _inserted_timestamp::DATE >= '2023-11-15' + {% endif %} + {% endset %} + + {% do run_query(base_query) %} + {% set between_stmts = fsc_utils.dynamic_range_predicate( + "silver.swaps_intermediate_raydium_clmm__intermediate_tmp", + "block_timestamp::date" + ) %} +{% endif %} + +WITH base AS ( + SELECT + * + FROM + silver.swaps_intermediate_raydium_clmm__intermediate_tmp +), +decoded AS ( + SELECT + block_timestamp, + block_id, + tx_id, + INDEX, + inner_index, + COALESCE(LEAD(inner_index) OVER (PARTITION BY tx_id, index + ORDER BY inner_index) -1, 999999 + ) AS inner_index_end, + program_id, + silver.udf_get_account_pubkey_by_name('payer', decoded_instruction:accounts) as swapper, + silver.udf_get_account_pubkey_by_name('inputTokenAccount', decoded_instruction:accounts) as source_token_account, + null as source_mint, + null as destination_mint, + silver.udf_get_account_pubkey_by_name('outputTokenAccount', decoded_instruction:accounts) as destination_token_account, + silver.udf_get_account_pubkey_by_name('outputVault', decoded_instruction:accounts) as program_destination_token_account, + silver.udf_get_account_pubkey_by_name('inputVault', decoded_instruction:accounts) as program_source_token_account, + _inserted_timestamp + FROM + base +), +transfers AS ( + SELECT + A.*, + COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1, + NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 + FROM + {{ ref('silver__transfers') }} A + INNER JOIN ( + SELECT + DISTINCT tx_id, + block_timestamp::DATE AS block_date + FROM + decoded + ) d + ON d.block_date = A.block_timestamp::DATE + AND d.tx_id = A.tx_id + WHERE + A.succeeded + AND {{ between_stmts }} +), +pre_final AS ( + SELECT + A.block_id, + A.block_timestamp, + A.program_id, + A.tx_id, + A.index, + A.inner_index, + A.inner_index_end, + C.succeeded, + A.swapper, + b.amount AS from_amt, + b.mint AS from_mint, + C.amount AS to_amt, + C.mint AS to_mint, + A._inserted_timestamp + FROM + decoded A + LEFT JOIN transfers b + ON A.tx_id = b.tx_id + AND A.source_token_account = b.source_token_account + AND A.program_source_token_account = b.dest_token_account + AND A.index = b.index_1 + AND ( + (b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + LEFT JOIN transfers C + ON A.tx_id = C.tx_id + AND A.destination_token_account = C.dest_token_account + AND A.program_destination_token_account = C.source_token_account + AND A.index = C.index_1 + AND ( + (C.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + QUALIFY ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX ORDER BY inner_index) = 1 +) +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + succeeded, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + INDEX, + inner_index + ) AS swap_index, + index, + inner_index, + swapper, + from_amt, + from_mint, + to_amt, + to_mint, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id','swap_index','program_id']) }} AS swaps_intermediate_raydium_clmm_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + pre_final diff --git a/models/silver/swaps/silver__swaps_intermediate_raydium_clmm.yml b/models/silver/swaps/silver__swaps_intermediate_raydium_clmm.yml new file mode 100644 index 00000000..f0e5376e --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_raydium_clmm.yml @@ -0,0 +1,105 @@ +version: 2 +models: + - name: silver__swaps_intermediate_raydium_clmm + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - SWAP_INDEX + - PROGRAM_ID + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_intermediate_raydium_clmm_business_logic_test + compare_model: ref('testing__swaps_intermediate_raydium_clmm') + compare_columns: + - tx_id + - swapper + - round(from_amt,8) + - from_mint + - round(to_amt,8) + - to_mint + - swap_index + model_condition: "where tx_id in ('25DbM3XCMsxZe7cUXG2eRmsbjUWD7KLrgpH7a8wtsRsnRg4magjPUFgqjGiwtZTFP4uXNCXit3Xb5PN7qHLkr28F', + 'yvyupEuKFdCdHyjuUivXS9NbVbpf9rZYKFSt7LxGZTaTaqJdZmA8h5SKJ3GfPuHKz55ejEGgdyCiVULXMosTZCK', + '22tk9BXRx3Tv7rBimVah6X6Yjfb7PMNA13g21o9VT8eMc1VKPUttq9sfBeiPmB7N4EqzLfb9rF2K9UWKrqy5kPRD' + )" + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - not_null: *recent_date_filter + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('index') }}" + tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_index') }}" + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null: *recent_date_filter + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: succeeded = TRUE + - name: FROM_AMT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null: *recent_date_filter + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null: *recent_date_filter + - name: TO_AMT + description: "{{ doc('swaps_to_amt') }}" + tests: + - not_null: *recent_date_filter + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + tests: + - not_null: *recent_date_filter + - name: SWAP_INDEX + description: "{{ doc('swaps_swap_index') }}" + tests: + - not_null: *recent_date_filter + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null: *recent_date_filter + - name: SWAPS_INTERMEDIATE_RAYDIUM_CLMM_ID + description: '{{ doc("pk") }}' + tests: + - unique: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + tests: + - not_null: + name: test_silver__not_null_swaps_intermediate_raydium_clmm__invocation_id + <<: *recent_date_filter \ No newline at end of file diff --git a/models/silver/swaps/silver__swaps_intermediate_raydium_stable.sql b/models/silver/swaps/silver__swaps_intermediate_raydium_stable.sql new file mode 100644 index 00000000..e04d784b --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_raydium_stable.sql @@ -0,0 +1,203 @@ +-- depends_on: {{ ref('silver__decoded_instructions_combined') }} +{{ config( + materialized = 'incremental', + unique_key = ['swaps_intermediate_raydium_stable_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% set base_query %} + CREATE + OR REPLACE temporary TABLE silver.swaps_intermediate_raydium_stable__intermediate_tmp AS + + SELECT + block_timestamp, + block_id, + tx_id, + signers, + succeeded, + INDEX, + inner_index, + program_id, + event_type, + decoded_instruction, + _inserted_timestamp + FROM + {{ ref('silver__decoded_instructions_combined') }} + WHERE + program_id = '5quBtoiQqxF9Jv6KYKctB59NT3gtJD2Y65kdnB1Uev3h' + AND event_type IN ( + 'swapBaseIn' + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} +) +{% else %} + AND _inserted_timestamp :: DATE >= '2024-05-20' +{% endif %} + +{% endset %} +{% do run_query(base_query) %} +{% set between_stmts = fsc_utils.dynamic_range_predicate( + "silver.swaps_intermediate_raydium_stable__intermediate_tmp", + "block_timestamp::date" +) %} +{% endif %} + +WITH base AS ( + SELECT + * + FROM + silver.swaps_intermediate_raydium_stable__intermediate_tmp +), +decoded AS ( + SELECT + block_timestamp, + block_id, + tx_id, + INDEX, + inner_index, + COALESCE(LEAD(inner_index) OVER (PARTITION BY tx_id, index + ORDER BY inner_index) -1, 999999 + ) AS inner_index_end, + program_id, + silver.udf_get_account_pubkey_by_name('userOwner', decoded_instruction:accounts) as swapper, + silver.udf_get_account_pubkey_by_name('userSourceToken', decoded_instruction:accounts) as source_token_account, + null as source_mint, + null as destination_mint, + silver.udf_get_account_pubkey_by_name('userDestinationToken', decoded_instruction:accounts) as destination_token_account, + silver.udf_get_account_pubkey_by_name('ammTargetOrders', decoded_instruction:accounts) as program_destination_token_account, + silver.udf_get_account_pubkey_by_name('poolTokenCoin', decoded_instruction:accounts) as program_source_token_account, + _inserted_timestamp + FROM + base +), +transfers AS ( + SELECT + A.*, + COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1, + NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 + FROM + {{ ref('silver__transfers') }} A + INNER JOIN ( + SELECT + DISTINCT tx_id, + block_timestamp :: DATE AS block_date + FROM + decoded + ) d + ON d.block_date = A.block_timestamp :: DATE + AND d.tx_id = A.tx_id + WHERE + A.succeeded + AND {{ between_stmts }} +), +pre_final AS ( + SELECT + A.block_id, + A.block_timestamp, + A.program_id, + A.tx_id, + A.index, + A.inner_index, + A.inner_index_end, + COALESCE ( + b.succeeded, + d.succeeded + ) AS succeeded, + A.swapper, + COALESCE ( + b.amount, + d.amount + ) AS from_amt, + COALESCE( + b.mint, + d.mint + ) AS from_mint, + COALESCE( + C.amount, + e.amount + ) AS to_amt, + COALESCE( + C.mint, + e.mint + ) AS to_mint, + A._inserted_timestamp + FROM + decoded A + -- join with transfers table to get details for source and destination tokens + LEFT JOIN transfers b + ON A.tx_id = b.tx_id + AND A.source_token_account = b.source_token_account + AND A.program_source_token_account = b.dest_token_account + AND A.index = b.index_1 + AND ( + (b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + LEFT JOIN transfers C + ON A.tx_id = C.tx_id + AND A.destination_token_account = C.dest_token_account + AND A.program_destination_token_account = C.source_token_account + AND A.index = C.index_1 + AND ( + (C.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL) + -- do a separate set of joins mirroring above because destination/source accounts are occasionaly flipped in a swap tx + LEFT JOIN transfers d + ON A.tx_id = d.tx_id + AND A.source_token_account = d.source_token_account + AND A.program_destination_token_account = d.dest_token_account + AND A.index = d.index_1 + AND ( + (d.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + LEFT JOIN transfers e + ON A.tx_id = e.tx_id + AND A.destination_token_account = e.dest_token_account + AND A.program_source_token_account = e.source_token_account + AND A.index = e.index_1 + AND ( + (e.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + QUALIFY + ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX + ORDER BY inner_index + ) = 1 +) +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + succeeded, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + INDEX, + inner_index + ) AS swap_index, + index, + inner_index, + swapper, + from_amt, + from_mint, + to_amt, + to_mint, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id','swap_index','program_id']) }} AS swaps_intermediate_raydium_stable_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + pre_final diff --git a/models/silver/swaps/silver__swaps_intermediate_raydium_stable.yml b/models/silver/swaps/silver__swaps_intermediate_raydium_stable.yml new file mode 100644 index 00000000..4c1c9bfc --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_raydium_stable.yml @@ -0,0 +1,104 @@ +version: 2 +models: + - name: silver__swaps_intermediate_raydium_stable + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - SWAP_INDEX + - PROGRAM_ID + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_intermediate_raydium_stable_business_logic_test + compare_model: ref('testing__swaps_intermediate_raydium_stable') + compare_columns: + - tx_id + - swapper + - round(from_amt,8) + - from_mint + - round(to_amt,8) + - to_mint + - swap_index + model_condition: "where tx_id in ('4hCaNMB42Gg1YedK91GzZKpJS25sGpgB2sri8vatLn6S4WUU3MKCANTXepsBPjPewzjhefeTUFKG36BTr8y7hweh', + '31JsUv3c9J5FMFJgxrBTeXghAVK3qnAKEjCBrZTXXjXvgxh6LUCc1yRuoNDXXDyywxwvcKxKGMaQh3GCsGLnAFAo' + )" + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - not_null: *recent_date_filter + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('index') }}" + tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_index') }}" + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null: *recent_date_filter + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: succeeded = TRUE + - name: FROM_AMT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null: *recent_date_filter + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null: *recent_date_filter + - name: TO_AMT + description: "{{ doc('swaps_to_amt') }}" + tests: + - not_null: *recent_date_filter + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + tests: + - not_null: *recent_date_filter + - name: SWAP_INDEX + description: "{{ doc('swaps_swap_index') }}" + tests: + - not_null: *recent_date_filter + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null: *recent_date_filter + - name: SWAPS_INTERMEDIATE_RAYDIUM_stable_ID + description: '{{ doc("pk") }}' + tests: + - unique: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + tests: + - not_null: + name: test_silver__not_null_swaps_intermediate_raydium_stable__invocation_id + <<: *recent_date_filter \ No newline at end of file diff --git a/models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.sql b/models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.sql new file mode 100644 index 00000000..7201a641 --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.sql @@ -0,0 +1,228 @@ +-- depends_on: {{ ref('silver__decoded_instructions_combined') }} +{{ config( + materialized = 'incremental', + unique_key = ['swaps_intermediate_raydium_v4_amm_id'], + incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"], + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['scheduled_non_core'] +) }} + +{% if execute %} + {% set base_query %} + CREATE + OR REPLACE temporary TABLE silver.swaps_intermediate_raydium_v4_amm__intermediate_tmp AS + + SELECT + block_timestamp, + block_id, + tx_id, + signers, + succeeded, + INDEX, + inner_index, + program_id, + event_type, + decoded_instruction, + _inserted_timestamp + FROM + {{ ref('silver__decoded_instructions_combined') }} + WHERE + program_id = '675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8' + AND event_type IN ( + 'swapBaseIn', + 'swapBaseOut' + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '1 hour' + FROM + {{ this }} +) +{% else %} + AND _inserted_timestamp :: DATE >= '2024-05-14' +{% endif %} + +{% endset %} +{% do run_query(base_query) %} +{% set between_stmts = fsc_utils.dynamic_range_predicate( + "silver.swaps_intermediate_raydium_v4_amm__intermediate_tmp", + "block_timestamp::date" +) %} +{% endif %} + +WITH base AS ( + SELECT + * + FROM + silver.swaps_intermediate_raydium_v4_amm__intermediate_tmp +), +decoded AS ( + SELECT + block_timestamp, + block_id, + tx_id, + INDEX, + inner_index, + COALESCE(LEAD(inner_index) over (PARTITION BY tx_id, INDEX + ORDER BY + inner_index) -1, 999999) AS inner_index_end, + program_id, + silver.udf_get_account_pubkey_by_name('userSourceOwner', decoded_instruction:accounts) as temp_user_source_owner, + CASE + WHEN temp_user_source_owner IS NULL THEN + signers[0] + ELSE temp_user_source_owner + END AS swapper, + CASE + WHEN temp_user_source_owner IS NOT NULL THEN + silver.udf_get_account_pubkey_by_name('uerSourceTokenAccount', decoded_instruction:accounts) + ELSE + silver.udf_get_account_pubkey_by_name('serumVaultSigner', decoded_instruction:accounts) + END AS source_token_account, + NULL AS source_mint, + NULL AS destination_mint, + CASE + WHEN temp_user_source_owner IS NOT NULL THEN + silver.udf_get_account_pubkey_by_name('uerDestinationTokenAccount', decoded_instruction:accounts) + ELSE silver.udf_get_account_pubkey_by_name('uerSourceTokenAccount', decoded_instruction:accounts) + END AS destination_token_account, + CASE + WHEN temp_user_source_owner IS NOT NULL AND event_type = 'swapBaseOut' THEN + silver.udf_get_account_pubkey_by_name('poolPcTokenAccount', decoded_instruction:accounts) + ELSE silver.udf_get_account_pubkey_by_name('poolCoinTokenAccount', decoded_instruction:accounts) + END AS program_destination_token_account, + CASE + WHEN temp_user_source_owner IS NOT NULL AND event_type = 'swapBaseOut' THEN + silver.udf_get_account_pubkey_by_name('poolCoinTokenAccount', decoded_instruction:accounts) + WHEN temp_user_source_owner IS NOT NULL AND event_type = 'swapBaseIn' THEN + silver.udf_get_account_pubkey_by_name('poolPcTokenAccount', decoded_instruction:accounts) + ELSE silver.udf_get_account_pubkey_by_name('ammTargetOrders', decoded_instruction:accounts) + END AS program_source_token_account, + _inserted_timestamp + FROM + base +), +transfers AS ( + SELECT + A.*, + COALESCE(SPLIT_PART(INDEX :: text, '.', 1) :: INT, INDEX :: INT) AS index_1, + NULLIF(SPLIT_PART(INDEX :: text, '.', 2), '') :: INT AS inner_index_1 + FROM + {{ ref('silver__transfers') }} A + INNER JOIN ( + SELECT + DISTINCT tx_id, + block_timestamp :: DATE AS block_date + FROM + decoded + ) d + ON d.block_date = A.block_timestamp :: DATE + AND d.tx_id = A.tx_id + WHERE + A.succeeded + AND {{ between_stmts }} +), +pre_final AS ( + SELECT + A.block_id, + A.block_timestamp, + A.program_id, + A.tx_id, + A.index, + A.inner_index, + A.inner_index_end, + COALESCE ( + b.succeeded, + d.succeeded + ) AS succeeded, + A.swapper, + COALESCE ( + b.amount, + d.amount + ) AS from_amt, + COALESCE( + b.mint, + d.mint + ) AS from_mint, + COALESCE( + C.amount, + e.amount + ) AS to_amt, + COALESCE( + C.mint, + e.mint + ) AS to_mint, + A._inserted_timestamp + FROM + decoded A + -- join with transfers table to get details for source and destination tokens + LEFT JOIN transfers b + ON A.tx_id = b.tx_id + AND A.source_token_account = b.source_token_account + AND A.program_source_token_account = b.dest_token_account + AND A.index = b.index_1 + AND ( + (b.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + LEFT JOIN transfers C + ON A.tx_id = C.tx_id + AND A.destination_token_account = C.dest_token_account + AND A.program_destination_token_account = C.source_token_account + AND A.index = C.index_1 + AND ( + (C.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL) + -- do a separate set of joins mirroring above because destination/source accounts are occasionaly flipped in a swap tx + LEFT JOIN transfers d + ON A.tx_id = d.tx_id + AND A.source_token_account = d.source_token_account + AND A.program_destination_token_account = d.dest_token_account + AND A.index = d.index_1 + AND ( + (d.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + LEFT JOIN transfers e + ON A.tx_id = e.tx_id + AND A.destination_token_account = e.dest_token_account + AND A.program_source_token_account = e.source_token_account + AND A.index = e.index_1 + AND ( + (e.inner_index_1 BETWEEN A.inner_index AND A.inner_index_end) + OR A.inner_index IS NULL + ) + QUALIFY + ROW_NUMBER() over (PARTITION BY A.tx_id, A.index, A.inner_INDEX + ORDER BY inner_index + ) = 1 +) +SELECT + block_id, + block_timestamp, + program_id, + tx_id, + succeeded, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + INDEX, + inner_index + ) AS swap_index, + index, + inner_index, + swapper, + from_amt, + from_mint, + to_amt, + to_mint, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id','swap_index','program_id']) }} AS swaps_intermediate_raydium_v4_amm_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + pre_final diff --git a/models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.yml b/models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.yml new file mode 100644 index 00000000..14bb5877 --- /dev/null +++ b/models/silver/swaps/silver__swaps_intermediate_raydium_v4_amm.yml @@ -0,0 +1,105 @@ +version: 2 +models: + - name: silver__swaps_intermediate_raydium_v4_amm + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - SWAP_INDEX + - PROGRAM_ID + where: block_timestamp::date > current_date - 30 + - compare_model_subset: + name: silver__swaps_intermediate_raydium_v4_amm_business_logic_test + compare_model: ref('testing__swaps_intermediate_raydium_v4_amm') + compare_columns: + - tx_id + - swapper + - round(from_amt,8) + - from_mint + - round(to_amt,8) + - to_mint + - swap_index + model_condition: "where tx_id in ('3PmcuNbasp87STwtHkVXK8wZpqkfTdBa9p3LYT3Sp3G2BUjvfLWoqniQn222dW8d1ZhUWi2VrURTAwNWhBvGRQRH', + '3x3HNjr34Vxzbsz29DAAm2Ngnrv7p9FXYGG5yw3MWiE6W4mt79Poe4axw2Lx4dXn3hMCDeDNVyiffE6kb3MhGGzp', + '3z249coLZuL2jViqt3LfNpDhi12dweanrv7VUJYVtGJ9t79GpN87R2HTF9x4dyF544jX6AxEMquQVKNxB9NehoDf' + )" + recent_date_filter: &recent_date_filter + config: + where: _inserted_timestamp >= current_date - 7 + columns: + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - not_null: *recent_date_filter + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null: *recent_date_filter + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null: *recent_date_filter + - name: SUCCEEDED + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null: *recent_date_filter + - name: INDEX + description: "{{ doc('index') }}" + tests: + - not_null: *recent_date_filter + - name: INNER_INDEX + description: "{{ doc('inner_index') }}" + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null: *recent_date_filter + - name: SWAPPER + description: "{{ doc('swaps_swapper') }}" + tests: + - not_null: + where: succeeded = TRUE + - name: FROM_AMT + description: "{{ doc('swaps_from_amt') }}" + tests: + - not_null: *recent_date_filter + - name: FROM_MINT + description: "{{ doc('swaps_from_mint') }}" + tests: + - not_null: *recent_date_filter + - name: TO_AMT + description: "{{ doc('swaps_to_amt') }}" + tests: + - not_null: *recent_date_filter + - name: TO_MINT + description: "{{ doc('swaps_to_mint') }}" + tests: + - not_null: *recent_date_filter + - name: SWAP_INDEX + description: "{{ doc('swaps_swap_index') }}" + tests: + - not_null: *recent_date_filter + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null: *recent_date_filter + - name: SWAPS_INTERMEDIATE_RAYDIUM_V4_AMM_ID + description: '{{ doc("pk") }}' + tests: + - unique: *recent_date_filter + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - not_null: *recent_date_filter + - name: _INVOCATION_ID + description: '{{ doc("_invocation_id") }}' + tests: + - not_null: + name: test_silver__not_null_swaps_intermediate_raydium_v4_amm__invocation_id + <<: *recent_date_filter \ No newline at end of file