An 5449/update jup v4v5 gold (#707)

* move jupv4/v5 to gold jupiter table

* add source

* add cols to view

* adjust test and limit cols

* null cols;suceeded filter;doc update

* convert to tables

* table desc update jup v4/v5 removal
This commit is contained in:
tarikceric 2024-11-29 12:47:56 -06:00 committed by GitHub
parent c9d89b3722
commit d2404d2348
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 310 additions and 67 deletions

View File

@ -41,65 +41,18 @@ SELECT
modified_timestamp
FROM
{{ ref('silver__swaps') }}
{% if is_incremental() %}
WHERE
program_id != 'JUP4Fb2cqiRUcaTHdrPC8h2gNsA2ETXiPDD33WcGuJB'
{% if is_incremental() %}
AND
modified_timestamp >= '{{ max_modified_timestamp }}'
{% else %}
WHERE
AND
modified_timestamp::date < '{{ backfill_to_date }}'
{% endif %}
)
,
swaps_individual as (
SELECT
block_timestamp,
block_id,
tx_id,
succeeded,
swapper,
from_amt AS swap_from_amount,
from_mint AS swap_from_mint,
to_amt AS swap_to_amount,
to_mint AS swap_to_mint,
program_id,
swap_index,
swaps_intermediate_jupiterv5_id as fact_swaps_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv5_1_view') }}
{% if is_incremental() %}
WHERE
modified_timestamp >= '{{ max_modified_timestamp }}'
{% else %}
WHERE
modified_timestamp::date < '{{ backfill_to_date }}'
{% endif %}
UNION ALL
SELECT
block_timestamp,
block_id,
tx_id,
succeeded,
swapper,
from_amt AS swap_from_amount,
from_mint AS swap_from_mint,
to_amt AS swap_to_amount,
to_mint AS swap_to_mint,
program_id,
swap_index,
swaps_intermediate_jupiterv5_id as fact_swaps_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv5_2_view') }}
{% if is_incremental() %}
WHERE modified_timestamp >= '{{ max_modified_timestamp }}'
{% else %}
WHERE
modified_timestamp::date < '{{ backfill_to_date }}'
{% endif %}
UNION ALL
SELECT
block_timestamp,
block_id,

View File

@ -1,7 +1,7 @@
version: 2
models:
- name: defi__fact_swaps
description: This table contains swaps performed on Jupiter (V4 and V5), Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. For Phoenix, we are not capturing swaps where there are separate transactions for placing the order and filling the order.
description: This table contains swaps performed on Jupiter (V4 and V5), Orca, Raydium, Saber, Bonkswap, Dooar, Phoenix and Meteora swap programs. Intermediate swaps are aggregated over the DEX programs, so the values showcase the final mint/amount swap values. Ie. a swap on Jupiter that swaps SOL->USDC->mSOL->ETH would show the initial amount in and the final amount out of SOL->ETH. For Phoenix, we are not capturing swaps where there are separate transactions for placing the order and filling the order. NOTE - Jupiter V4 and V5 swaps will be removed from this table on December 9th, and the corresponding swaps will exist in defi.fact_swaps_jupiter_summary.
recent_date_filter: &recent_date_filter
config:
where: block_timestamp >= current_date - 7
@ -11,7 +11,6 @@ models:
tests:
- reference_tx_missing:
reference_tables:
- 'silver__swaps'
- 'silver__swaps_intermediate_bonkswap'
- 'silver__swaps_intermediate_meteora'
- 'silver__swaps_intermediate_dooar'

View File

@ -1,9 +1,73 @@
{{ config(
materialized = 'view',
materialized = 'incremental',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }}},
unique_key = ['fact_swaps_jupiter_inner_id'],
incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
cluster_by = ['block_timestamp::DATE','swap_program_id'],
merge_exclude_columns = ["inserted_timestamp"],
post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, swapper, swap_from_mint, swap_to_mint, swap_program_id, aggregator_program_id, fact_swaps_jupiter_inner_id)'),
tags = ['scheduled_non_core']
) }}
{% if execute %}
{% if is_incremental() %}
{% set query %}
SELECT MAX(modified_timestamp) AS max_modified_timestamp
FROM {{ this }}
{% endset %}
{% set max_modified_timestamp = run_query(query).columns[0].values()[0] %}
{% endif %}
{% endif %}
-- Select from the non-active models only during the initial FR
{% if not is_incremental() %}
SELECT
block_timestamp,
block_id,
tx_id,
index,
inner_index,
swap_index,
succeeded,
swapper,
from_mint AS swap_from_mint,
from_amt AS swap_from_amount,
to_mint AS swap_to_mint,
to_amt AS swap_to_amount,
inner_swap_program_id AS swap_program_id,
program_id AS aggregator_program_id,
{{ dbt_utils.generate_surrogate_key(['tx_id','index','inner_index']) }} AS fact_swaps_jupiter_inner_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv4_view') }}
WHERE
block_timestamp <= '2023-10-31'
and succeeded
UNION ALL
SELECT
block_timestamp,
block_id,
tx_id,
index,
inner_index,
swap_index,
succeeded,
swapper,
from_mint AS swap_from_mint,
from_amount AS swap_from_amount,
to_mint AS swap_to_mint,
to_amount AS swap_to_amount,
swap_program_id,
aggregator_program_id,
swaps_inner_intermediate_jupiterv5_id AS fact_swaps_jupiter_inner_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_inner_intermediate_jupiterv5') }}
UNION ALL
{% endif %}
-- Only select from active models during incremental
SELECT
block_timestamp,
block_id,
@ -21,6 +85,36 @@ SELECT
aggregator_program_id,
swaps_inner_intermediate_jupiterv6_id AS fact_swaps_jupiter_inner_id,
inserted_timestamp,
modified_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_inner_intermediate_jupiterv6') }}
{{ ref('silver__swaps_inner_intermediate_jupiterv6') }}
{% if is_incremental() %}
WHERE modified_timestamp >= '{{ max_modified_timestamp }}'
{% endif %}
UNION ALL
SELECT
block_timestamp,
block_id,
tx_id,
index,
inner_index,
swap_index,
succeeded,
swapper,
from_mint AS swap_from_mint,
from_amount AS swap_from_amount,
to_mint AS swap_to_mint,
to_amount AS swap_to_amount,
swap_program_id,
aggregator_program_id,
swaps_inner_intermediate_jupiterv4_id AS fact_swaps_jupiter_inner_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_inner_intermediate_jupiterv4') }}
WHERE
block_timestamp > '2023-10-31'
{% if is_incremental() %}
AND modified_timestamp >= '{{ max_modified_timestamp }}'
{% endif %}

View File

@ -2,69 +2,93 @@ version: 2
models:
- name: defi__fact_swaps_jupiter_inner
description: This table contain each intermediate swap that is a part of a Jupiter route. These are the individual steps that are executed by Jupiter to complete a swap.
recent_date_filter: &recent_date_filter
config:
where: modified_timestamp >= current_date - 7
tests:
- reference_tx_missing:
reference_tables:
- 'silver__swaps_inner_intermediate_jupiterv6'
- 'silver__swaps_inner_intermediate_jupiterv4'
id_column: 'tx_id'
columns:
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: INDEX
description: "{{ doc('event_index') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: INNER_INDEX
description: "{{ doc('inner_index') }}. This is the inner index of the log event listing the inner swap"
tests:
- dbt_expectations.expect_column_to_exist
- name: SWAP_INDEX
description: "{{ doc('swaps_swap_index') }} as it relates to the top level Jupiter V6 swap instruction"
description: "{{ doc('swaps_swap_index') }} as it relates to the top level Jupiter swap instruction"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SUCCEEDED
description: "{{ doc('tx_succeeded') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_PROGRAM_ID
description: "{{ doc('program_id') }}. This is the AMM performing the swap."
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: AGGREGATOR_PROGRAM_ID
description: "{{ doc('program_id') }}. This is the aggregator calling the different AMMs."
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAPPER
description: "{{ doc('swaps_swapper') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_FROM_AMOUNT
description: "{{ doc('swaps_from_amt') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_FROM_MINT
description: "{{ doc('swaps_from_mint') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_TO_AMOUNT
description: "{{ doc('swaps_to_amt') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_TO_MINT
description: "{{ doc('swaps_to_mint') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: FACT_SWAPS_JUPITER_INNER_ID
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: INSERTED_TIMESTAMP
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'
tests:

View File

@ -1,9 +1,30 @@
{{ config(
materialized = 'view',
materialized = 'incremental',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }}},
unique_key = ['fact_swaps_jupiter_summary_id'],
incremental_predicates = ["dynamic_range_predicate", "block_timestamp::date"],
cluster_by = ['block_timestamp::DATE','program_id'],
merge_exclude_columns = ["inserted_timestamp"],
post_hook = enable_search_optimization('{{this.schema}}','{{this.identifier}}','ON EQUALITY(tx_id, swapper, swap_from_mint, swap_to_mint, program_id, fact_swaps_jupiter_summary_id)'),
tags = ['scheduled_non_core']
) }}
{% if execute %}
{% if is_incremental() %}
{% set query %}
SELECT MAX(modified_timestamp) AS max_modified_timestamp
FROM {{ this }}
{% endset %}
{% set max_modified_timestamp = run_query(query).columns[0].values()[0] %}
{% else %}
{% set backfill_to_date = '2024-06-09' %}
{% endif %}
{% endif %}
-- Select from the non-active models only during the initial FR
{% if not is_incremental() %}
SELECT
block_timestamp,
block_id,
@ -24,12 +45,91 @@ SELECT
NULL as limit_requester,
swaps_intermediate_jupiterv6_id AS fact_swaps_jupiter_summary_id,
inserted_timestamp,
modified_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv6_view') }}
WHERE
block_timestamp::date < '2023-08-03'
UNION ALL
SELECT
block_timestamp,
block_id,
tx_id,
swap_index as index,
NULL as inner_index,
row_number() OVER (PARTITION BY tx_id ORDER BY index)-1 AS swap_index,
succeeded,
swapper,
from_mint AS swap_from_mint,
from_amt AS swap_from_amount,
to_mint AS swap_to_mint,
to_amt AS swap_to_amount,
program_id,
NULL as is_dca_swap,
NULL as dca_requester,
NULL as is_limit_swap,
NULL as limit_requester,
swaps_intermediate_jupiterv5_id as fact_swaps_jupiter_summary_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv5_1_view') }}
UNION ALL
SELECT
block_timestamp,
block_id,
tx_id,
swap_index as index,
NULL as inner_index,
row_number() OVER (PARTITION BY tx_id ORDER BY index)-1 AS swap_index,
succeeded,
swapper,
from_mint AS swap_from_mint,
from_amt AS swap_from_amount,
to_mint AS swap_to_mint,
to_amt AS swap_to_amount,
program_id,
NULL as is_dca_swap,
NULL as dca_requester,
NULL as is_limit_swap,
NULL as limit_requester,
swaps_intermediate_jupiterv5_id as fact_swaps_jupiter_summary_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv5_2_view') }}
UNION ALL
SELECT
block_timestamp,
block_id,
tx_id,
NULL as index,
NULL as inner_index,
0 as swap_index,
succeeded,
swapper,
from_mint AS swap_from_mint,
from_amt AS swap_from_amount,
to_mint AS swap_to_mint,
to_amt AS swap_to_amount,
program_id,
NULL as is_dca_swap,
NULL as dca_requester,
NULL as is_limit_swap,
NULL as limit_requester,
swaps_id as fact_swaps_jupiter_summary_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps') }}
WHERE
program_id = 'JUP4Fb2cqiRUcaTHdrPC8h2gNsA2ETXiPDD33WcGuJB'
and block_timestamp::date <= '2023-10-31'
and succeeded
UNION ALL
{% endif %}
-- Only select from active models during incremental
SELECT
block_timestamp,
block_id,
@ -50,8 +150,42 @@ SELECT
limit_requester,
swaps_intermediate_jupiterv6_id AS fact_swaps_jupiter_summary_id,
inserted_timestamp,
modified_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv6_2') }}
WHERE
block_timestamp::date >= '2023-08-03'
block_timestamp::date >= '2023-08-03'
{% if is_incremental() %}
AND modified_timestamp >= '{{ max_modified_timestamp }}'
{% endif %}
UNION ALL
SELECT
block_timestamp,
block_id,
tx_id,
index,
inner_index,
swap_index,
succeeded,
swapper,
from_mint AS swap_from_mint,
from_amount AS swap_from_amount,
to_mint AS swap_to_mint,
to_amount AS swap_to_amount,
program_id,
NULL as is_dca_swap,
NULL as dca_requester,
NULL as is_limit_swap,
NULL as limit_requester,
swaps_intermediate_jupiterv4_id as fact_swaps_jupiter_summary_id,
inserted_timestamp,
modified_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv4_2') }}
WHERE
block_timestamp::date > '2023-10-31'
and not truncated_log
{% if is_incremental() %}
AND modified_timestamp >= '{{ max_modified_timestamp }}'
{% endif %}

View File

@ -1,60 +1,82 @@
version: 2
models:
- name: defi__fact_swaps_jupiter_summary
description: This table contains summarized swaps for trades initiated on Jupiter. This showcases the the initial amount/mint that is swapped from and the final amount/mint that is swapped to, without the intermediate swaps that Jupiter routes through. This also includes columns for DCA (dollar cost averaging) information to identify the initial requester of the DCA swaps.
description: This table contains summarized swaps for trades initiated on Jupiter. This showcases the the initial amount/mint that is swapped from and the final amount/mint that is swapped to, without the intermediate swaps that Jupiter routes through. This also includes columns for DCA (dollar cost averaging) information to identify the initial requester of the DCA swaps. Jupiter V4 swaps before 2023-10-31 are aggregated at the transaction level, so these contain NULL index and inner_index values, and default to 0 for swap_index.
recent_date_filter: &recent_date_filter
config:
where: modified_timestamp >= current_date - 7
tests:
- reference_tx_missing:
reference_tables:
- 'silver__swaps_intermediate_jupiterv6_2'
- 'silver__swaps_intermediate_jupiterv4_2'
id_column: 'tx_id'
columns:
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: INDEX
description: "{{ doc('event_index') }}"
description: "{{ doc('event_index') }}. Jupiter V4 swaps prior to 2023-10-31 are NULL"
tests:
- dbt_expectations.expect_column_to_exist
- not_null:
where: modified_timestamp >= current_date - 7 AND (program_id <> 'JUP4Fb2cqiRUcaTHdrPC8h2gNsA2ETXiPDD33WcGuJB' AND block_timestamp::date >= '2023-10-31')
- name: INNER_INDEX
description: "{{ doc('inner_index') }}. This is the inner index of the log event listing the inner swap"
description: "{{ doc('inner_index') }}. This is the inner index of the log event listing the inner swap. This value is null for Jupiter v5 swaps (which do not have swaps within the inner instruction), as well as Jupiter v4 swaps prior to 2023-10-31'"
tests:
- dbt_expectations.expect_column_to_exist
- name: SWAP_INDEX
description: "{{ doc('swaps_swap_index') }} as it relates to the top level Jupiter V6 swap instruction"
description: "{{ doc('swaps_swap_index') }} as it relates to the top level Jupiter swap instruction. This value is 0 for Jupiter v4 swaps prior to 2023-10-31'"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SUCCEEDED
description: "{{ doc('tx_succeeded') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: PROGRAM_ID
description: "{{ doc('program_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAPPER
description: "{{ doc('swaps_swapper') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_FROM_AMOUNT
description: "{{ doc('swaps_from_amt') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_FROM_MINT
description: "{{ doc('swaps_from_mint') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_TO_AMOUNT
description: "{{ doc('swaps_to_amt') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: SWAP_TO_MINT
description: "{{ doc('swaps_to_mint') }}"
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: IS_DCA_SWAP
description: "Whether the swap was initiated by a Jupiter DCA. If value is NULL then it is NOT a DCA Swap"
tests:
@ -74,9 +96,12 @@ models:
- name: FACT_SWAPS_JUPITER_SUMMARY_ID
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- unique: *recent_date_filter
- name: INSERTED_TIMESTAMP
tests:
- dbt_expectations.expect_column_to_exist
- not_null: *recent_date_filter
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'
tests:

View File

@ -8,7 +8,7 @@
'{{this.identifier}}',
'ON EQUALITY(tx_id, swapper, from_mint, to_mint)'
),
tags = ['scheduled_non_core']
enabled=false
) }}
WITH base_events AS(

View File

@ -0,0 +1,13 @@
{{ config(
materialized = 'view'
) }}
SELECT
*,
'2000-01-01' as inserted_timestamp,
'2000-01-01' AS modified_timestamp
FROM
{{ source(
'solana_silver',
'swaps_intermediate_jupiterv4'
) }}

View File

@ -85,7 +85,7 @@ SELECT
_log_id,
_inserted_timestamp
FROM
{{ ref('silver__swaps_intermediate_jupiterv4') }}
{{ ref('silver__swaps_intermediate_jupiterv4_view') }}
{% if is_incremental() %}
WHERE

View File

@ -102,6 +102,7 @@ sources:
- name: nft_sales_amm_sell
- name: nft_sales_hyperspace
- name: stake_pool_actions_lido
- name: swaps_intermediate_jupiterv4
- name: nft_collection
- name: solana_streamline
database: solana