From a4fb1c61e552d243574e838a28ade67678eb6aa3 Mon Sep 17 00:00:00 2001 From: Matt Romano <42412983+mattromano@users.noreply.github.com> Date: Fri, 9 May 2025 12:49:23 -0700 Subject: [PATCH] revert-vertex-back-to-local-repo (#333) * add local vertex models and update to gold table refs * add event id to tables * fix gold staking yml * exclude non-updated tests --- .github/workflows/dbt_test_daily.yml | 2 +- dbt_project.yml | 2 +- .../protocols/vertex/vertex__dim_products.sql | 31 ++ .../vertex/vertex__ez_account_stats.sql | 57 +++ .../vertex__ez_clearing_house_events.sql | 37 ++ .../vertex/vertex__ez_edge_trades.sql | 58 +++ .../vertex/vertex__ez_liquidations.sql | 45 +++ .../vertex/vertex__ez_market_depth_stats.sql | 31 ++ .../vertex/vertex__ez_market_stats.sql | 45 +++ .../vertex/vertex__ez_money_markets.sql | 28 ++ .../vertex/vertex__ez_perp_trades.sql | 51 +++ .../vertex/vertex__ez_spot_trades.sql | 51 +++ .../vertex/vertex__ez_staking_actions.sql | 34 ++ .../vertex/vertex__ez_staking_actions.yml | 36 ++ .../vertex/silver__vertex_account_stats.sql | 234 +++++++++++ .../vertex/silver__vertex_collateral.sql | 149 +++++++ .../vertex/silver__vertex_dim_products.sql | 162 ++++++++ .../vertex/silver__vertex_edge_trades.sql | 166 ++++++++ .../vertex/silver__vertex_liquidations.sql | 288 +++++++++++++ .../vertex/silver__vertex_market_depth.sql | 105 +++++ .../vertex/silver__vertex_market_stats.sql | 175 ++++++++ .../vertex/silver__vertex_money_markets.sql | 55 +++ .../protocols/vertex/silver__vertex_perps.sql | 379 ++++++++++++++++++ .../protocols/vertex/silver__vertex_spot.sql | 379 ++++++++++++++++++ .../vertex/silver__vertex_staking.sql | 101 +++++ .../vertex/silver__vertex_staking.yml | 84 ++++ 26 files changed, 2783 insertions(+), 2 deletions(-) create mode 100644 models/gold/protocols/vertex/vertex__dim_products.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_account_stats.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_clearing_house_events.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_edge_trades.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_liquidations.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_market_depth_stats.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_market_stats.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_money_markets.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_perp_trades.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_spot_trades.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_staking_actions.sql create mode 100644 models/gold/protocols/vertex/vertex__ez_staking_actions.yml create mode 100644 models/silver/protocols/vertex/silver__vertex_account_stats.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_collateral.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_dim_products.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_edge_trades.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_liquidations.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_market_depth.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_market_stats.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_money_markets.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_perps.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_spot.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_staking.sql create mode 100644 models/silver/protocols/vertex/silver__vertex_staking.yml diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index eaa9635..540dfe8 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -19,5 +19,5 @@ jobs: dbt run -m "fsc_evm,tag:daily_test" command_name_2: Run Daily Tests (all tests excluding full, recent and misc. others) command_2: | - dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models + dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models --exclude models/silver/protocols/vertex/silver__vertex_liquidations.sql models/silver/protocols/vertex/silver__vertex_spot.sql models/silver/protocols/vertex/silver__vertex_market_stats.sql secrets: inherit \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 56a9cde..f98f164 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -103,7 +103,7 @@ models: protocols: +enabled: false vertex: - +enabled: true + +enabled: false stats: +enabled: true scores_package: diff --git a/models/gold/protocols/vertex/vertex__dim_products.sql b/models/gold/protocols/vertex/vertex__dim_products.sql new file mode 100644 index 0000000..5e26105 --- /dev/null +++ b/models/gold/protocols/vertex/vertex__dim_products.sql @@ -0,0 +1,31 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, PRODUCTS' + } + } + } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + product_id, + book_address, + product_type, + ticker_id, + symbol, + name, + version, + vertex_products_id AS dim_products_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_dim_products') }} +ORDER BY product_id \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_account_stats.sql b/models/gold/protocols/vertex/vertex__ez_account_stats.sql new file mode 100644 index 0000000..d1cefd3 --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_account_stats.sql @@ -0,0 +1,57 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta ={ + 'database_tags':{ + 'table':{ + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, STATS' + } + } + } +) }} + +SELECT + subaccount, + trader, + first_trade_timestamp, + last_trade_timestamp, + account_age, + trade_count, + DENSE_RANK() over ( + ORDER BY + trade_count DESC + ) AS trade_count_rank, + trade_count_24h, + DENSE_RANK() over ( + ORDER BY + trade_count_24h DESC + ) AS trade_count_rank_24h, + perp_trade_count, + spot_trade_count, + long_count, + short_count, + total_usd_volume, + DENSE_RANK() over ( + ORDER BY + total_usd_volume DESC + ) AS total_usd_volume_rank, + total_usd_volume_24h, + DENSE_RANK() over ( + ORDER BY + total_usd_volume_24h DESC + ) AS total_usd_volume_rank_24h, + avg_usd_trade_size, + total_fee_amount, + total_base_delta_amount, + total_quote_delta_amount, + total_liquidation_amount, + total_liquidation_amount_quote, + total_liquidation_count, + vertex_account_id as ez_account_stats_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_account_stats') }} +ORDER BY total_usd_volume_rank DESC diff --git a/models/gold/protocols/vertex/vertex__ez_clearing_house_events.sql b/models/gold/protocols/vertex/vertex__ez_clearing_house_events.sql new file mode 100644 index 0000000..a384190 --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_clearing_house_events.sql @@ -0,0 +1,37 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, CLEARINGHOUSE' + } + } + } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + modification_type, + symbol, + trader, + subaccount, + token_address, + amount_unadj, + amount, + amount_usd, + vertex_collateral_id AS ez_clearing_house_events_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_collateral') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_edge_trades.sql b/models/gold/protocols/vertex/vertex__ez_edge_trades.sql new file mode 100644 index 0000000..5bece0e --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_edge_trades.sql @@ -0,0 +1,58 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX' + } + } + } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + edge_event_index, + user_event_index, + edge_digest, + user_digest, + trader, + subaccount, + symbol, + edge_order_type, + user_order_type, + edge_trade_type, + user_trade_type, + edge_is_taker, + user_is_taker, + edge_price_amount_unadj, + user_price_amount_unadj, + edge_price_amount, + user_price_amount, + edge_amount_unadj, + user_amount_unadj, + edge_amount, + user_amount, + edge_amount_usd, + user_amount_usd, + edge_fee_amount_unadj, + user_fee_amount_unadj, + edge_fee_amount, + user_fee_amount, + edge_base_delta_amount_unadj, + user_base_delta_amount_unadj, + edge_base_delta_amount, + user_base_delta_amount, + edge_quote_delta_amount_unadj, + user_quote_delta_amount_unadj, + edge_quote_delta_amount, + user_quote_delta_amount, + vertex_edge_trade_id as ez_edge_trades_id, + inserted_timestamp, + modified_timestamp, +FROM + {{ ref('silver__vertex_edge_trades') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_liquidations.sql b/models/gold/protocols/vertex/vertex__ez_liquidations.sql new file mode 100644 index 0000000..ab0bd6f --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_liquidations.sql @@ -0,0 +1,45 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, LIQUIDATION' + } + } + } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + digest, + trader, + subaccount, + version, + MODE, + product_id, + health_group, + health_group_symbol, + amount_unadj, + amount, + amount_quote_unadj, + amount_quote, + insurance_cover_unadj, + insurance_cover, + is_encoded_spread, + spread_product_ids, + vertex_liquidation_id AS ez_liquidations_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_liquidations') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_market_depth_stats.sql b/models/gold/protocols/vertex/vertex__ez_market_depth_stats.sql new file mode 100644 index 0000000..3a9bbe2 --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_market_depth_stats.sql @@ -0,0 +1,31 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, STATS' + } + } + } +) }} + +SELECT + hour, + ticker_id, + product_id, + orderbook_side, + volume, + price, + round_price_0_01, + round_price_0_1, + round_price_1, + round_price_10, + round_price_100, + vertex_market_depth_id as ez_market_depth_stats_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_market_depth') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_market_stats.sql b/models/gold/protocols/vertex/vertex__ez_market_stats.sql new file mode 100644 index 0000000..56ff2cf --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_market_stats.sql @@ -0,0 +1,45 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, STATS' + } + } + } +) }} + +SELECT + hour, + ticker_id, + product_id, + symbol, + distinct_sequencer_batches, + distinct_trader_count, + distinct_subaccount_count, + trade_count, + amount_usd, + fee_amount, + base_delta_amount, + quote_delta_amount, + base_volume_24h, + quote_volume_24h, + funding_rate, + index_price, + last_price, + mark_price, + next_funding_rate_timestamp, + open_interest, + open_interest_usd, + price_change_percent_24h, + product_type, + quote_currency, + quote_volume, + vertex_market_stats_id as ez_market_stats_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_market_stats') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_money_markets.sql b/models/gold/protocols/vertex/vertex__ez_money_markets.sql new file mode 100644 index 0000000..8eeac4b --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_money_markets.sql @@ -0,0 +1,28 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, STATS' + } + } + } +) }} + + +SELECT + hour, + ticker_id, + symbol, + product_id, + deposit_apr, + borrow_apr, + tvl, + vertex_money_markets_id as ez_money_markets_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_money_markets') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_perp_trades.sql b/models/gold/protocols/vertex/vertex__ez_perp_trades.sql new file mode 100644 index 0000000..0fa5b37 --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_perp_trades.sql @@ -0,0 +1,51 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, PERPS' + } + } + } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + digest, + trader, + subaccount, + version, + trade_type, + order_type, + market_reduce_flag, + expiration, + nonce, + is_taker, + price_amount_unadj, + price_amount, + amount_unadj, + amount, + amount_usd, + fee_amount_unadj, + fee_amount, + base_delta_amount_unadj, + base_delta_amount, + quote_delta_amount_unadj, + quote_delta_amount, + vertex_perps_id AS ez_perp_trades_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_perps') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_spot_trades.sql b/models/gold/protocols/vertex/vertex__ez_spot_trades.sql new file mode 100644 index 0000000..234dfb2 --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_spot_trades.sql @@ -0,0 +1,51 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, SPOT' + } + } + } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + digest, + trader, + subaccount, + version, + trade_type, + order_type, + market_reduce_flag, + expiration, + nonce, + is_taker, + price_amount_unadj, + price_amount, + amount_unadj, + amount, + amount_usd, + fee_amount_unadj, + fee_amount, + base_delta_amount_unadj, + base_delta_amount, + quote_delta_amount_unadj, + quote_delta_amount, + vertex_spot_id AS ez_spot_trades_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_spot') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_staking_actions.sql b/models/gold/protocols/vertex/vertex__ez_staking_actions.sql new file mode 100644 index 0000000..40ac80c --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_staking_actions.sql @@ -0,0 +1,34 @@ + {{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'VERTEX', + 'PURPOSE': 'CLOB, DEX, STAKING' + } + } + } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + to_address, + from_address, + stake_action, + symbol, + amount_unadj, + amount, + amount_usd, + vertex_staking_id as ez_staking_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__vertex_staking') }} \ No newline at end of file diff --git a/models/gold/protocols/vertex/vertex__ez_staking_actions.yml b/models/gold/protocols/vertex/vertex__ez_staking_actions.yml new file mode 100644 index 0000000..094cd7c --- /dev/null +++ b/models/gold/protocols/vertex/vertex__ez_staking_actions.yml @@ -0,0 +1,36 @@ +version: 2 +models: + - name: vertex__ez_staking_actions + description: "All staking actions taken with the VRTX staking contract." + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("evm_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("evm_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("evm_logs_tx_hash") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("evm_logs_contract_address") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("evm_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("evm_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("evm_origin_to") }}' + - name: STAKE_ACTION + description: "The staking action with the VRTX staking address" + - name: SYMBOL + description: "The specific Vertex product symbol, if it is a futures product it will have a -PERP suffix." + - name: AMOUNT_UNADJ + description: "The total size of the trade in units of the asset being traded." + - name: AMOUNT + description: "The total size of the trade in units of the asset being traded across one digest, decimal adjusted. All amounts and prices are adjusted 18 decimals points regardless of underlying asset contract." + - name: AMOUNT_USD + description: "The size of the trade in USD. Base Delta multiplied by the price amount." + - name: EZ_STAKING_ID + description: '{{ doc("evm_pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("evm_inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/protocols/vertex/silver__vertex_account_stats.sql b/models/silver/protocols/vertex/silver__vertex_account_stats.sql new file mode 100644 index 0000000..b222109 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_account_stats.sql @@ -0,0 +1,234 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'subaccount', + tags = 'curated' +) }} + +WITH + +{% if is_incremental() %} +new_subaccount_actions AS ( + + SELECT + DISTINCT(subaccount) + FROM + {{ ref('silver__vertex_perps') }} + WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '12 hours' + FROM + {{ this }} + ) + UNION + SELECT + DISTINCT(subaccount) + FROM + {{ ref('silver__vertex_spot') }} + WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '12 hours' + FROM + {{ this }} + ) + UNION + SELECT + DISTINCT(subaccount) + FROM + {{ ref('silver__vertex_liquidations') }} + WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '12 hours' + FROM + {{ this }} + ) + UNION + SELECT + DISTINCT(subaccount) + FROM + {{this}} + WHERE + total_usd_volume_24h > 0 +), +{% endif %} + +trades_union AS ( + SELECT + subaccount, + trader, + digest, + 'perp' AS product_type, + trade_type, + block_timestamp, + amount, + amount_usd, + fee_amount, + base_delta_amount, + quote_delta_amount, + _inserted_timestamp + FROM + {{ ref('silver__vertex_perps') }} + +{% if is_incremental() %} +WHERE + subaccount IN ( + SELECT + subaccount + FROM + new_subaccount_actions + ) +{% endif %} +UNION ALL +SELECT + subaccount, + trader, + digest, + 'spot' AS product_type, + trade_type, + block_timestamp, + amount, + amount_usd, + fee_amount, + base_delta_amount, + quote_delta_amount, + _inserted_timestamp +FROM + {{ ref('silver__vertex_spot') }} + +{% if is_incremental() %} +WHERE + subaccount IN ( + SELECT + subaccount + FROM + new_subaccount_actions + ) +{% endif %} +), +liquidations AS ( + SELECT + trader, + subaccount, + SUM(amount) AS total_liquidation_amount, + SUM(amount_quote) AS total_liquidation_amount_quote, + COUNT(*) AS liquidation_count + FROM + {{ ref('silver__vertex_liquidations') }} + +{% if is_incremental() %} +WHERE + subaccount IN ( + SELECT + subaccount + FROM + new_subaccount_actions + ) +{% endif %} +GROUP BY + 1, + 2 +), +FINAL AS ( + SELECT + t.subaccount, + t.trader, + MIN( + t.block_timestamp + ) AS first_trade_timestamp, + MAX( + t.block_timestamp + ) AS last_trade_timestamp, + DATEDIFF( + 'day', + first_trade_timestamp, + last_trade_timestamp + ) AS account_age, + COUNT(DISTINCT(digest)) AS trade_count, + COUNT(DISTINCT(CASE + WHEN t.block_timestamp >= SYSDATE() - INTERVAL '24 HOURS' + THEN digest + END)) AS trade_count_24h, + SUM( + CASE + WHEN product_type = 'perp' THEN + 1 + ELSE 0 + END + ) AS perp_trade_count, + SUM( + CASE + WHEN product_type = 'spot' THEN + 1 + ELSE 0 + END + ) AS spot_trade_count, + SUM( + CASE + WHEN trade_type = 'buy/long' THEN + 1 + ELSE 0 + END + ) AS long_count, + SUM( + CASE + WHEN trade_type = 'sell/short' THEN + 1 + ELSE 0 + END + ) AS short_count, + SUM(amount_usd) AS total_usd_volume, + SUM( + CASE + WHEN t.block_timestamp >= SYSDATE() - INTERVAL '24 HOURS' THEN amount_usd + ELSE 0 + END + ) AS total_usd_volume_24h, + AVG(amount_usd) AS avg_usd_trade_size, + SUM(fee_amount) AS total_fee_amount, + SUM(base_delta_amount) AS total_base_delta_amount, + SUM(quote_delta_amount) AS total_quote_delta_amount, + MAX( + l.total_liquidation_amount + ) AS total_liquidation_amount, + MAX( + l.total_liquidation_amount_quote + ) AS total_liquidation_amount_quote, + MAX(liquidation_count) AS total_liquidation_count, + MAX( + t._inserted_timestamp + ) AS _inserted_timestamp + FROM + trades_union t + LEFT JOIN liquidations l + ON t.subaccount = l.subaccount + GROUP BY + 1, + 2 +) +SELECT + f.*, + {{ dbt_utils.generate_surrogate_key( + ['f.subaccount'] + ) }} AS vertex_account_id, + COALESCE( + {% if is_incremental() %} + a.inserted_timestamp, + {% endif %} + SYSDATE(), + NULL + ) AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL f +{% if is_incremental() %} +LEFT JOIN + {{this}} a +ON + a.subaccount = f.subaccount +{% endif %} \ No newline at end of file diff --git a/models/silver/protocols/vertex/silver__vertex_collateral.sql b/models/silver/protocols/vertex/silver__vertex_collateral.sql new file mode 100644 index 0000000..78fc7a1 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_collateral.sql @@ -0,0 +1,149 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = '_log_id', + cluster_by = ['block_timestamp::DATE'], + tags = ['curated','reorg'] +) }} + +WITH logs_pull AS ( + + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + 'ModifyCollateral' AS event_name, + event_index, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + LEFT( + topics [1] :: STRING, + 42 + ) AS trader, + topics [1] :: STRING AS subaccount, + utils.udf_hex_to_int( + 's2c', + segmented_data [0] :: STRING + ) :: INT AS amount, + utils.udf_hex_to_int( + segmented_data [1] :: STRING + ) :: INT AS product_id, + fact_event_logs_id, + concat(tx_hash, '-', event_index) AS _log_id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__fact_event_logs') }} + WHERE + topics [0] :: STRING = '0xfe53084a731040f869d38b1dcd00fbbdbc14e10d7d739160559d77f5bc80cf05' + AND contract_address = '0xae1ec28d6225dce2ff787dcb8ce11cf6d3ae064f' --clearing house + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '36 hours' + FROM + {{ this }} +) +{% endif %} +), +product_id_join AS ( + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + CASE + WHEN amount < 0 THEN 'withdraw' + WHEN amount > 0 THEN 'deposit' + WHEN amount = 0 THEN 'no-change' + END AS modification_type, + trader, + subaccount, + l.product_id, + p.symbol, + CASE + WHEN p.symbol = 'USDC' THEN '0xff970a61a04b1ca14834a43f5de4533ebddb5cc8' + WHEN p.symbol = 'WETH' THEN '0x82af49447d8a07e3bd95bd0d56f35241523fbab1' + when p.symbol = 'wstETH' then '0x5979d7b546e38e414f7e9822514be443a4800529' + WHEN p.symbol = 'WBTC' THEN '0x2f2a2543b76a4166549f7aab2e75bef0aefc5b0f' + WHEN p.symbol = 'ARB' THEN '0x912ce59144191c1204e64559fe8253a0e49e6548' + WHEN p.symbol = 'USDT' THEN '0xfd086bc7cd5c481dcc9c85ebe478a1c0b69fcbb9' + WHEN p.symbol = 'VRTX' THEN '0x95146881b86b3ee99e63705ec87afe29fcc044d9' + WHEN p.symbol = 'TRUMPWIN' THEN '0xe215d028551d1721c6b61675aec501b1224bd0a1' + WHEN p.symbol = 'HARRISWIN' THEN '0xFBAC82A384178cA5dd6DF72965d0e65b1b8A028f' + END AS token_address, + amount, + l.fact_event_logs_id, + l._log_id, + l._inserted_timestamp + FROM + logs_pull l + LEFT JOIN {{ ref('silver__vertex_dim_products') }} + p + ON l.product_id = p.product_id +), +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + A.contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + modification_type, + trader, + subaccount, + product_id, + A.symbol, + A.token_address, + amount AS amount_unadj, + amount / pow( + 10, + 18 + ) AS amount, + ( + amount / pow( + 10, + 18 + ) * p.price + ) :: FLOAT AS amount_usd, + A.fact_event_logs_id, + A._log_id, + A._inserted_timestamp + FROM + product_id_join A + LEFT JOIN {{ ref('price__ez_prices_hourly') }} + p + ON A.token_address = p.token_address + AND DATE_TRUNC( + 'hour', + block_timestamp + ) = p.hour + LEFT JOIN {{ ref('silver__contracts') }} C + ON A.token_address = C.contract_address +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS vertex_collateral_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_dim_products.sql b/models/silver/protocols/vertex/silver__vertex_dim_products.sql new file mode 100644 index 0000000..e6e6c66 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_dim_products.sql @@ -0,0 +1,162 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'product_id', + cluster_by = ['block_timestamp::DATE'], + tags = ['curated','reorg'] +) }} + +WITH logs_pull AS ( + + SELECT + topics, + contract_address, + DATA, + tx_hash, + block_number, + block_timestamp, + fact_event_logs_id, + concat(tx_hash, '-', event_index) AS _log_id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__fact_event_logs') }} + WHERE + topics [0] :: STRING IN ( + '0x3286b0394bf1350245290b7226c92ed186bd716f28938e62dbb895298f018172', + '0x7f26b83ff96e1f2b6a682f133852f6798a09c465da95921460cefb3847402498' + ) +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' + +{% endif %} +), +new_prod AS ( + SELECT + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + utils.udf_hex_to_int( + 's2c', + segmented_data [0] :: STRING + ) :: INT AS product_id, + tx_hash, + block_number, + block_timestamp, + _inserted_timestamp, + _log_id, + fact_event_logs_id + FROM + logs_pull + WHERE + topics [0] :: STRING = '0x3286b0394bf1350245290b7226c92ed186bd716f28938e62dbb895298f018172' +), +book_address_pull AS ( + SELECT + tx_hash, + contract_address AS book_address, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + utils.udf_hex_to_int( + 's2c', + segmented_data [0] :: STRING + ) :: INT AS version + FROM + logs_pull + WHERE + topics [0] :: STRING = '0x7f26b83ff96e1f2b6a682f133852f6798a09c465da95921460cefb3847402498' +), +api_pull AS ( + SELECT + PARSE_JSON( + live.udf_api( + 'https://gateway.prod.vertexprotocol.com/api/v2/assets' + ) + ) :data AS response +), +api_lateral_flatten AS ( + SELECT + r.value + FROM + api_pull, + LATERAL FLATTEN (response) AS r +), +product_metadata AS ( + SELECT + VALUE :product_id AS product_id, + VALUE :ticker_id AS ticker_id, + VALUE :symbol AS symbol, + VALUE :name AS NAME, + VALUE :market_type AS market_type, + VALUE :taker_fee AS taker_fee, + VALUE :maker_fee AS maker_fee + FROM + api_lateral_flatten +), +FINAL AS ( + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.product_id, + CASE + WHEN l.product_id % 2 = 0 THEN 'perp' + ELSE 'spot' + END AS product_type, + CASE + WHEN l.product_id = 0 THEN 'USDC' + ELSE p.ticker_id :: STRING + END AS ticker_id, + p.symbol :: STRING AS symbol, + p.name :: STRING AS NAME, + C.book_address, + CASE + WHEN l.product_id = 0 THEN NULL + ELSE FLOOR((l.product_id - 1) / 2) + END AS health_group, + CASE + WHEN SPLIT( + p.symbol, + '-' + ) [0] = 'WBTC' THEN 'BTC' + WHEN SPLIT( + p.symbol, + '-' + ) [0] = 'WETH' THEN 'ETH' + ELSE SPLIT( + p.symbol, + '-' + ) [0] + END AS health_group_symbol, + p.taker_fee, + p.maker_fee, + CASE + WHEN C.version IS NOT NULL THEN 1 + ELSE 2 + END AS version, + l.fact_event_logs_id, + _inserted_timestamp, + _log_id + FROM + new_prod l + LEFT JOIN book_address_pull C + ON l.tx_hash = C.tx_hash + LEFT JOIN product_metadata p + ON l.product_id = p.product_id + WHERE + p.ticker_id IS NOT NULL +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','product_id'] + ) }} AS vertex_products_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY product_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_edge_trades.sql b/models/silver/protocols/vertex/silver__vertex_edge_trades.sql new file mode 100644 index 0000000..c3da098 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_edge_trades.sql @@ -0,0 +1,166 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = '_log_id', + cluster_by = ['block_timestamp::DATE'], + tags = ['curated','reorg'] +) }} + +WITH perp_trades AS ( + + SELECT + * + FROM + {{ ref('silver__vertex_perps') }} + p + WHERE + 1 = 1 +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' + +{% endif %} +), +edge_trades AS ( + SELECT + event_index - 1 AS trader_event_before, + event_index + 1 AS trader_event_after,* + FROM + perp_trades + WHERE + trader = '0x0000000000000000000000000000000000000000' +), +FINAL AS ( + SELECT + e.block_number, + e.block_timestamp, + e.tx_hash, + e.event_index AS edge_event_index, + e.trader_event_before AS user_event_index, + e.digest AS edge_digest, + p.digest AS user_digest, + p.trader, + p.subaccount, + p.symbol, + e.order_type AS edge_order_type, + p.order_type AS user_order_type, + e.is_taker AS edge_is_taker, + p.is_taker AS user_is_taker, + e.trade_type AS edge_trade_type, + p.trade_type AS user_trade_type, + e.price_amount_unadj AS edge_price_amount_unadj, + p.price_amount_unadj AS user_price_amount_unadj, + e.price_amount AS edge_price_amount, + p.price_amount AS user_price_amount, + e.amount_unadj AS edge_amount_unadj, + p.amount_unadj AS user_amount_unadj, + e.amount AS edge_amount, + p.amount AS user_amount, + e.amount_usd AS edge_amount_usd, + p.amount_usd AS user_amount_usd, + e.fee_amount_unadj AS edge_fee_amount_unadj, + p.fee_amount_unadj AS user_fee_amount_unadj, + e.fee_amount AS edge_fee_amount, + p.fee_amount AS user_fee_amount, + e.base_delta_amount_unadj AS edge_base_delta_amount_unadj, + p.base_delta_amount_unadj AS user_base_delta_amount_unadj, + e.base_delta_amount AS edge_base_delta_amount, + p.base_delta_amount AS user_base_delta_amount, + e.quote_delta_amount_unadj AS edge_quote_delta_amount_unadj, + p.quote_delta_amount_unadj AS user_quote_delta_amount_unadj, + e.quote_delta_amount AS edge_quote_delta_amount, + p.quote_delta_amount AS user_quote_delta_amount, + e.fact_event_logs_id, + e._log_id, + e._inserted_timestamp + FROM + edge_trades e + LEFT JOIN ( + SELECT + * + FROM + perp_trades + WHERE + trader <> '0x0000000000000000000000000000000000000000' + ) p + ON e.tx_hash = p.tx_hash + AND e.trader_event_before = p.event_index + AND e.product_id = p.product_id + WHERE + user_digest IS NOT NULL + UNION ALL + SELECT + e.block_number, + e.block_timestamp, + e.tx_hash, + e.event_index AS edge_event_index, + e.trader_event_after AS user_event_index, + e.digest AS edge_digest, + p.digest AS user_digest, + p.trader, + p.subaccount, + p.symbol, + e.order_type AS edge_order_type, + p.order_type AS user_order_type, + e.is_taker AS edge_is_taker, + p.is_taker AS user_is_taker, + e.trade_type AS edge_trade_type, + p.trade_type AS user_trade_type, + e.price_amount_unadj AS edge_price_amount_unadj, + p.price_amount_unadj AS user_price_amount_unadj, + e.price_amount AS edge_price_amount, + p.price_amount AS user_price_amount, + e.amount_unadj AS edge_amount_unadj, + p.amount_unadj AS user_amount_unadj, + e.amount AS edge_amount, + p.amount AS user_amount, + e.amount_usd AS edge_amount_usd, + p.amount_usd AS user_amount_usd, + e.fee_amount_unadj AS edge_fee_amount_unadj, + p.fee_amount_unadj AS user_fee_amount_unadj, + e.fee_amount AS edge_fee_amount, + p.fee_amount AS user_fee_amount, + e.base_delta_amount_unadj AS edge_base_delta_amount_unadj, + p.base_delta_amount_unadj AS user_base_delta_amount_unadj, + e.base_delta_amount AS edge_base_delta_amount, + p.base_delta_amount AS user_base_delta_amount, + e.quote_delta_amount_unadj AS edge_quote_delta_amount_unadj, + p.quote_delta_amount_unadj AS user_quote_delta_amount_unadj, + e.quote_delta_amount AS edge_quote_delta_amount, + p.quote_delta_amount AS user_quote_delta_amount, + e.fact_event_logs_id, + e._log_id, + e._inserted_timestamp + FROM + edge_trades e + LEFT JOIN ( + SELECT + * + FROM + perp_trades + WHERE + trader <> '0x0000000000000000000000000000000000000000' + ) p + ON e.tx_hash = p.tx_hash + AND e.trader_event_after = p.event_index + AND e.product_id = p.product_id + WHERE + user_digest IS NOT NULL +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','edge_event_index'] + ) }} AS vertex_edge_trade_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_liquidations.sql b/models/silver/protocols/vertex/silver__vertex_liquidations.sql new file mode 100644 index 0000000..81c2687 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_liquidations.sql @@ -0,0 +1,288 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = '_log_id', + cluster_by = ['block_timestamp::DATE'], + tags = ['curated','reorg'] +) }} + +WITH health_groups AS ( + + SELECT + product_id, + health_group, + health_group_symbol + FROM + {{ ref('silver__vertex_dim_products') }} + GROUP BY + ALL +), +logs AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + event_index, + topics, + DATA, + fact_event_logs_id, + concat(tx_hash, '-', event_index) AS _log_id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__fact_event_logs') }} + WHERE + topics [0] :: STRING IN ( + '0xf96988bb67b73fa61d64c48dc2e91ae6b697ebb8c5a496d238309aa20fbf6458', + '0x494f937f5cc892f798248aa831acfb4ad7c4bf35edd8498c5fb431ce1e38b035' + ) + AND contract_address = '0xae1ec28d6225dce2ff787dcb8ce11cf6d3ae064f' +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' + +{% endif %} +), +logs_pull_v1 AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + 'Liquidation' AS event_name, + event_index, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + topics [1] :: STRING AS digest, + LEFT( + topics [2] :: STRING, + 42 + ) AS trader, + topics [2] :: STRING AS subaccount, + utils.udf_hex_to_int( + topics [3] :: STRING + ) :: INT AS MODE, + utils.udf_hex_to_int( + segmented_data [0] :: STRING + ) :: INT AS health_group, + utils.udf_hex_to_int( + 's2c', + segmented_data [1] :: STRING + ) :: INT AS amount, + utils.udf_hex_to_int( + 's2c', + segmented_data [2] :: STRING + ) :: INT AS amount_quote, + utils.udf_hex_to_int( + segmented_data [3] :: STRING + ) :: INT AS insurance_cover, + NULL AS is_encoded_spread, + _log_id, + fact_event_logs_id, + _inserted_timestamp + FROM + logs + WHERE + topics [0] :: STRING = '0xf96988bb67b73fa61d64c48dc2e91ae6b697ebb8c5a496d238309aa20fbf6458' + AND contract_address = '0xae1ec28d6225dce2ff787dcb8ce11cf6d3ae064f' +), +logs_pull_v2 AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + 'Liquidation' AS event_name, + event_index, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + utils.udf_hex_to_int( + segmented_data [0] :: STRING + ) :: INT AS product_id, + topics [1] :: STRING AS digest, + NULL AS MODE, + LEFT( + topics [2] :: STRING, + 42 + ) AS trader, + topics [2] :: STRING AS subaccount, + utils.udf_hex_to_int( + 's2c', + segmented_data [2] :: STRING + ) :: INT AS amount, + utils.udf_hex_to_int( + 's2c', + segmented_data [3] :: STRING + ) :: INT AS amount_quote, + NULL AS insurance_cover, + udf_hex_to_int( + segmented_data [1] :: STRING + ) AS is_encoded_spread, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + logs + WHERE + topics [0] :: STRING = '0x494f937f5cc892f798248aa831acfb4ad7c4bf35edd8498c5fb431ce1e38b035' + AND contract_address = '0xae1ec28d6225dce2ff787dcb8ce11cf6d3ae064f' +), +v2_vertex_decode AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + event_name, + event_index, + is_encoded_spread, + digest, + MODE, + trader, + subaccount, + amount, + amount_quote, + insurance_cover, + CASE + WHEN is_encoded_spread = 1 THEN utils.udf_int_to_binary(product_id) + ELSE NULL + END AS bin_product_ids, + CASE + WHEN is_encoded_spread = 1 THEN ARRAY_CONSTRUCT( + utils.udf_binary_to_int(SUBSTR(bin_product_ids, -16)), + utils.udf_binary_to_int( + SUBSTR( + bin_product_ids, + 1, + GREATEST(len(bin_product_ids) - 16, 1)) + ) + ) + ELSE NULL + END AS decoded_spread_product_ids, + CASE + WHEN is_encoded_spread = 1 THEN decoded_spread_product_ids [0] :: STRING + ELSE product_id + END AS product_id, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + logs_pull_v2 + ), + FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + digest, + trader, + subaccount, + 'v1' AS version, + MODE, + NULL AS product_id, + l.health_group, + p.health_group_symbol, + amount AS amount_unadj, + amount / pow( + 10, + 18 + ) AS amount, + amount_quote AS amount_quote_unadj, + amount_quote / pow( + 10, + 18 + ) AS amount_quote, + insurance_cover AS insurance_cover_unadj, + insurance_cover / pow( + 10, + 18 + ) AS insurance_cover, + is_encoded_spread, + ARRAY_CONSTRUCT( + NULL, + NULL + ) AS spread_product_ids, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + logs_pull_v1 l + LEFT JOIN health_groups p + ON l.health_group = p.health_group + UNION ALL + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + digest, + trader, + subaccount, + 'v2' AS version, + MODE, + l.product_id, + p.health_group, + p.health_group_symbol, + amount AS amount_unadj, + amount / pow( + 10, + 18 + ) AS amount, + amount_quote AS amount_quote_unadj, + amount_quote / pow( + 10, + 18 + ) AS amount_quote, + NULL AS insurance_cover_unadj, + insurance_cover, + CASE + WHEN is_encoded_spread = 1 THEN TRUE + ELSE FALSE + END AS is_encoded_spread, + decoded_spread_product_ids AS spread_product_ids, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + v2_vertex_decode l + LEFT JOIN health_groups p + ON l.product_id = p.product_id + ) + SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS vertex_liquidation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_market_depth.sql b/models/silver/protocols/vertex/silver__vertex_market_depth.sql new file mode 100644 index 0000000..03d9a7e --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_market_depth.sql @@ -0,0 +1,105 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + full_refresh = false, + unique_key = ['product_id','hour','price'], + cluster_by = ['hour::DATE'], + tags = 'curated' +) }} + +WITH market_depth AS ({% for item in range(55) %} + + SELECT + t.ticker_id, t.product_id, DATE_TRUNC('hour', TRY_TO_TIMESTAMP(t.timestamp)) AS HOUR, 'asks' AS orderbook_side, A.value [0] :: FLOAT AS price, A.value [1] :: FLOAT AS volume, SYSDATE() AS _inserted_timestamp + FROM + ( + SELECT + response :ticker_id AS ticker_id, response :timestamp :: STRING AS TIMESTAMP, response :asks AS asks, response :bids AS bids, product_id + FROM + ( + SELECT + PARSE_JSON(live.udf_api(CONCAT('https://gateway.prod.vertexprotocol.com/v2/orderbook?ticker_id=', ticker_id, '&depth=1000000'))) :data AS response, product_id + FROM + ( + SELECT + ROW_NUMBER() over ( + ORDER BY + product_id) AS row_num, product_id, ticker_id + FROM + {{ ref('silver__vertex_dim_products') }} + WHERE + product_id > 0 + ORDER BY + product_id) + WHERE + row_num = {{ item + 1 }})) t, LATERAL FLATTEN(input => t.asks) A + UNION ALL + SELECT + t.ticker_id, t.product_id, DATE_TRUNC('hour', TRY_TO_TIMESTAMP(t.timestamp)) AS HOUR, 'bids' AS orderbook_side, A.value [0] :: FLOAT AS price, A.value [1] :: FLOAT AS volume, SYSDATE() AS _inserted_timestamp + FROM + ( + SELECT + response :ticker_id AS ticker_id, response :timestamp :: STRING AS TIMESTAMP, response :asks AS asks, response :bids AS bids, product_id + FROM + ( + SELECT + PARSE_JSON(live.udf_api(CONCAT('https://gateway.prod.vertexprotocol.com/v2/orderbook?ticker_id=', ticker_id, '&depth=1000000'))) :data AS response, product_id + FROM + ( + SELECT + ROW_NUMBER() over ( + ORDER BY + product_id) AS row_num, product_id, ticker_id + FROM + {{ ref('silver__vertex_dim_products') }} + WHERE + product_id > 0 + ORDER BY + product_id) + WHERE + row_num = {{ item + 1 }})) t, LATERAL FLATTEN(input => t.bids) A {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %}), + FINAL AS ( + SELECT + ticker_id :: STRING AS ticker_id, + product_id, + HOUR, + orderbook_side, + price, + ROUND( + price, + 2 + ) AS round_price_0_01, + ROUND( + price, + 1 + ) AS round_price_0_1, + ROUND( + price, + 0 + ) AS round_price_1, + ROUND( + price, + -1 + ) AS round_price_10, + ROUND( + price, + -2 + ) AS round_price_100, + volume, + _inserted_timestamp + FROM + market_depth + ) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['product_id','hour','price'] + ) }} AS vertex_market_depth_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL diff --git a/models/silver/protocols/vertex/silver__vertex_market_stats.sql b/models/silver/protocols/vertex/silver__vertex_market_stats.sql new file mode 100644 index 0000000..4da0fe5 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_market_stats.sql @@ -0,0 +1,175 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + full_refresh = false, + unique_key = ['ticker_id','hour'], + cluster_by = ['HOUR::DATE'], + tags = 'curated' +) }} + +WITH api_pull AS ( + + SELECT + PARSE_JSON( + live.udf_api( + 'https://archive.prod.vertexprotocol.com/v2/contracts' + ) + ) :data AS response +), +market_stats AS ( + SELECT + DATE_TRUNC('hour', SYSDATE()) AS HOUR, + f.value :base_currency :: STRING AS base_currency, + f.value :base_volume :: FLOAT AS base_volume, + f.value :contract_price :: FLOAT AS contract_price, + f.value :contract_price_currency :: STRING AS contract_price_currency, + f.value :funding_rate :: FLOAT AS funding_rate, + f.value :index_price :: FLOAT AS index_price, + f.value :last_price :: FLOAT AS last_price, + f.value :mark_price :: FLOAT AS mark_price, + TRY_TO_TIMESTAMP( + f.value :next_funding_rate_timestamp :: STRING + ) AS next_funding_rate_timestamp, + f.value :open_interest :: FLOAT AS open_interest, + f.value :open_interest_usd :: FLOAT AS open_interest_usd, + f.value :price_change_percent_24h :: FLOAT AS price_change_percent_24h, + f.value :product_type :: STRING AS product_type, + f.value :quote_currency :: STRING AS quote_currency, + f.value :quote_volume :: FLOAT AS quote_volume, + f.key AS ticker_id, + SYSDATE() AS inserted_timestamp + FROM + api_pull A, + LATERAL FLATTEN( + input => response + ) AS f +), +trade_snapshot AS ( + SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS HOUR, + CONCAT( + symbol, + '_USDC' + ) AS ticker_id, + symbol, + product_id, + COUNT(DISTINCT(tx_hash)) AS distinct_sequencer_batches, + COUNT(DISTINCT(trader)) AS distinct_trader_count, + COUNT(DISTINCT(subaccount)) AS distinct_subaccount_count, + COUNT(DISTINCT(digest)) AS trade_count, + SUM(amount_usd) AS amount_usd, + SUM(fee_amount) AS fee_amount, + SUM(base_delta_amount) AS base_delta_amount, + SUM(quote_delta_amount) AS quote_delta_amount, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('silver__vertex_perps') }} + p + WHERE + block_timestamp >= '2024-04-04 00:00:00.000' + GROUP BY + 1, + 2, + 3, + 4 +), +products AS ( + SELECT + * + FROM + {{ ref('silver__vertex_dim_products') }} +), +FINAL AS ( + SELECT + s.hour, + s.ticker_id, + p.symbol, + p.product_id, + t.distinct_sequencer_batches, + t.distinct_trader_count, + t.distinct_subaccount_count, + t.trade_count, + t.amount_usd, + t.fee_amount, + t.base_delta_amount, + t.quote_delta_amount, + s.base_volume AS base_volume_24h, + s.quote_volume AS quote_volume_24h, + s.contract_price, + s.contract_price_currency, + s.funding_rate, + s.index_price, + s.last_price, + s.mark_price, + s.next_funding_rate_timestamp, + s.open_interest, + s.open_interest_usd, + s.price_change_percent_24h, + s.product_type, + s.quote_currency, + s.quote_volume, + t._inserted_timestamp, + FROM + market_stats s + LEFT JOIN trade_snapshot t + ON t.ticker_id = s.ticker_id + AND s.hour = t.hour + LEFT JOIN products p + ON s.ticker_id = p.ticker_id + +{% if is_incremental() %} +UNION ALL +SELECT + s.hour, + s.ticker_id, + p.symbol, + p.product_id, + t.distinct_sequencer_batches, + t.distinct_trader_count, + t.distinct_subaccount_count, + t.trade_count, + t.amount_usd, + t.fee_amount, + t.base_delta_amount, + t.quote_delta_amount, + s.base_volume_24h, + s.quote_volume_24h, + s.contract_price, + s.contract_price_currency, + s.funding_rate, + s.index_price, + s.last_price, + s.mark_price, + s.next_funding_rate_timestamp, + s.open_interest, + s.open_interest_usd, + s.price_change_percent_24h, + s.product_type, + s.quote_currency, + s.quote_volume, + t._inserted_timestamp +FROM + {{ this }} + s + LEFT JOIN trade_snapshot t + ON t.ticker_id = s.ticker_id + AND s.hour = t.hour + LEFT JOIN products p + ON s.ticker_id = p.ticker_id +{% endif %} +) +SELECT + *, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['ticker_id','hour'] + ) }} AS vertex_market_stats_id, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL +WHERE FUNDING_RATE <> 0 qualify(ROW_NUMBER() over(PARTITION BY ticker_id, HOUR +ORDER BY + _inserted_timestamp DESC NULLS LAST)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_money_markets.sql b/models/silver/protocols/vertex/silver__vertex_money_markets.sql new file mode 100644 index 0000000..abaa2aa --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_money_markets.sql @@ -0,0 +1,55 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + unique_key = ['ticker_id','hour'], + cluster_by = ['HOUR::DATE'], + tags = 'curated' +) }} + +WITH apr AS ( + + SELECT + PARSE_JSON( + live.udf_api( + 'https://gateway.prod.vertexprotocol.com/v2/apr' + ) + ) :data AS response +), +flattened AS ( + SELECT + DATE_TRUNC('hour', SYSDATE()) AS HOUR, + CONCAT( + f.value :symbol :: STRING, + '_USDC' + ) AS ticker_id, + f.value :symbol :: STRING AS symbol, + f.value :product_id :: STRING AS product_id, + f.value :deposit_apr :: FLOAT AS deposit_apr, + f.value :borrow_apr :: FLOAT AS borrow_apr, + f.value :tvl :: FLOAT AS tvl + FROM + apr A, + LATERAL FLATTEN( + input => response + ) AS f +) +SELECT + HOUR, + ticker_id, + symbol, + product_id, + deposit_apr, + borrow_apr, + tvl, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['ticker_id','hour'] + ) }} AS vertex_money_markets_id, + '{{ invocation_id }}' AS _invocation_id +FROM + flattened +WHERE + product_id NOT IN ('121', '125') qualify(ROW_NUMBER() over(PARTITION BY ticker_id, HOUR +ORDER BY + inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_perps.sql b/models/silver/protocols/vertex/silver__vertex_perps.sql new file mode 100644 index 0000000..4aab80b --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_perps.sql @@ -0,0 +1,379 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = '_log_id', + cluster_by = ['block_timestamp::DATE'], + tags = ['curated','reorg'] +) }} + +WITH vertex_products AS ( + + SELECT + block_number, + block_timestamp, + tx_hash, + product_id :: STRING AS product_id, + book_address, + product_type, + ticker_id, + symbol, + NAME, + version + FROM + {{ ref('silver__vertex_dim_products') }} + WHERE + product_type = 'perp' +), +logs AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + event_index, + topics, + DATA, + fact_event_logs_id, + concat(tx_hash, '-', event_index) AS _log_id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__fact_event_logs') }} + WHERE + ( + contract_address IN ( + SELECT + book_address + FROM + vertex_products + WHERE + product_type = 'perp' + ) + OR contract_address = LOWER('0xa4369d8e3dc847aedf17f4125f1abb1bc18fc060') + ) +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' + +{% endif %} +), +order_fill_decode AS ( + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.contract_address, + 'FillOrder' AS event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + s.product_id, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + topics [1] :: STRING AS digest, + --unique hash of the order + LEFT( + topics [2] :: STRING, + 42 + ) AS trader, + topics [2] :: STRING AS subaccount, + utils.udf_hex_to_int( + segmented_data [0] :: STRING + ) :: INT AS pricex18, + utils.udf_hex_to_int( + 's2c', + segmented_data [1] :: STRING + ) :: INT AS amount, + utils.udf_hex_to_int( + segmented_data [2] :: STRING + ) AS expiration, + utils.udf_hex_to_int( + segmented_data [3] :: STRING + ) :: INT AS nonce, + utils.udf_hex_to_int( + 's2c', + segmented_data [4] :: STRING + ) :: INT AS isTaker, + utils.udf_hex_to_int( + 's2c', + segmented_data [5] :: STRING + ) :: INT AS feeAmount, + utils.udf_hex_to_int( + 's2c', + segmented_data [6] :: STRING + ) :: INT AS baseDelta, + utils.udf_hex_to_int( + 's2c', + segmented_data [7] :: STRING + ) :: INT AS quoteDelta, + l.fact_event_logs_id, + l._log_id, + l._inserted_timestamp + FROM + logs l + INNER JOIN vertex_products s + ON s.book_address = l.contract_address + WHERE + topics [0] :: STRING = '0x224253ad5cda2459ff587f559a41374ab9243acbd2daff8c13f05473db79d14c' +), +order_fill_decode_v2 AS ( + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.contract_address, + 'FillOrder' AS event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + utils.udf_hex_to_int( + topics [1] :: STRING + ) :: STRING AS product_id, + s.symbol, + topics [2] :: STRING AS digest, + --unique hash of the order + LEFT( + topics [3] :: STRING, + 42 + ) AS trader, + topics [3] :: STRING AS subaccount, + utils.udf_hex_to_int( + segmented_data [0] :: STRING + ) :: INT AS pricex18, + utils.udf_hex_to_int( + 's2c', + segmented_data [1] :: STRING + ) :: INT AS amount, + utils.udf_hex_to_int( + segmented_data [2] :: STRING + ) AS expiration, + utils.udf_hex_to_int( + segmented_data [3] :: STRING + ) :: INT AS nonce, + utils.udf_hex_to_int( + 's2c', + segmented_data [4] :: STRING + ) :: INT AS isTaker, + utils.udf_hex_to_int( + 's2c', + segmented_data [5] :: STRING + ) :: INT AS feeAmount, + utils.udf_hex_to_int( + 's2c', + segmented_data [6] :: STRING + ) :: INT AS baseDelta, + utils.udf_hex_to_int( + 's2c', + segmented_data [7] :: STRING + ) :: INT AS quoteDelta, + l.fact_event_logs_id, + l._log_id, + l._inserted_timestamp + FROM + logs l + INNER JOIN vertex_products s + ON s.product_id = utils.udf_hex_to_int( + topics [1] :: STRING + ) + WHERE + topics [0] :: STRING = '0x7c57459d6f4f0fb2fc5b1e298c8c0eb238422944964aa1e249eaa78747f0cca9' + AND contract_address = LOWER('0xa4369d8e3dc847aedf17f4125f1abb1bc18fc060') +), +order_fill_format AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + product_id, + digest, + trader, + subaccount, + 'v1' AS version, + expiration AS expiration_raw, + utils.udf_int_to_binary(TRY_TO_NUMBER(expiration)) AS exp_binary, + utils.udf_binary_to_int(SUBSTR(exp_binary, -2)) AS order_type, + utils.udf_binary_to_int(SUBSTR(exp_binary, -3, 1)) AS market_reduce_flag, + CASE + WHEN len(expiration) < 11 THEN TRY_TO_TIMESTAMP(utils.udf_binary_to_int(exp_binary) :: STRING) + ELSE TRY_TO_TIMESTAMP( + utils.udf_binary_to_int(SUBSTR(exp_binary, 24)) :: STRING + ) + END AS expiration, + nonce, + isTaker, + feeAmount AS fee_amount_unadj, + feeAmount / pow( + 10, + 18 + ) AS fee_amount, + pricex18 AS price_amount_unadj, + pricex18 / pow( + 10, + 18 + ) AS price_amount, + amount AS amount_unadj, + amount / pow( + 10, + 18 + ) AS amount, + baseDelta AS base_delta_unadj, + baseDelta / pow( + 10, + 18 + ) AS base_delta, + quoteDelta AS quote_delta_unadj, + quoteDelta / pow( + 10, + 18 + ) AS quote_delta, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + order_fill_decode + UNION ALL + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + product_id, + digest, + trader, + subaccount, + 'v2' AS version, + expiration AS expiration_raw, + utils.udf_int_to_binary(TRY_TO_NUMBER(expiration)) AS exp_binary, + utils.udf_binary_to_int(SUBSTR(exp_binary, -2)) AS order_type, + utils.udf_binary_to_int(SUBSTR(exp_binary, -3, 1)) AS market_reduce_flag, + CASE + WHEN len(expiration) < 11 THEN TRY_TO_TIMESTAMP(utils.udf_binary_to_int(exp_binary) :: STRING) + ELSE TRY_TO_TIMESTAMP( + utils.udf_binary_to_int(SUBSTR(exp_binary, 24)) :: STRING + ) + END AS expiration, + nonce, + isTaker, + feeAmount AS fee_amount_unadj, + feeAmount / pow( + 10, + 18 + ) AS fee_amount, + pricex18 AS price_amount_unadj, + pricex18 / pow( + 10, + 18 + ) AS price_amount, + amount AS amount_unadj, + amount / pow( + 10, + 18 + ) AS amount, + baseDelta AS base_delta_unadj, + baseDelta / pow( + 10, + 18 + ) AS base_delta, + quoteDelta AS quote_delta_unadj, + quoteDelta / pow( + 10, + 18 + ) AS quote_delta, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + order_fill_decode_v2 +), +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + product_id, + digest, + trader, + subaccount, + version, + CASE + WHEN amount < 0 THEN 'sell/short' + WHEN amount > 0 THEN 'buy/long' + END AS trade_type, + expiration_raw, + exp_binary, + order_type AS order_type_raw, + CASE + WHEN order_type = 0 THEN 'default' + WHEN order_type = 1 THEN 'immediate-or-cancel' + WHEN order_type = 2 THEN 'fill-or-kill' + WHEN order_type = 3 THEN 'post-only' + END AS order_type, + market_reduce_flag, + expiration, + nonce, + CASE + WHEN isTaker = 1 THEN TRUE + WHEN isTaker = 0 THEN FALSE + END AS is_taker, + price_amount_unadj, + price_amount, + amount_unadj, + amount, + CASE + WHEN quote_delta < 0 THEN quote_delta * -1 + ELSE quote_delta + END AS amount_usd, + fee_amount_unadj, + fee_amount, + base_delta_unadj AS base_delta_amount_unadj, + base_delta AS base_delta_amount, + quote_delta_unadj AS quote_delta_amount_unadj, + quote_delta AS quote_delta_amount, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + order_fill_format +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS vertex_perps_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_spot.sql b/models/silver/protocols/vertex/silver__vertex_spot.sql new file mode 100644 index 0000000..8e01f24 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_spot.sql @@ -0,0 +1,379 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = '_log_id', + cluster_by = ['block_timestamp::DATE'], + tags = ['curated','reorg'] +) }} + +WITH vertex_products AS ( + + SELECT + block_number, + block_timestamp, + tx_hash, + product_id :: STRING AS product_id, + book_address, + product_type, + ticker_id, + symbol, + NAME, + version + FROM + {{ ref('silver__vertex_dim_products') }} + WHERE + product_type = 'spot' +), +logs AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + event_index, + topics, + DATA, + fact_event_logs_id, + concat(tx_hash, '-', event_index) AS _log_id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__fact_event_logs') }} + WHERE + ( + contract_address IN ( + SELECT + book_address + FROM + vertex_products + WHERE + product_type = 'spot' + ) + OR contract_address = LOWER('0xa4369d8e3dc847aedf17f4125f1abb1bc18fc060') + ) +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' + +{% endif %} +), +order_fill_decode AS ( + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.contract_address, + 'FillOrder' AS event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + s.product_id, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + topics [1] :: STRING AS digest, + --unique hash of the order + LEFT( + topics [2] :: STRING, + 42 + ) AS trader, + topics [2] :: STRING AS subaccount, + utils.udf_hex_to_int( + segmented_data [0] :: STRING + ) :: INT AS pricex18, + utils.udf_hex_to_int( + 's2c', + segmented_data [1] :: STRING + ) :: INT AS amount, + utils.udf_hex_to_int( + segmented_data [2] :: STRING + ) AS expiration, + utils.udf_hex_to_int( + segmented_data [3] :: STRING + ) :: INT AS nonce, + utils.udf_hex_to_int( + 's2c', + segmented_data [4] :: STRING + ) :: INT AS isTaker, + utils.udf_hex_to_int( + 's2c', + segmented_data [5] :: STRING + ) :: INT AS feeAmount, + utils.udf_hex_to_int( + 's2c', + segmented_data [6] :: STRING + ) :: INT AS baseDelta, + utils.udf_hex_to_int( + 's2c', + segmented_data [7] :: STRING + ) :: INT AS quoteDelta, + l.fact_event_logs_id, + l._log_id, + l._inserted_timestamp + FROM + logs l + INNER JOIN vertex_products s + ON s.book_address = l.contract_address + WHERE + topics [0] :: STRING = '0x224253ad5cda2459ff587f559a41374ab9243acbd2daff8c13f05473db79d14c' +), +order_fill_decode_v2 AS ( + SELECT + l.block_number, + l.block_timestamp, + l.tx_hash, + l.contract_address, + 'FillOrder' AS event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + utils.udf_hex_to_int( + topics [1] :: STRING + ) AS product_id, + s.symbol, + topics [2] :: STRING AS digest, + --unique hash of the order + LEFT( + topics [3] :: STRING, + 42 + ) AS trader, + topics [3] :: STRING AS subaccount, + utils.udf_hex_to_int( + segmented_data [0] :: STRING + ) :: INT AS pricex18, + utils.udf_hex_to_int( + 's2c', + segmented_data [1] :: STRING + ) :: INT AS amount, + utils.udf_hex_to_int( + segmented_data [2] :: STRING + ) AS expiration, + utils.udf_hex_to_int( + segmented_data [3] :: STRING + ) :: INT AS nonce, + utils.udf_hex_to_int( + 's2c', + segmented_data [4] :: STRING + ) :: INT AS isTaker, + utils.udf_hex_to_int( + 's2c', + segmented_data [5] :: STRING + ) :: INT AS feeAmount, + utils.udf_hex_to_int( + 's2c', + segmented_data [6] :: STRING + ) :: INT AS baseDelta, + utils.udf_hex_to_int( + 's2c', + segmented_data [7] :: STRING + ) :: INT AS quoteDelta, + l.fact_event_logs_id, + l._log_id, + l._inserted_timestamp + FROM + logs l + INNER JOIN vertex_products s + ON s.product_id = utils.udf_hex_to_int( + topics [1] :: STRING + ) + WHERE + topics [0] :: STRING = '0x7c57459d6f4f0fb2fc5b1e298c8c0eb238422944964aa1e249eaa78747f0cca9' + AND contract_address = LOWER('0xa4369d8e3dc847aedf17f4125f1abb1bc18fc060') +), +order_fill_format AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + product_id, + digest, + trader, + subaccount, + 'v1' AS version, + expiration AS expiration_raw, + utils.udf_int_to_binary(TRY_TO_NUMBER(expiration)) AS exp_binary, + utils.udf_binary_to_int(SUBSTR(exp_binary, -2)) AS order_type, + utils.udf_binary_to_int(SUBSTR(exp_binary, -3, 1)) AS market_reduce_flag, + CASE + WHEN len(expiration) < 11 THEN TRY_TO_TIMESTAMP(utils.udf_binary_to_int(exp_binary) :: STRING) + ELSE TRY_TO_TIMESTAMP( + utils.udf_binary_to_int(SUBSTR(exp_binary, 24)) :: STRING + ) + END AS expiration, + nonce, + isTaker, + feeAmount AS fee_amount_unadj, + feeAmount / pow( + 10, + 18 + ) AS fee_amount, + pricex18 AS price_amount_unadj, + pricex18 / pow( + 10, + 18 + ) AS price_amount, + amount AS amount_unadj, + amount / pow( + 10, + 18 + ) AS amount, + baseDelta AS base_delta_unadj, + baseDelta / pow( + 10, + 18 + ) AS base_delta, + quoteDelta AS quote_delta_unadj, + quoteDelta / pow( + 10, + 18 + ) AS quote_delta, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + order_fill_decode + UNION ALL + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + product_id, + digest, + trader, + subaccount, + 'v2' AS version, + expiration AS expiration_raw, + utils.udf_int_to_binary(TRY_TO_NUMBER(expiration)) AS exp_binary, + utils.udf_binary_to_int(SUBSTR(exp_binary, -2)) AS order_type, + utils.udf_binary_to_int(SUBSTR(exp_binary, -3, 1)) AS market_reduce_flag, + CASE + WHEN len(expiration) < 11 THEN TRY_TO_TIMESTAMP(utils.udf_binary_to_int(exp_binary) :: STRING) + ELSE TRY_TO_TIMESTAMP( + utils.udf_binary_to_int(SUBSTR(exp_binary, 24)) :: STRING + ) + END AS expiration, + nonce, + isTaker, + feeAmount AS fee_amount_unadj, + feeAmount / pow( + 10, + 18 + ) AS fee_amount, + pricex18 AS price_amount_unadj, + pricex18 / pow( + 10, + 18 + ) AS price_amount, + amount AS amount_unadj, + amount / pow( + 10, + 18 + ) AS amount, + baseDelta AS base_delta_unadj, + baseDelta / pow( + 10, + 18 + ) AS base_delta, + quoteDelta AS quote_delta_unadj, + quoteDelta / pow( + 10, + 18 + ) AS quote_delta, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + order_fill_decode_v2 +), +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + contract_address, + event_name, + event_index, + origin_function_signature, + origin_from_address, + origin_to_address, + symbol, + product_id, + digest, + trader, + subaccount, + version, + CASE + WHEN amount < 0 THEN 'sell/short' + WHEN amount > 0 THEN 'buy/long' + END AS trade_type, + expiration_raw, + exp_binary, + order_type AS order_type_raw, + CASE + WHEN order_type = 0 THEN 'default' + WHEN order_type = 1 THEN 'immediate-or-cancel' + WHEN order_type = 2 THEN 'fill-or-kill' + WHEN order_type = 3 THEN 'post-only' + END AS order_type, + market_reduce_flag, + expiration, + nonce, + CASE + WHEN isTaker = 1 THEN TRUE + WHEN isTaker = 0 THEN FALSE + END AS is_taker, + price_amount_unadj, + price_amount, + amount_unadj, + amount, + CASE + WHEN quote_delta < 0 THEN quote_delta * -1 + ELSE quote_delta + END AS amount_usd, + fee_amount_unadj, + fee_amount, + base_delta_unadj AS base_delta_amount_unadj, + base_delta AS base_delta_amount, + quote_delta_unadj AS quote_delta_amount_unadj, + quote_delta AS quote_delta_amount, + fact_event_logs_id, + _log_id, + _inserted_timestamp + FROM + order_fill_format +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} AS vertex_spot_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_staking.sql b/models/silver/protocols/vertex/silver__vertex_staking.sql new file mode 100644 index 0000000..9be2040 --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_staking.sql @@ -0,0 +1,101 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = '_log_id', + cluster_by = ['block_timestamp::DATE'], + tags = ['curated','reorg'] +) }} + +WITH stake_pull AS ( + + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + from_address, + to_address, + raw_amount_precise :: INT AS amount_unadj, + amount, + amount_usd, + symbol, + concat(tx_hash, '-', event_index) AS _log_id, + modified_timestamp AS _inserted_timestamp + FROM + {{ ref('core__ez_token_transfers') }} + WHERE + --v1 staking contract + ( + to_address = LOWER('0x5Be754aD77766089c4284d914F0cC37E8E3F669A') + OR from_address = LOWER('0x5Be754aD77766089c4284d914F0cC37E8E3F669A') + ) + --v2 staking contract + OR ( + contract_address = LOWER('0x95146881b86B3ee99e63705eC87AfE29Fcc044D9') + AND ( + to_address = LOWER('0x6e89C20F182b1744405603958eC5E3fd93441cc4') + OR from_address = LOWER('0x6e89C20F182b1744405603958eC5E3fd93441cc4') + ) + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '12 hours' + FROM + {{ this }} +) +AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' + +{% endif %} +), +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + contract_address, + from_address, + to_address, + CASE + WHEN to_address = LOWER('0x5Be754aD77766089c4284d914F0cC37E8E3F669A') THEN 'stake' + WHEN to_address = LOWER('0x6e89C20F182b1744405603958eC5E3fd93441cc4') + AND from_address = LOWER('0x5Be754aD77766089c4284d914F0cC37E8E3F669A') THEN 'migrate-stake' + WHEN to_address = LOWER('0x6e89C20F182b1744405603958eC5E3fd93441cc4') THEN 'stake' + WHEN from_address = LOWER('0x6e89C20F182b1744405603958eC5E3fd93441cc4') THEN 'withdraw/claim' + WHEN from_address = LOWER('0x5Be754aD77766089c4284d914F0cC37E8E3F669A') THEN 'withdraw/claim' + ELSE NULL + END AS stake_action, + amount_unadj, + amount, + amount_usd, + symbol, + _log_id, + _inserted_timestamp + FROM + stake_pull + WHERE + symbol IN ( + 'USDC', + 'VRTX' + ) + AND to_address <> from_address +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['_log_id'] + ) }} AS vertex_staking_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/vertex/silver__vertex_staking.yml b/models/silver/protocols/vertex/silver__vertex_staking.yml new file mode 100644 index 0000000..2ea948f --- /dev/null +++ b/models/silver/protocols/vertex/silver__vertex_staking.yml @@ -0,0 +1,84 @@ +version: 2 +models: + - name: silver__vertex_staking + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: STAKE_ACTION + tests: + - not_null + - name: AMOUNT_UNADJ + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: AMOUNT + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: AMOUNT_USD + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SYMBOL + tests: + - not_null + - name: _LOG_ID + tests: + - not_null +