diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index da75f001..4718c138 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -15,5 +15,5 @@ jobs: with: command_name: Run Main Models command: | - dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "ethereum_models,tag:silver,tag:balances" "ethereum_models,tag:gold,tag:balances" "ethereum_models,tag:silver,tag:reads" "ethereum_models,tag:gold,tag:reads" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" "fsc_evm,tag:streamline,tag:token_reads,tag:realtime" "fsc_evm,tag:streamline,tag:token_reads,tag:complete" "ethereum_models,tag:streamline_balances_realtime" "ethereum_models,tag:streamline_balances_complete" "ethereum_models,tag:streamline_decoded_traces_complete" "ethereum_models,tag:streamline_decoded_traces_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "ethereum_models,tag:silver,tag:balances" "ethereum_models,tag:gold,tag:balances" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" "fsc_evm,tag:streamline,tag:token_reads,tag:realtime" "fsc_evm,tag:streamline,tag:token_reads,tag:complete" "ethereum_models,tag:streamline_balances_realtime" "ethereum_models,tag:streamline_balances_complete" "ethereum_models,tag:streamline_decoded_traces_complete" "ethereum_models,tag:streamline_decoded_traces_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_reads.yml b/.github/workflows/dbt_run_streamline_reads.yml deleted file mode 100644 index 96c64330..00000000 --- a/.github/workflows/dbt_run_streamline_reads.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: dbt_run_streamline_reads -run-name: dbt_run_streamline_reads - -on: - workflow_dispatch: - branches: - - "main" - -jobs: - called_workflow_template: - uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main - with: - command_name: Run History Models - command: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_reads_curated" "ethereum_models,tag:streamline_reads_realtime" "ethereum_models,tag:streamline_reads_complete" - secrets: inherit diff --git a/models/gold/core/core__fact_contract_reads.sql b/models/gold/core/core__fact_contract_reads.sql deleted file mode 100644 index ec9e15e1..00000000 --- a/models/gold/core/core__fact_contract_reads.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - tags = ['gold','reads'] -) }} - -SELECT - contract_address, - block_number, - function_signature, - function_input, - read_output, - segmented_data AS segmented_output, - COALESCE ( - reads_id, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'contract_address', 'function_signature', 'function_input'] - ) }} - ) AS fact_contract_reads_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__reads') }} diff --git a/models/gold/core/core__fact_contract_reads.yml b/models/gold/core/core__fact_contract_reads.yml deleted file mode 100644 index 3bc5210b..00000000 --- a/models/gold/core/core__fact_contract_reads.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 -models: - - name: core__fact_contract_reads - description: 'This table contains contract reads for a variety of functions and contracts on the Ethereum Blockchain.' - - columns: - - name: CONTRACT_ADDRESS - description: 'The address of the contract read.' - - name: BLOCK_NUMBER - description: 'The block number the contract address was read at.' - - name: FUNCTION_SIGNATURE - description: 'The function signature called by this read. See `dim_function_signatures` for text versions.' - - name: FUNCTION_INPUT - description: 'The function inputs called in this read, if applicable.' - - name: READ_OUTPUT - description: 'The return of the contract call.' - - name: SEGMENTED_OUTPUT - description: 'The return of the contract call, segmented into 64 character strings.' - - name: FACT_CONTRACT_READS_ID - description: '{{ doc("evm_pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("evm_inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_lp_actions.sql b/models/gold/protocols/uniswapv3/uniswapv3__ez_lp_actions.sql deleted file mode 100644 index 2c57c4a9..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_lp_actions.sql +++ /dev/null @@ -1,64 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - meta={ - 'database_tags':{ - 'table': { - 'PROTOCOL': 'UNISWAPV3', - 'PURPOSE': 'DEFI, DEX' - } - } - }, - tags = ['gold','uniswap','curated'] -) }} - -SELECT - blockchain, - block_number, - block_timestamp, - tx_hash, - action, - amount0_adjusted, - amount1_adjusted, - amount0_usd, - amount1_usd, - token0_address, - token1_address, - token0_symbol, - token1_symbol, - token0_price, - token1_price, - liquidity, - liquidity_adjusted, - liquidity_provider, - nf_position_manager_address, - nf_token_id, - pool_address, - pool_name, - tick_lower, - tick_upper, - price_lower_1_0, - price_upper_1_0, - price_lower_0_1, - price_upper_0_1, - price_lower_1_0_usd, - price_upper_1_0_usd, - price_lower_0_1_usd, - price_upper_0_1_usd, - COALESCE ( - univ3_lp_actions_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'event_index'] - ) }} - ) AS ez_lp_actions_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__univ3_lp_actions') }} diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_lp_actions.yml b/models/gold/protocols/uniswapv3/uniswapv3__ez_lp_actions.yml deleted file mode 100644 index 5de24a6f..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_lp_actions.yml +++ /dev/null @@ -1,80 +0,0 @@ -version: 2 -models: - - name: uniswapv3__ez_lp_actions - description: '{{ doc("evm_lp_actions_table_doc") }}' - - columns: - - name: BLOCKCHAIN - description: '{{ doc("evm_blockchain") }}' - - name: BLOCK_NUMBER - description: '{{ doc("evm_block_number") }}' - - name: BLOCK_TIMESTAMP - description: '{{ doc("evm_block_timestamp") }}' - - name: TX_HASH - description: '{{ doc("evm_transfer_tx_hash") }}' - - name: ACTION - description: '{{ doc("evm_lp_actions_action") }}' - - name: AMOUNT0_ADJUSTED - description: '{{ doc("evm_swaps_amount0_adjusted") }}' - - name: AMOUNT1_ADJUSTED - description: '{{ doc("evm_swaps_amount1_adjusted") }}' - - name: AMOUNT0_USD - description: '{{ doc("evm_swaps_amount0_usd") }}' - - name: AMOUNT1_USD - description: '{{ doc("evm_swaps_amount1_usd") }}' - - name: TOKEN0_ADDRESS - description: '{{ doc("evm_all_token0_address") }}' - - name: TOKEN1_ADDRESS - description: '{{ doc("evm_all_token0_address") }}' - - name: TOKEN0_SYMBOL - description: '{{ doc("evm_all_token0_symbol") }}' - - name: TOKEN1_SYMBOL - description: '{{ doc("evm_all_token1_symbol") }}' - - name: TOKEN0_PRICE - description: '{{ doc("evm_all_token0_price") }}' - - name: TOKEN1_PRICE - description: '{{ doc("evm_all_token1_price") }}' - - name: LIQUIDITY - description: '{{ doc("evm_all_liquidity") }}' - - name: LIQUIDITY_ADJUSTED - description: '{{ doc("evm_all_liquidity_adjusted") }}' - - name: LIQUIDITY_PROVIDER - description: '{{ doc("evm_positions_liquidity_provider") }}' - - name: NF_POSITION_MANAGER_ADDRESS - description: '{{ doc("evm_positions_nf_position_manager_address") }}' - - name: NF_TOKEN_ID - description: '{{ doc("evm_positions_nf_token_id") }}' - - name: POOL_ADDRESS - description: '{{ doc("evm_all_pool_address") }}' - - name: POOL_NAME - description: '{{ doc("evm_all_pool_name") }}' - - name: TICK_LOWER - description: '{{ doc("evm_positions_tick_lower") }}' - - name: TICK_UPPER - description: '{{ doc("evm_positions_tick_upper") }}' - - name: PRICE_LOWER_1_0 - description: '{{ doc("evm_positions_price_lower_1_0") }}' - - name: PRICE_UPPER_1_0 - description: '{{ doc("evm_positions_price_upper_1_0") }}' - - name: PRICE_LOWER_0_1 - description: '{{ doc("evm_positions_price_lower_0_1") }}' - - name: PRICE_UPPER_0_1 - description: '{{ doc("evm_positions_price_upper_0_1") }}' - - name: PRICE_LOWER_1_0_USD - description: '{{ doc("evm_positions_price_lower_1_0_usd") }}' - - name: PRICE_UPPER_1_0_USD - description: '{{ doc("evm_positions_price_upper_1_0_usd") }}' - - name: PRICE_LOWER_0_1_USD - description: '{{ doc("evm_positions_price_lower_0_1_usd") }}' - - name: PRICE_UPPER_0_1_USD - description: '{{ doc("evm_positions_price_upper_0_1_usd") }}' - - name: EZ_LP_ACTIONS_ID - description: '{{ doc("evm_pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("evm_inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("evm_modified_timestamp") }}' - - name: _INSERTED_TIMESTAMP - description: '{{ doc("evm_internal_column") }}' - - name: _LOG_ID - description: '{{ doc("evm_internal_column") }}' \ No newline at end of file diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_pool_stats.sql b/models/gold/protocols/uniswapv3/uniswapv3__ez_pool_stats.sql deleted file mode 100644 index 84305baf..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_pool_stats.sql +++ /dev/null @@ -1,54 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'UNISWAPV3', - 'PURPOSE': 'DEFI, DEX' } } }, - tags = ['gold','uniswap','curated'] -) }} - -SELECT - blockchain, - block_number, - block_timestamp, - fee_growth_global0_x128, - fee_growth_global1_x128, - pool_address, - pool_name, - price_0_1, - price_1_0, - protocol_fees_token0_adjusted, - protocol_fees_token1_adjusted, - token0_address, - token1_address, - token0_symbol, - token1_symbol, - tick, - unlocked, - virtual_liquidity_adjusted, - virtual_reserves_token0_adjusted, - virtual_reserves_token1_adjusted, - virtual_reserves_token0_usd, - virtual_reserves_token1_usd, - token0_balance_adjusted, - token1_balance_adjusted, - token0_balance_usd, - token1_balance_usd, - token0_balance, - token1_balance, - COALESCE ( - univ3_pool_stats_id, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'pool_address'] - ) }} - ) AS ez_pool_stats_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__univ3_pool_stats') }} \ No newline at end of file diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_pool_stats.yml b/models/gold/protocols/uniswapv3/uniswapv3__ez_pool_stats.yml deleted file mode 100644 index 563c0a74..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_pool_stats.yml +++ /dev/null @@ -1,68 +0,0 @@ -version: 2 -models: - - name: uniswapv3__ez_pool_stats - description: '{{ doc("evm_pool_stats_table_doc") }}' - - columns: - - name: BLOCK_NUMBER - description: '{{ doc("evm_block_number") }}' - - name: BLOCK_TIMESTAMP - description: '{{ doc("evm_block_timestamp") }}' - - name: BLOCKCHAIN - description: '{{ doc("evm_blockchain") }}' - - name: FEE_GROWTH_global0_X128 - description: '{{ doc("evm_pool_stats_fee_growth_global0_x128") }}' - - name: FEE_GROWTH_global1_X128 - description: '{{ doc("evm_pool_stats_fee_growth_global1_x128") }}' - - name: POOL_ADDRESS - description: '{{ doc("evm_all_pool_address") }}' - - name: POOL_NAME - description: '{{ doc("evm_all_pool_name") }}' - - name: PRICE_0_1 - description: '{{ doc("evm_swaps_price_0_1") }}' - - name: PRICE_1_0 - description: '{{ doc("evm_swaps_price_1_0") }}' - - name: PROTOCOL_FEES_TOKEN0_ADJUSTED - description: '{{ doc("evm_pool_stats_protocol_fees_token0_adjusted") }}' - - name: PROTOCOL_FEES_TOKEN1_ADJUSTED - description: '{{ doc("evm_pool_stats_protocol_fees_token1_adjusted") }}' - - name: TOKEN0_ADDRESS - description: '{{ doc("evm_all_token0_address") }}' - - name: TOKEN1_ADDRESS - description: '{{ doc("evm_all_token1_address") }}' - - name: TOKEN0_SYMBOL - description: '{{ doc("evm_all_token0_symbol") }}' - - name: TOKEN1_SYMBOL - description: '{{ doc("evm_all_token1_symbol") }}' - - name: TICK - description: '{{ doc("evm_pool_stats_tick") }}' - - name: UNLOCKED - description: '{{ doc("evm_pool_stats_unlocked") }}' - - name: VIRTUAL_LIQUIDITY_ADJUSTED - description: '{{ doc("evm_pool_stats_virtual_liquidity_adjusted") }}' - - name: VIRTUAL_RESERVES_TOKEN0_ADJUSTED - description: '{{ doc("evm_pool_stats_virtual_reserves_token0_adjusted") }}' - - name: VIRTUAL_RESERVES_TOKEN1_ADJUSTED - description: '{{ doc("evm_pool_stats_virtual_reserves_token1_adjusted") }}' - - name: VIRTUAL_RESERVES_TOKEN0_USD - description: '{{ doc("evm_pool_stats_virtual_reserves_token0_usd") }}' - - name: VIRTUAL_RESERVES_TOKEN1_USD - description: '{{ doc("evm_pool_stats_virtual_reserves_token1_usd") }}' - - name: TOKEN0_BALANCE_ADJUSTED - description: '{{ doc("evm_pool_stats_token0_balance_adjusted") }}' - - name: TOKEN1_BALANCE_ADJUSTED - description: '{{ doc("evm_pool_stats_token1_balance_adjusted") }}' - - name: TOKEN0_BALANCE_USD - description: '{{ doc("evm_pool_stats_token0_balance_usd") }}' - - name: TOKEN1_BALANCE_USD - description: '{{ doc("evm_pool_stats_token1_balance_usd") }}' - - name: TOKEN0_BALANCE - description: '{{ doc("evm_pool_stats_token0_balance") }}' - - name: TOKEN1_BALANCE - description: '{{ doc("evm_pool_stats_token1_balance") }}' - - name: EZ_POOL_STATS_ID - description: '{{ doc("evm_pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("evm_inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_position_collected_fees.sql b/models/gold/protocols/uniswapv3/uniswapv3__ez_position_collected_fees.sql deleted file mode 100644 index bbab4817..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_position_collected_fees.sql +++ /dev/null @@ -1,54 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - meta={ - 'database_tags':{ - 'table': { - 'PROTOCOL': 'UNISWAPV3', - 'PURPOSE': 'DEFI, DEX' - } - } - }, - tags = ['gold','uniswap','curated'] -) }} - -SELECT - blockchain, - block_number, - block_timestamp, - tx_hash, - event_index, - pool_address, - pool_name, - liquidity_provider, - nf_token_id, - nf_position_manager_address, - token0_symbol, - token1_symbol, - amount0_adjusted, - amount1_adjusted, - amount0_usd, - amount1_usd, - tick_lower, - tick_upper, - price_lower, - price_upper, - price_lower_usd, - price_upper_usd, - COALESCE ( - univ3_position_collected_fees_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'event_index'] - ) }} - ) AS ez_position_collected_fees_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__univ3_position_collected_fees') }} \ No newline at end of file diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_position_collected_fees.yml b/models/gold/protocols/uniswapv3/uniswapv3__ez_position_collected_fees.yml deleted file mode 100644 index ea850c4c..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_position_collected_fees.yml +++ /dev/null @@ -1,71 +0,0 @@ -version: 2 -models: - - name: uniswapv3__ez_position_collected_fees - description: '{{ doc("evm_position_collected_fees_table_doc") }}' - - columns: - - name: BLOCKCHAIN - description: '{{ doc("evm_blockchain") }}' - - - name: BLOCK_NUMBER - description: '{{ doc("evm_block_number") }}' - - - name: BLOCK_TIMESTAMP - description: '{{ doc("evm_block_timestamp") }}' - - - name: TX_HASH - description: '{{ doc("evm_transfer_tx_hash") }}' - - - name: POOL_ADDRESS - description: '{{ doc("evm_all_pool_address") }}' - - - name: LIQUIDITY_PROVIDER - description: '{{ doc("evm_positions_liquidity_provider") }}' - - - name: NF_POSITION_MANAGER_ADDRESS - description: '{{ doc("evm_positions_nf_position_manager_address") }}' - - - name: NF_TOKEN_ID - description: '{{ doc("evm_positions_nf_token_id") }}' - - - name: TOKEN0_SYMBOL - description: '{{ doc("evm_all_token0_symbol") }}' - - - name: TOKEN1_SYMBOL - description: '{{ doc("evm_all_token1_symbol") }}' - - - name: AMOUNT0_ADJUSTED - description: '{{ doc("evm_swaps_amount0_adjusted") }}' - - - name: AMOUNT1_ADJUSTED - description: '{{ doc("evm_swaps_amount1_adjusted") }}' - - - name: AMOUNT0_USD - description: '{{ doc("evm_swaps_amount0_usd") }}' - - - name: AMOUNT1_USD - description: '{{ doc("evm_swaps_amount1_usd") }}' - - - name: TICK_LOWER - description: '{{ doc("evm_positions_tick_lower") }}' - - - name: TICK_UPPER - description: '{{ doc("evm_positions_tick_upper") }}' - - - name: PRICE_LOWER - description: '{{ doc("evm_positions_collected_fees_price_lower") }}' - - - name: PRICE_UPPER - description: '{{ doc("evm_positions_collected_fees_price_upper") }}' - - - name: PRICE_LOWER_USD - description: '{{ doc("evm_positions_collected_fees_price_lower_usd") }}' - - - name: PRICE_UPPER_USD - description: '{{ doc("evm_positions_collected_fees_price_upper_usd") }}' - - name: EZ_POSITION_COLLECTED_FEES_ID - description: '{{ doc("evm_pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("evm_inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("evm_modified_timestamp") }}' diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_positions.sql b/models/gold/protocols/uniswapv3/uniswapv3__ez_positions.sql deleted file mode 100644 index 5b9e20ca..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_positions.sql +++ /dev/null @@ -1,64 +0,0 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true }, - meta={ - 'database_tags':{ - 'table': { - 'PROTOCOL': 'UNISWAPV3', - 'PURPOSE': 'DEFI, DEX' - } - } - }, - tags = ['gold','uniswap','curated'] -) }} - -SELECT - blockchain, - block_number, - block_timestamp, - tx_hash, - fee_percent, - fee_growth_inside0_last_x128, - fee_growth_inside1_last_x128, - is_active, - liquidity_adjusted, - liquidity_provider, - nf_position_manager_address, - nf_token_id, - pool_address, - pool_name, - tick_upper, - tick_lower, - price_upper_1_0, - price_lower_1_0, - price_upper_0_1, - price_lower_0_1, - price_upper_1_0_usd, - price_lower_1_0_usd, - price_upper_0_1_usd, - price_lower_0_1_usd, - tokens_owed0_adjusted, - tokens_owed1_adjusted, - tokens_owed0_usd, - tokens_owed1_usd, - token0_address, - token1_address, - token0_symbol, - token1_symbol, - COALESCE ( - univ3_positions_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'event_index'] - ) }} - ) AS ez_positions_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp -FROM - {{ ref('silver__univ3_positions') }} \ No newline at end of file diff --git a/models/gold/protocols/uniswapv3/uniswapv3__ez_positions.yml b/models/gold/protocols/uniswapv3/uniswapv3__ez_positions.yml deleted file mode 100644 index 17e21e25..00000000 --- a/models/gold/protocols/uniswapv3/uniswapv3__ez_positions.yml +++ /dev/null @@ -1,76 +0,0 @@ -version: 2 -models: - - name: uniswapv3__ez_positions - description: '{{ doc("evm_positions_table_doc") }}' - - columns: - - name: BLOCKCHAIN - description: '{{ doc("evm_blockchain") }}' - - name: BLOCK_NUMBER - description: '{{ doc("evm_block_number") }}' - - name: BLOCK_TIMESTAMP - description: '{{ doc("evm_block_timestamp") }}' - - name: TX_HASH - description: '{{ doc("evm_transfer_tx_hash") }}' - - name: FEE_PERCENT - description: '{{ doc("evm_positions_fee_percent") }}' - - name: FEE_GROWTH_INSIDE0_LAST_x128 - description: '{{ doc("evm_positions_fee_growth_inside0_last_x128") }}' - - name: FEE_GROWTH_INSIDE1_LAST_x128 - description: '{{ doc("evm_positions_fee_growth_inside1_last_x128") }}' - - name: IS_ACTIVE - description: '{{ doc("evm_positions_is_active") }}' - - name: LIQUIDITY_ADJUSTED - description: '{{ doc("evm_all_liquidity_adjusted") }}' - - name: LIQUIDITY_PROVIDER - description: '{{ doc("evm_positions_liquidity_provider") }}' - - name: NF_POSITION_MANAGER_ADDRESS - description: '{{ doc("evm_positions_nf_position_manager_address") }}' - - name: NF_TOKEN_ID - description: '{{ doc("evm_positions_nf_token_id") }}' - - name: POOL_ADDRESS - description: '{{ doc("evm_all_pool_address") }}' - - name: POOL_NAME - description: '{{ doc("evm_all_pool_name") }}' - - name: TICK_LOWER - description: '{{ doc("evm_positions_tick_lower") }}' - - name: TICK_UPPER - description: '{{ doc("evm_positions_tick_upper") }}' - - name: PRICE_UPPER_1_0 - description: '{{ doc("evm_positions_price_upper_1_0") }}' - - name: PRICE_LOWER_1_0 - description: '{{ doc("evm_positions_price_lower_1_0") }}' - - name: PRICE_UPPER_0_1 - description: '{{ doc("evm_positions_price_upper_0_1") }}' - - name: PRICE_LOWER_0_1 - description: '{{ doc("evm_positions_price_lower_0_1") }}' - - name: PRICE_UPPER_1_0_USD - description: '{{ doc("evm_positions_price_upper_1_0_usd") }}' - - name: PRICE_LOWER_1_0_USD - description: '{{ doc("evm_positions_price_lower_1_0_usd") }}' - - name: PRICE_UPPER_0_1_USD - description: '{{ doc("evm_positions_price_upper_0_1_usd") }}' - - name: PRICE_LOWER_0_1_USD - description: '{{ doc("evm_positions_price_lower_0_1_usd") }}' - - name: TOKENS_OWED0_ADJUSTED - description: '{{ doc("evm_positions_token_owed0_adjusted") }}' - - name: TOKENS_OWED1_ADJUSTED - description: '{{ doc("evm_positions_token_owed1_adjusted") }}' - - name: TOKENS_OWED0_USD - description: '{{ doc("evm_positions_tokens_owed0_usd") }}' - - name: TOKENS_OWED1_USD - description: '{{ doc("evm_positions_tokens_owed1_usd") }}' - - name: TOKEN0_ADDRESS - description: '{{ doc("evm_all_token0_address") }}' - - name: TOKEN1_ADDRESS - description: '{{ doc("evm_all_token1_address") }}' - - name: TOKEN0_SYMBOL - description: '{{ doc("evm_all_token0_symbol") }}' - - name: TOKEN1_SYMBOL - description: '{{ doc("evm_all_token1_symbol") }}' - - name: EZ_POSITION_ID - description: '{{ doc("evm_pk") }}' - - name: INSERTED_TIMESTAMP - description: '{{ doc("evm_inserted_timestamp") }}' - - name: MODIFIED_TIMESTAMP - description: '{{ doc("evm_modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/core/silver__token_meta_reads.sql b/models/silver/core/silver__token_meta_reads.sql deleted file mode 100644 index 505b3441..00000000 --- a/models/silver/core/silver__token_meta_reads.sql +++ /dev/null @@ -1,213 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = 'contract_address', - tags = ['silver','reads'] -) }} - -WITH - -{% if is_incremental() %} -heal_table AS ( - - SELECT - contract_address - FROM - {{ this }} - WHERE - token_name IS NULL - OR len(REGEXP_REPLACE(token_name, '[^a-zA-Z0-9]+')) <= 0 - OR token_symbol IS NULL - OR len(REGEXP_REPLACE(token_symbol, '[^a-zA-Z0-9]+')) <= 0 - OR token_decimals IS NULL -), -{% endif %} - -reads_base_metadata AS ( - SELECT - contract_address, - block_number, - function_sig AS function_signature, - function_input, - read_result AS read_output, - _inserted_timestamp - FROM - {{ ref('bronze_api__token_reads') }} - WHERE - read_result IS NOT NULL - AND read_result <> '0x00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000' - AND read_output <> '0x' - - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) :: DATE - 2 - FROM - {{ this }} -) -OR contract_address IN ( - SELECT - DISTINCT contract_address - FROM - heal_table -) -{% endif %} -), -uni_base_metadata AS ( - SELECT - * - FROM - {{ ref('silver__reads') }} - WHERE - function_signature IN ( - '0x06fdde03', - '0x313ce567', - '0x95d89b41' - ) - AND call_name = 'uni_v3_token_reads' - -{% if is_incremental() %} -AND ( -_inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) :: DATE - 2 - FROM - {{ this }} -) -OR contract_address IN ( - SELECT - DISTINCT contract_address - FROM - heal_table -)) -AND _inserted_timestamp >= dateadd(day, -7, sysdate()) -{% endif %} -), -base_metadata AS ( - SELECT - contract_address, - block_number, - function_signature, - function_input, - read_output, - _inserted_timestamp - FROM - reads_base_metadata - UNION ALL - SELECT - contract_address, - block_number, - function_signature, - function_input, - read_output, - _inserted_timestamp - FROM - uni_base_metadata -), -token_names AS ( - SELECT - contract_address, - block_number, - function_signature, - read_output, - len(read_output) AS output_len, - CASE - WHEN SUBSTR( - read_output, - 3, - 1 - ) <> '0' - AND SUBSTR( - read_output, - 3, - 1 - ) <> '' THEN utils.udf_hex_to_string(SUBSTR(read_output, 3, len(read_output))) - ELSE utils.udf_hex_to_string(SUBSTR(read_output,(64 * 2 + 3), len(read_output))) END AS token_name - FROM - base_metadata - WHERE - function_signature = '0x06fdde03' - qualify(ROW_NUMBER() over(PARTITION BY contract_address - ORDER BY - _inserted_timestamp DESC)) = 1 - ), -token_symbols AS ( - SELECT - contract_address, - block_number, - function_signature, - read_output, - len(read_output) AS output_len, - CASE - WHEN SUBSTR( - read_output, - 3, - 1 - ) <> '0' - AND SUBSTR( - read_output, - 3, - 1 - ) <> '' THEN utils.udf_hex_to_string(SUBSTR(read_output, 3, len(read_output))) - ELSE utils.udf_hex_to_string(SUBSTR(read_output,(64 * 2 + 3), len(read_output))) END AS token_symbol - FROM - base_metadata - WHERE - function_signature = '0x95d89b41' - qualify(ROW_NUMBER() over(PARTITION BY contract_address - ORDER BY - _inserted_timestamp DESC)) = 1 - ), -token_decimals AS ( - SELECT - contract_address, - MAX( - CASE - WHEN len(read_output) > 66 THEN udf_hex_to_int(REGEXP_REPLACE(read_output, '0+$', '')) - ELSE udf_hex_to_int(read_output) - END - ) AS token_decimals - FROM - base_metadata - WHERE - function_signature = '0x313ce567' - GROUP BY - 1 -), -contracts AS ( - SELECT - contract_address, - MAX(_inserted_timestamp) AS _inserted_timestamp - FROM - base_metadata - GROUP BY - 1 - ) -SELECT - c1.contract_address AS contract_address, - token_name, - token_decimals, - token_symbol, - CASE - WHEN token_name IS NULL - OR len(REGEXP_REPLACE(token_name, '[^a-zA-Z0-9]+')) <= 0 - OR token_decimals IS NULL - OR token_symbol IS NULL - OR len(REGEXP_REPLACE(token_symbol, '[^a-zA-Z0-9]+')) <= 0 THEN 'incomplete' - ELSE 'complete' - END AS complete_f, - _inserted_timestamp -FROM - contracts c1 - LEFT JOIN token_names - ON c1.contract_address = token_names.contract_address - LEFT JOIN token_symbols - ON c1.contract_address = token_symbols.contract_address - LEFT JOIN token_decimals - ON c1.contract_address = token_decimals.contract_address qualify(ROW_NUMBER() over(PARTITION BY c1.contract_address -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__token_meta_reads.yml b/models/silver/core/silver__token_meta_reads.yml deleted file mode 100644 index 8f003b92..00000000 --- a/models/silver/core/silver__token_meta_reads.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: 2 -models: - - name: silver__token_meta_reads - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - contract_address - columns: - - name: contract_address - tests: - - not_null - - name: _INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/defi/lending/compound/silver__comp_market_stats.sql b/models/silver/defi/lending/compound/silver__comp_market_stats.sql deleted file mode 100644 index 0026a145..00000000 --- a/models/silver/defi/lending/compound/silver__comp_market_stats.sql +++ /dev/null @@ -1,374 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = 'id', - cluster_by = ['_inserted_timestamp::date'], - merge_exclude_columns = ["inserted_timestamp"], - tags = ['stale'] -) }} - -WITH market_reads AS ( - - SELECT - contract_address, - block_number, - function_signature, - read_output, - segmented_data, - _inserted_timestamp - FROM - {{ ref('silver__reads') }} - WHERE - function_signature IN ( - '0x18160ddd', - '0xf8f9da28', - '0x182df0f5', - '0xae9d70b0', - '0x47bd3718', - '0x8f840ddd' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '48 hours' - FROM - {{ this }} -) -{% endif %} -), -comptroller_reads AS ( - SELECT - contract_address, - block_number, - function_signature, - read_output, - segmented_data - FROM - {{ ref('silver__reads') }} - WHERE - function_signature IN ( - '0x1d7b33d7', - '0x6aa875b5', - '0xf4a433c0' - ) - AND read_output :: STRING <> '0x' - AND contract_address = '0x3d9819210a31b4961b30ef54be2aed79b9c9cd3b' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} -), -spine AS ( - SELECT - block_number, - contract_address, - MAX(_inserted_timestamp) AS _inserted_timestamp - FROM - market_reads - GROUP BY - 1, - 2 -), -total_supply AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS total_supply - FROM - market_reads - WHERE - function_signature = '0x18160ddd' -), -borrow_rate_per_block AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS borrow_rate_per_block - FROM - market_reads - WHERE - function_signature = '0xf8f9da28' -), -exchange_rate_stored AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS exchange_rate_stored - FROM - market_reads - WHERE - function_signature = '0x182df0f5' -), -supply_rate_per_block AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS supply_rate_per_block - FROM - market_reads - WHERE - function_signature = '0xae9d70b0' -), -total_borrows AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS total_borrows - FROM - market_reads - WHERE - function_signature = '0x47bd3718' -), -total_reserves AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS total_reserves - FROM - market_reads - WHERE - function_signature = '0x8f840ddd' -), -comp_supply_speeds AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS comp_supply_speeds - FROM - comptroller_reads - WHERE - function_signature = '0x6aa875b5' -), -comp_borrow_speeds AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS comp_borrow_speeds - FROM - comptroller_reads - WHERE - function_signature = '0xf4a433c0' -), -comp_speeds AS ( - SELECT - block_number, - contract_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS comp_speeds - FROM - comptroller_reads - WHERE - function_signature = '0x1d7b33d7' -), -token_meta AS ( - SELECT - ctoken_address, - ctoken_symbol, - ctoken_name, - ctoken_decimals, - underlying_asset_address, - underlying_decimals, - underlying_symbol - FROM - {{ ref('silver__comp_asset_details') }} -), -token_prices AS ( - SELECT - HOUR, - token_address, - AVG(price) AS price - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - token_address IN ( - SELECT - DISTINCT underlying_asset_address - FROM - token_meta - ) - GROUP BY - 1, - 2 -), -blocks AS ( - SELECT - block_number, - block_timestamp - FROM - {{ ref('core__fact_blocks') }} -) -SELECT - s.block_number, - b.block_timestamp, - DATE_TRUNC( - 'hour', - b.block_timestamp - ) AS block_hour, - s.contract_address AS ctoken_address, - s._inserted_timestamp, - total_supply, - total_supply / pow( - 10, - ctoken_decimals - ) AS supply_token_amount, - borrow_rate_per_block, - pow( - borrow_rate_per_block / 1e18 * ((60 / 13.15) * 60 * 24) + 1, - 365 - ) - 1 AS borrow_apy, - exchange_rate_stored, - exchange_rate_stored / pow( - 10, - 18 + ( - underlying_decimals - ctoken_decimals - ) - ) * token_prices.price AS ctoken_price, - supply_rate_per_block, - pow( - supply_rate_per_block / 1e18 * ((60 / 13.15) * 60 * 24) + 1, - 365 - ) - 1 AS supply_apy, - total_borrows, - total_borrows / pow( - 10, - underlying_decimals - ) AS borrows_token_amount, - total_reserves, - total_reserves / pow( - 10, - underlying_decimals - ) AS reserves_token_amount, - NULLIF( - comp_supply_speeds / 1e18 * comp_p.price, - 0 - ) AS comp_supply_usd, - NULLIF( - comp_borrow_speeds / 1e18 * comp_p.price, - 0 - ) AS comp_borrow_usd, - NULLIF( - comp_speeds / 1e18 * comp_p.price, - 0 - ) AS comp_speed_usd, - CONCAT( - s.block_number, - '-', - s.contract_address - ) AS id, - supply_token_amount * ctoken_price AS supply_usd, - reserves_token_amount * token_prices.price AS reserves_usd, - borrows_token_amount * token_prices.price AS borrows_usd, - NULLIF( - CASE - WHEN borrows_usd != 0 THEN power( - ( - 1 + ( - ( - COALESCE(NULLIF(comp_speed_usd, 0), comp_borrow_usd) * 24 - ) / borrows_usd - ) - ), - 365 - ) -1 - ELSE NULL - END, - 0 - ) AS comp_apy_borrow, - NULLIF( - CASE - WHEN supply_usd != 0 THEN power( - ( - 1 + ( - ( - COALESCE(NULLIF(comp_speed_usd, 0), comp_supply_usd) * 24 - ) / supply_usd - ) - ), - 365 - ) -1 - ELSE NULL - END, - 0 - ) AS comp_apy_supply, - underlying_asset_address AS underlying_contract, - underlying_symbol, - token_prices.price AS token_price, - comp_p.price AS comp_price, - ctoken_symbol AS contract_name, - comp_speeds AS comp_speed, - {{ dbt_utils.generate_surrogate_key( - ['s.block_number', 's.contract_address'] - ) }} AS comp_market_stats_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - spine s - JOIN blocks b - ON s.block_number = b.block_number - LEFT JOIN total_supply - ON total_supply.block_number = s.block_number - AND total_supply.contract_address = s.contract_address - LEFT JOIN borrow_rate_per_block - ON borrow_rate_per_block.block_number = s.block_number - AND borrow_rate_per_block.contract_address = s.contract_address - LEFT JOIN exchange_rate_stored - ON exchange_rate_stored.block_number = s.block_number - AND exchange_rate_stored.contract_address = s.contract_address - LEFT JOIN supply_rate_per_block - ON supply_rate_per_block.block_number = s.block_number - AND supply_rate_per_block.contract_address = s.contract_address - LEFT JOIN total_borrows - ON total_borrows.block_number = s.block_number - AND total_borrows.contract_address = s.contract_address - LEFT JOIN total_reserves - ON total_reserves.block_number = s.block_number - AND total_reserves.contract_address = s.contract_address - LEFT JOIN comp_supply_speeds - ON comp_supply_speeds.block_number = s.block_number - AND comp_supply_speeds.contract_address = s.contract_address - LEFT JOIN comp_borrow_speeds - ON comp_borrow_speeds.block_number = s.block_number - AND comp_borrow_speeds.contract_address = s.contract_address - LEFT JOIN comp_speeds - ON comp_speeds.block_number = s.block_number - AND comp_speeds.contract_address = s.contract_address - INNER JOIN token_meta - ON s.contract_address = token_meta.ctoken_address - LEFT JOIN token_prices - ON DATE_TRUNC( - 'hour', - b.block_timestamp - ) = token_prices.hour - AND token_prices.token_address = underlying_asset_address - LEFT JOIN token_prices comp_p - ON DATE_TRUNC( - 'hour', - b.block_timestamp - ) = comp_p.hour - AND comp_p.token_address = '0xc00e94cb662c3520282e6f5717214004a7f26888' qualify(ROW_NUMBER() over(PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/silver/protocols/uniswapv3/silver__univ3_lp_actions.sql b/models/silver/protocols/uniswapv3/silver__univ3_lp_actions.sql deleted file mode 100644 index 8072a5b1..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_lp_actions.sql +++ /dev/null @@ -1,313 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = "block_number", - cluster_by = ['block_timestamp::DATE'], - tags = ['silver','curated','uniswap'] -) }} - -WITH lp_actions_base AS ( - - SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - contract_address, - topics, - topic_0, - topic_1, - topic_2, - topic_3, - DATA, - event_removed, - origin_from_address, - origin_to_address, - origin_function_signature, - tx_succeeded, - fact_event_logs_id, - inserted_timestamp, - modified_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, - modified_timestamp AS _inserted_timestamp, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data - FROM - {{ ref('core__fact_event_logs') }} - WHERE - block_timestamp :: DATE > '2021-04-01' - AND tx_succeeded - AND event_removed = 'false' - AND topics [0] :: STRING IN ( - '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c', - '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde', - '0x3067048beee31b25b2f1681f88dac838c8bba36af25bfb2b7cf7473a5847e35f', - '0x26f6a048ee9138f2c0ce266f322cb99228e8d619ae2bff30c67f8dcf9d2377b4' - ) -- burn / mint / IncreaseLiquidity / DecreaseLiquidity - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} -), -uni_pools AS ( - SELECT - token0_address, - token1_address, - fee, - fee_percent, - tick_spacing, - pool_address, - token0_symbol, - token1_symbol, - token0_decimals, - token1_decimals, - pool_name - FROM - {{ ref('silver__univ3_pools') }} -), --- pulls info for increases or decreases (mint / burn events) in liquidity -lp_amounts AS ( - SELECT - tx_hash, - event_index, - block_timestamp, - block_number, - 'ethereum' AS blockchain, - _log_id, - A._inserted_timestamp, - topics, - segmented_data, - CASE - WHEN topics [0] :: STRING = '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c' THEN 'DECREASE_LIQUIDITY' - WHEN topics [0] :: STRING = '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde' THEN 'INCREASE_LIQUIDITY' - END AS action, - contract_address AS pool_address, - CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS vault_address, - utils.udf_hex_to_int( - 's2c', - topics [2] :: STRING - ) :: FLOAT AS tick_lower, - utils.udf_hex_to_int( - 's2c', - topics [3] :: STRING - ) :: FLOAT AS tick_upper, - CASE - WHEN topics [0] :: STRING = '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde' THEN utils.udf_hex_to_int( - 's2c', - segmented_data [2] :: STRING - ) - WHEN topics [0] :: STRING = '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c' THEN utils.udf_hex_to_int( - 's2c', - segmented_data [1] :: STRING - ) - END AS amount0, - CASE - WHEN topics [0] :: STRING = '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde' THEN utils.udf_hex_to_int( - 's2c', - segmented_data [3] :: STRING - ) - WHEN topics [0] :: STRING = '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c' THEN utils.udf_hex_to_int( - 's2c', - segmented_data [2] :: STRING - ) - END AS amount1, - CASE - WHEN topics [0] :: STRING = '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c' THEN utils.udf_hex_to_int( - 's2c', - segmented_data [0] :: STRING - ) - WHEN topics [0] :: STRING = '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde' THEN utils.udf_hex_to_int( - 's2c', - segmented_data [1] :: STRING - ) - END AS amount, - token0_address, - token1_address, - origin_to_address, - origin_from_address, - amount + amount0 + amount1 AS total_amount, - ROW_NUMBER() over( - PARTITION BY tx_hash, - amount, - amount0, - amount1 - ORDER BY - event_index ASC - ) AS agg_id - FROM - lp_actions_base A - INNER JOIN uni_pools - ON contract_address = pool_address - WHERE - topics [0] :: STRING IN ( - '0x0c396cd989a39f4459b5fa1aed6a9a8dcdbc45908acfd67e028cd568da98982c', - '0x7a53080ba414158be7ec69b987b5fb7d07dee101fe85488f0853ae16239d0bde' - ) -), --- pulls info for token IDs minted or burned, if applicable (not all positions have token IDs) -nf_info AS ( - SELECT - tx_hash, - utils.udf_hex_to_int( - topics [1] :: STRING - ) :: INTEGER AS nf_token_id, - contract_address AS nf_position_manager_address, - CASE - WHEN topics [0] :: STRING = '0x3067048beee31b25b2f1681f88dac838c8bba36af25bfb2b7cf7473a5847e35f' THEN 'INCREASE_LIQUIDITY' - ELSE 'DECREASE_LIQUIDITY' - END AS action, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) AS liquidity, - utils.udf_hex_to_int( - segmented_data [1] :: STRING - ) AS amount0, - utils.udf_hex_to_int( - segmented_data [2] :: STRING - ) AS amount1, - ROW_NUMBER() over( - PARTITION BY tx_hash, - liquidity, - amount0, - amount1 - ORDER BY - event_index ASC - ) AS agg_id - FROM - lp_actions_base - WHERE - contract_address = '0xc36442b4a4522e871399cd717abdd847ab11fe88' - AND topics [0] :: STRING IN ( - '0x3067048beee31b25b2f1681f88dac838c8bba36af25bfb2b7cf7473a5847e35f', - '0x26f6a048ee9138f2c0ce266f322cb99228e8d619ae2bff30c67f8dcf9d2377b4' - ) -), -FINAL AS ( - SELECT - blockchain, - block_number, - block_timestamp, - A.tx_hash AS tx_hash, - A.action AS action, - token0_address, - token1_address, - A.amount0 AS amount0, - A.amount1 AS amount1, - A.amount :: INTEGER AS liquidity, - A.origin_from_address AS liquidity_provider, - nf_token_id, - CASE - WHEN nf_token_id IS NULL THEN vault_address - ELSE nf_position_manager_address - END AS nf_position_manager_address, - pool_address, - A.tick_lower AS tick_lower, - A.tick_upper AS tick_upper, - A._log_id AS _log_id, - _inserted_timestamp, - a.event_index - FROM - lp_amounts A - LEFT JOIN nf_info C - ON A.tx_hash = C.tx_hash - AND A.amount = C.liquidity - AND A.amount0 = C.amount0 - AND A.amount1 = C.amount1 - AND A.agg_id = C.agg_id -), -silver_lp_actions AS ( - SELECT - * - FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id - ORDER BY - _inserted_timestamp DESC)) = 1 -), -token_prices AS ( - SELECT - HOUR, - LOWER(token_address) AS token_address, - price - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - HOUR :: DATE IN ( - SELECT - DISTINCT block_timestamp :: DATE - FROM - silver_lp_actions - ) -) -SELECT - 'ethereum' AS blockchain, - block_number, - block_timestamp, - tx_hash, - action, - amount0 / pow( - 10, - token0_decimals - ) AS amount0_adjusted, - amount1 / pow( - 10, - token1_decimals - ) AS amount1_adjusted, - amount0_adjusted * p0.price AS amount0_usd, - amount1_adjusted * p1.price AS amount1_usd, - A.token0_address, - A.token1_address, - token0_symbol, - token1_symbol, - p0.price AS token0_price, - p1.price AS token1_price, - liquidity, - liquidity / pow(10, (token1_decimals + token0_decimals) / 2) AS liquidity_adjusted, - liquidity_provider, - nf_position_manager_address, - nf_token_id, - A.pool_address, - pool_name, - tick_lower, - tick_upper, - pow(1.0001, (tick_lower)) / pow(10,(token1_decimals - token0_decimals)) AS price_lower_1_0, - pow(1.0001, (tick_upper)) / pow(10,(token1_decimals - token0_decimals)) AS price_upper_1_0, - pow(1.0001, -1 * (tick_upper)) / pow(10,(token0_decimals - token1_decimals)) AS price_lower_0_1, - pow(1.0001, -1 * (tick_lower)) / pow(10,(token0_decimals - token1_decimals)) AS price_upper_0_1, - price_lower_1_0 * p1.price AS price_lower_1_0_usd, - price_upper_1_0 * p1.price AS price_upper_1_0_usd, - price_lower_0_1 * p0.price AS price_lower_0_1_usd, - price_upper_0_1 * p0.price AS price_upper_0_1_usd, - _log_id, - _inserted_timestamp, - event_index, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'event_index'] - ) }} AS univ3_lp_actions_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - silver_lp_actions A - INNER JOIN uni_pools p - ON A.pool_address = p.pool_address - LEFT JOIN token_prices p0 - ON p0.token_address = A.token0_address - AND p0.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) - LEFT JOIN token_prices p1 - ON p1.token_address = A.token1_address - AND p1.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) diff --git a/models/silver/protocols/uniswapv3/silver__univ3_lp_actions.yml b/models/silver/protocols/uniswapv3/silver__univ3_lp_actions.yml deleted file mode 100644 index 0700126c..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_lp_actions.yml +++ /dev/null @@ -1,56 +0,0 @@ -version: 2 -models: - - name: silver__univ3_lp_actions - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - _LOG_ID - columns: - - name: ACTION - tests: - - not_null - - name: BLOCK_NUMBER - tests: - - not_null - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - name: AMOUNT0_ADJUSTED - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - decimal - - float - - name: AMOUNT1_ADJUSTED - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - decimal - - float - - name: BLOCKCHAIN - tests: - - not_null - - name: LIQUIDITY_ADJUSTED - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - decimal - - float - - name: LIQUIDITY_PROVIDER - tests: - - not_null - - name: POOL_ADDRESS - tests: - - not_null - - name: TICK_LOWER - tests: - - not_null - - name: TICK_UPPER - tests: - - not_null - - name: TX_HASH - tests: - - not_null diff --git a/models/silver/protocols/uniswapv3/silver__univ3_pool_stats.sql b/models/silver/protocols/uniswapv3/silver__univ3_pool_stats.sql deleted file mode 100644 index 94535c65..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_pool_stats.sql +++ /dev/null @@ -1,417 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['block_timestamp::date', 'pool_address'], - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'id', - tags = ['silver','curated','uniswap'] -) }} - -WITH block_date AS ( - - SELECT - block_timestamp, - block_number - FROM - {{ ref("core__fact_blocks") }} -), -base_pool_data AS ( - SELECT - A.*, - b.block_timestamp, - segmented_data AS segmented_output - FROM - {{ ref('silver__reads') }} A - JOIN block_date b - ON A.block_number = b.block_number - WHERE - function_signature IN ( - '0x1ad8b03b', - '0x1a686502', - '0x46141319', - '0x3850c7bd', - '0xf3058399' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - INTERVAL '36 hours' - FROM - {{ this }} -) -{% endif %} -), -pool_meta AS ( - SELECT - token0_address, - token1_address, - fee_percent, - tick_spacing, - pool_address, - token0_symbol, - token1_symbol, - token0_decimals, - token1_decimals, - pool_name - FROM - {{ ref('silver__univ3_pools') }} -), -protocol_fees_base AS ( - SELECT - contract_address, - block_number, - utils.udf_hex_to_int( - segmented_output [0] :: STRING - ) :: FLOAT AS token0_protocol_fees, - utils.udf_hex_to_int( - segmented_output [1] :: STRING - ) :: FLOAT AS token1_protocol_fees - FROM - base_pool_data - WHERE - function_signature = '0x06fdde03' -), -liquidity_base AS ( - SELECT - contract_address, - block_number, - utils.udf_hex_to_int( - segmented_output [0] :: STRING - ) :: FLOAT AS liquidity - FROM - base_pool_data - WHERE - function_signature = '0x1a686502' -), -feeGrowthGlobal1X128_base AS ( - SELECT - contract_address, - block_number, - utils.udf_hex_to_int( - segmented_output [0] :: STRING - ) :: FLOAT AS feeGrowthGlobal1X128 - FROM - base_pool_data - WHERE - function_signature = '0x46141319' -), -feeGrowthGlobal0X128_base AS ( - SELECT - contract_address, - block_number, - utils.udf_hex_to_int( - segmented_output [0] :: STRING - ) :: FLOAT AS feeGrowthGlobal0X128 - FROM - base_pool_data - WHERE - function_signature = '0xf3058399' -), -slot0_base AS ( - SELECT - contract_address, - block_number, - utils.udf_hex_to_int( - segmented_output [0] :: STRING - ) :: FLOAT AS sqrtPriceX96, - utils.udf_hex_to_int( - 's2c', - segmented_output [1] :: STRING - ) :: FLOAT AS tick, - utils.udf_hex_to_int( - segmented_output [2] :: STRING - ) :: FLOAT AS observationIndex, - utils.udf_hex_to_int( - segmented_output [3] :: STRING - ) :: FLOAT AS observationCardinality, - utils.udf_hex_to_int( - segmented_output [4] :: STRING - ) :: FLOAT AS observationCardinalityNext, - utils.udf_hex_to_int( - segmented_output [5] :: STRING - ) :: FLOAT AS feeProtocol, - utils.udf_hex_to_int( - segmented_output [6] :: STRING - ) :: FLOAT AS unlocked - FROM - base_pool_data - WHERE - function_signature = '0x3850c7bd' -), -contract_range AS ( - SELECT - block_number, - block_timestamp, - contract_address, - MAX(_inserted_timestamp) AS _inserted_timestamp - FROM - base_pool_data - GROUP BY - 1, - 2, - 3 -), -join_meta AS ( - SELECT - c1.block_number AS block_number, - c1.contract_address AS pool_address, - c1.block_timestamp AS block_timestamp, - liquidity, - feeGrowthGlobal1X128, - feeGrowthGlobal0X128, - sqrtPriceX96, - tick, - observationIndex, - observationCardinality, - observationCardinalityNext, - feeProtocol, - token0_protocol_fees, - token1_protocol_fees, - CASE - WHEN unlocked = 1 THEN TRUE - ELSE FALSE - END AS unlocked, - CONCAT( - c1.block_number, - '-', - c1.contract_address - ) AS id, - c1._inserted_timestamp AS _inserted_timestamp, - token0_address, - token1_address, - ((sqrtpricex96 * sqrtpricex96)) / pow( - 2, - 192 - ) AS human_price, - SQRT(human_price) AS sqrt_hp - FROM - contract_range c1 - LEFT JOIN protocol_fees_base - ON c1.contract_address = protocol_fees_base.contract_address - AND c1.block_number = protocol_fees_base.block_number - LEFT JOIN liquidity_base - ON c1.contract_address = liquidity_base.contract_address - AND c1.block_number = liquidity_base.block_number - LEFT JOIN feeGrowthGlobal1X128_base - ON c1.contract_address = feeGrowthGlobal1X128_base.contract_address - AND c1.block_number = feeGrowthGlobal1X128_base.block_number - LEFT JOIN feeGrowthGlobal0X128_base - ON c1.contract_address = feeGrowthGlobal0X128_base.contract_address - AND c1.block_number = feeGrowthGlobal0X128_base.block_number - LEFT JOIN slot0_base - ON c1.contract_address = slot0_base.contract_address - AND c1.block_number = slot0_base.block_number - LEFT JOIN pool_meta - ON c1.contract_address = pool_address -), -token_balances AS ( - SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS block_hour, - address, - contract_address, - AVG( - balance :: FLOAT - ) AS balance - FROM - {{ ref('silver__token_balances') }} - WHERE - block_timestamp :: DATE > '2021-04-01' - AND address IN ( - SELECT - DISTINCT pool_address - FROM - pool_meta - ) - AND block_timestamp :: DATE IN ( - SELECT - DISTINCT block_timestamp :: DATE - FROM - join_meta - ) - GROUP BY - 1, - 2, - 3 -), -daily_balances AS ( - SELECT - block_hour :: DATE AS block_date, - address, - contract_address, - AVG(balance) AS daily_balance - FROM - token_balances - GROUP BY - 1, - 2, - 3 -), -max_balances AS ( - SELECT - address, - contract_address, - balance AS max_balance - FROM - token_balances qualify(ROW_NUMBER() over(PARTITION BY address, contract_address - ORDER BY - block_hour DESC)) = 1 -), -silver_pool_stats AS ( - SELECT - A.*, - COALESCE( - b0.balance, - db0.daily_balance, - mb0.max_balance, - 0 - ) AS token0_balance, - COALESCE( - b1.balance, - db1.daily_balance, - mb1.max_balance, - 0 - ) AS token1_balance - FROM - join_meta A - LEFT JOIN token_balances AS b0 - ON A.pool_address = b0.address - AND A.token0_address = b0.contract_address - AND DATE_TRUNC( - 'hour', - A.block_timestamp - ) = b0.block_hour - LEFT JOIN token_balances AS b1 - ON A.pool_address = b1.address - AND A.token1_address = b1.contract_address - AND DATE_TRUNC( - 'hour', - A.block_timestamp - ) = b1.block_hour - LEFT JOIN daily_balances db0 - ON A.pool_address = db0.address - AND A.token0_address = db0.contract_address - AND A.block_timestamp :: DATE = db0.block_date - LEFT JOIN daily_balances db1 - ON A.pool_address = db1.address - AND A.token1_address = db1.contract_address - AND A.block_timestamp :: DATE = db1.block_date - LEFT JOIN max_balances mb0 - ON A.pool_address = mb0.address - AND A.token0_address = mb0.contract_address - LEFT JOIN max_balances mb1 - ON A.pool_address = mb1.address - AND A.token1_address = mb1.contract_address qualify(ROW_NUMBER() over(PARTITION BY id - ORDER BY - A._inserted_timestamp DESC)) = 1 -), -token_prices AS ( - SELECT - HOUR, - LOWER(token_address) AS token_address, - price - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - HOUR :: DATE IN ( - SELECT - DISTINCT block_timestamp :: DATE - FROM - silver_pool_stats - ) -) -SELECT - 'ethereum' AS blockchain, - block_number, - block_timestamp, - feeGrowthGlobal0X128 AS fee_growth_global0_x128, - feeGrowthGlobal1X128 AS fee_growth_global1_x128, - A.pool_address, - pool_name, - pow( - 1.0001, - tick - ) / pow( - 10, - token1_decimals - token0_decimals - ) AS price_1_0, - 1 / price_1_0 AS price_0_1, - COALESCE( - token0_protocol_fees / pow( - 10, - token0_decimals - ), - 0 - ) AS protocol_fees_token0_adjusted, - COALESCE( - token1_protocol_fees / pow( - 10, - token1_decimals - ), - 0 - ) AS protocol_fees_token1_adjusted, - A.token0_address, - A.token1_address, - token0_symbol, - token1_symbol, - tick, - unlocked, - COALESCE( - liquidity / pow(10,(token1_decimals + token0_decimals) / 2), - 0 - ) AS virtual_liquidity_adjusted, - div0( - liquidity, - sqrt_hp - ) / pow( - 10, - token0_decimals - ) AS virtual_reserves_token0_adjusted, - ( - liquidity * sqrt_hp - ) / pow( - 10, - token1_decimals - ) AS virtual_reserves_token1_adjusted, - virtual_reserves_token0_adjusted * p0.price AS virtual_reserves_token0_usd, - virtual_reserves_token1_adjusted * p1.price AS virtual_reserves_token1_usd, - token0_balance, - token1_balance, - token0_balance / pow( - 10, - token0_decimals - ) AS token0_balance_adjusted, - token1_balance / pow( - 10, - token1_decimals - ) AS token1_balance_adjusted, - token0_balance_adjusted * p0.price AS token0_balance_usd, - token1_balance_adjusted * p1.price AS token1_balance_usd, - id, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'a.pool_address'] - ) }} AS univ3_pool_stats_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - silver_pool_stats A - LEFT JOIN pool_meta p - ON A.pool_address = p.pool_address - LEFT JOIN token_prices p0 - ON p0.token_address = A.token0_address - AND p0.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) - LEFT JOIN token_prices p1 - ON p1.token_address = A.token1_address - AND p1.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) diff --git a/models/silver/protocols/uniswapv3/silver__univ3_pool_stats.yml b/models/silver/protocols/uniswapv3/silver__univ3_pool_stats.yml deleted file mode 100644 index 5cf37015..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_pool_stats.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: 2 -models: - - name: silver__univ3_pool_stats - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - BLOCK_NUMBER - - POOL_ADDRESS - columns: - - name: BLOCK_NUMBER - tests: - - not_null - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 3 - - name: feeGrowthGlobal1X128 - tests: - - not_null: - enabled: False - - name: feeGrowthGlobal0X128 - tests: - - not_null: - enabled: False - - name: VIRTUAL_LIQUIDITY_ADJUSTED - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - decimal - - float - - name: POOL_ADDRESS - tests: - - not_null - - name: PROTOCOL_FEES_TOKEN0 - # always null in data - - name: PROTOCOL_FEES_TOKEN1 - # always null in data - - name: TICK - tests: - - not_null: - enabled: False - - name: TOKEN0_BALANCE - tests: - - not_null - - name: TOKEN1_BALANCE - tests: - - not_null - - name: UNLOCKED - tests: - - not_null \ No newline at end of file diff --git a/models/silver/protocols/uniswapv3/silver__univ3_position_collected_fees.sql b/models/silver/protocols/uniswapv3/silver__univ3_position_collected_fees.sql deleted file mode 100644 index 3a4585f0..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_position_collected_fees.sql +++ /dev/null @@ -1,243 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = "block_number", - cluster_by = ['block_timestamp::DATE'], - tags = ['silver','curated','uniswap'] -) }} - -WITH all_collected AS ( - - SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - contract_address, - topics, - topic_0, - topic_1, - topic_2, - topic_3, - DATA, - event_removed, - origin_from_address, - origin_to_address, - origin_function_signature, - tx_succeeded, - fact_event_logs_id, - inserted_timestamp, - modified_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__fact_event_logs') }} - WHERE - block_timestamp :: DATE > '2021-04-01' - AND tx_succeeded - AND event_removed = 'false' - AND topics [0] :: STRING IN ( - '0x70935338e69775456a85ddef226c395fb668b63fa0115f5f20610b388e6ca9c0', - '0x40d0efd1a53d60ecbf40971b9daf7dc90178c3aadc7aab1765632738fa8b8f01' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} -), -collected_base AS ( - SELECT - *, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS vault_address, - CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS owner, - utils.udf_hex_to_int( - 's2c', - topics [2] :: STRING - ) AS tick_lower, - utils.udf_hex_to_int( - 's2c', - topics [3] :: STRING - ) AS tick_upper, - utils.udf_hex_to_int( - 's2c', - segmented_data [1] :: STRING - ) AS amount0, - utils.udf_hex_to_int( - 's2c', - segmented_data [2] :: STRING - ) AS amount1 - FROM - all_collected - WHERE - topics [0] :: STRING = '0x70935338e69775456a85ddef226c395fb668b63fa0115f5f20610b388e6ca9c0' -), -nf_token_id_base AS ( - SELECT - tx_hash, - contract_address AS nf_position_manager_address, - utils.udf_hex_to_int( - 's2c', - topics [1] :: STRING - ) AS nf_token_id, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 27, 40)) AS liquidity_provider, - utils.udf_hex_to_int( - 's2c', - segmented_data [1] :: STRING - ) AS amount0, - utils.udf_hex_to_int( - 's2c', - segmented_data [2] :: STRING - ) AS amount1, - event_index, - event_index - 1 AS event_index_join - FROM - all_collected - WHERE - topics [0] :: STRING = '0x40d0efd1a53d60ecbf40971b9daf7dc90178c3aadc7aab1765632738fa8b8f01' -), -pool_data AS ( - SELECT - token0_address, - token1_address, - fee, - fee_percent, - tick_spacing, - pool_address, - token0_symbol, - token1_symbol, - token0_decimals, - token1_decimals, - pool_name - FROM - {{ ref('silver__univ3_pools') }} -), -silver_positions AS ( - SELECT - 'ethereum' AS blockchain, - b.block_number AS block_number, - b.block_timestamp AS block_timestamp, - b.tx_hash AS tx_hash, - b.event_index AS event_index, - b.contract_address AS pool_address, - b.origin_from_address AS liquidity_provider, - nf_token_id, - CASE - WHEN nf_token_id IS NULL THEN vault_address - ELSE nf_position_manager_address - END AS nf_position_manager_address, - token0_address, - token1_address, - b.amount0, - b.amount1, - b.tick_lower, - b.tick_upper, - _inserted_timestamp, - _log_id - FROM - collected_base b - LEFT JOIN nf_token_id_base - ON b.tx_hash = nf_token_id_base.tx_hash - AND b.event_index = nf_token_id_base.event_index_join - LEFT JOIN pool_data - ON b.contract_address = pool_data.pool_address -), -token_prices AS ( - SELECT - HOUR, - LOWER(token_address) AS token_address, - price - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - HOUR :: DATE IN ( - SELECT - DISTINCT block_timestamp :: DATE - FROM - silver_positions - ) -) -SELECT - blockchain, - block_number, - block_timestamp, - tx_hash, - event_index, - A.pool_address, - pool_name, - liquidity_provider, - nf_token_id, - nf_position_manager_address, - token0_symbol, - token1_symbol, - amount0, - amount1, - amount0 / pow( - 10, - token0_decimals - ) AS amount0_adjusted, - amount1 / pow( - 10, - token1_decimals - ) AS amount1_adjusted, - ROUND( - amount0_adjusted * p0.price, - 2 - ) AS amount0_usd, - ROUND( - amount1_adjusted * p1.price, - 2 - ) AS amount1_usd, - tick_lower, - tick_upper, - pow( - 1.0001, - tick_lower - ) / pow(10,(token1_decimals - token0_decimals)) AS price_lower, - pow( - 1.0001, - tick_upper - ) / pow(10,(token1_decimals - token0_decimals)) AS price_upper, - ROUND( - price_lower * p1.price, - 2 - ) AS price_lower_usd, - ROUND( - price_upper * p1.price, - 2 - ) AS price_upper_usd, - _log_id, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'event_index'] - ) }} AS univ3_position_collected_fees_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - silver_positions A - LEFT JOIN pool_data p - ON A.pool_address = p.pool_address - LEFT JOIN token_prices p0 - ON p0.token_address = A.token0_address - AND p0.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) - LEFT JOIN token_prices p1 - ON p1.token_address = A.token1_address - AND p1.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) diff --git a/models/silver/protocols/uniswapv3/silver__univ3_position_collected_fees.yml b/models/silver/protocols/uniswapv3/silver__univ3_position_collected_fees.yml deleted file mode 100644 index 4cc60c4a..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_position_collected_fees.yml +++ /dev/null @@ -1,40 +0,0 @@ -version: 2 -models: - - name: silver__univ3_position_collected_fees - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - _LOG_ID - columns: - - name: BLOCKCHAIN - tests: - - not_null - - name: BLOCK_NUMBER - tests: - - not_null - - name: BLOCK_TIMESTAMP - tests: - - not_null - - name: TX_HASH - tests: - - not_null - - name: EVENT_INDEX - tests: - - not_null - - name: POOL_ADDRESS - tests: - - not_null - - name: LIQUIDITY_PROVIDER - tests: - - not_null - - name: AMOUNT0 - tests: - - not_null - - name: AMOUNT1 - tests: - - not_null - - name: _LOG_ID - tests: - - not_null - - diff --git a/models/silver/protocols/uniswapv3/silver__univ3_positions.sql b/models/silver/protocols/uniswapv3/silver__univ3_positions.sql deleted file mode 100644 index 84f5e715..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_positions.sql +++ /dev/null @@ -1,265 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = "block_number", - cluster_by = ['block_timestamp::date'], - tags = ['silver','curated','uniswap'] -) }} - -WITH lp_events AS ( - - SELECT - blockchain, - block_number, - block_timestamp, - tx_hash, - liquidity_provider, - nf_position_manager_address, - nf_token_id, - pool_address, - tick_upper, - tick_lower, - token0_address, - token1_address, - _log_id, - _inserted_timestamp, - event_index - FROM - {{ ref('silver__univ3_lp_actions') }} - WHERE - nf_token_id IS NOT NULL - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} -), -position_reads_base AS ( - SELECT - contract_address, - block_number, - utils.udf_hex_to_int(CONCAT('0x', function_input :: STRING)) :: STRING AS nf_token_id, - regexp_substr_all(SUBSTR(read_output, 3, len(read_output)), '.{64}') AS segmented_data, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) :: FLOAT AS nonce, - CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 27, 40)) AS OPERATOR, - CONCAT('0x', SUBSTR(segmented_data [2] :: STRING, 27, 40)) AS token0, - CONCAT('0x', SUBSTR(segmented_data [3] :: STRING, 27, 40)) AS token1, - utils.udf_hex_to_int( - segmented_data [4] :: STRING - ) :: FLOAT AS fee, - utils.udf_hex_to_int( - 's2c', - segmented_data [5] :: STRING - ) :: FLOAT AS tickLower, - utils.udf_hex_to_int( - 's2c', - segmented_data [6] :: STRING - ) :: FLOAT AS tickUpper, - utils.udf_hex_to_int( - segmented_data [7] :: STRING - ) :: FLOAT AS liquidity, - utils.udf_hex_to_int( - segmented_data [8] :: STRING - ) :: FLOAT AS feeGrowthInside0LastX128, - utils.udf_hex_to_int( - segmented_data [9] :: STRING - ) :: FLOAT AS feeGrowthInside1LastX128, - utils.udf_hex_to_int( - segmented_data [10] :: STRING - ) :: FLOAT AS tokensOwed0, - utils.udf_hex_to_int( - segmented_data [11] :: STRING - ) :: FLOAT AS tokensOwed1 - FROM - {{ ref('silver__reads') }} - WHERE - function_signature = '0x99fbab88' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} -), -pool_data AS ( - SELECT - token0_address, - token1_address, - fee_percent, - tick_spacing, - pool_address, - token0_symbol, - token1_symbol, - token0_decimals, - token1_decimals, - pool_name - FROM - {{ ref('silver__univ3_pools') }} -), -FINAL AS ( - SELECT - blockchain, - A.block_number, - block_timestamp, - tx_hash, - COALESCE( - fee, - 0 - ) AS fee, - COALESCE( - fee, - 0 - ) / 10000 AS fee_percent, - COALESCE( - feeGrowthInside0LastX128, - 0 - ) AS fee_growth_inside0_last_x128, - COALESCE( - feeGrowthInside1LastX128, - 0 - ) AS fee_growth_inside1_last_x128, - CASE - WHEN fee_percent <> 0 THEN TRUE - ELSE FALSE - END AS is_active, - liquidity, - liquidity_provider, - nf_position_manager_address, - A.nf_token_id, - A.pool_address, - COALESCE( - tick_upper, - 0 - ) AS tick_upper, - COALESCE( - tick_lower, - 0 - ) AS tick_lower, - tokensOwed0, - tokensOwed1, - A.token0_address, - A.token1_address, - _log_id, - _inserted_timestamp, - event_index - FROM - lp_events A - LEFT JOIN position_reads_base b - ON A.block_number = b.block_number - AND A.nf_token_id :: STRING = b.nf_token_id :: STRING - LEFT JOIN pool_data C - ON A.pool_address = C.pool_address -), -silver_positions AS ( - SELECT - * - FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY _log_id - ORDER BY - _inserted_timestamp DESC)) = 1 -), -token_prices AS ( - SELECT - HOUR, - LOWER(token_address) AS token_address, - price - FROM - {{ ref('price__ez_prices_hourly') }} - WHERE - HOUR :: DATE IN ( - SELECT - DISTINCT block_timestamp :: DATE - FROM - silver_positions - ) -) -SELECT - blockchain, - block_number, - block_timestamp, - tx_hash, - A.fee_percent, - fee_growth_inside0_last_x128, - fee_growth_inside1_last_x128, - is_active, - COALESCE( - liquidity / pow(10, (token1_decimals + token0_decimals) / 2), - 0 - ) AS liquidity_adjusted, - liquidity_provider, - nf_position_manager_address, - nf_token_id, - A.pool_address, - pool_name, - A.tick_upper, - A.tick_lower, - pow(1.0001, (tick_lower)) / pow(10,(token1_decimals - token0_decimals)) AS price_lower_1_0, - pow(1.0001, (tick_upper)) / pow(10,(token1_decimals - token0_decimals)) AS price_upper_1_0, - pow(1.0001, -1 * (tick_upper)) / pow(10,(token0_decimals - token1_decimals)) AS price_lower_0_1, - pow(1.0001, -1 * (tick_lower)) / pow(10,(token0_decimals - token1_decimals)) AS price_upper_0_1, - price_lower_1_0 * p1.price AS price_lower_1_0_usd, - price_upper_1_0 * p1.price AS price_upper_1_0_usd, - price_lower_0_1 * p0.price AS price_lower_0_1_usd, - price_upper_0_1 * p0.price AS price_upper_0_1_usd, - COALESCE( - tokensOwed0 / pow( - 10, - token0_decimals - ), - 0 - ) AS tokens_owed0_adjusted, - COALESCE( - tokensOwed1 / pow( - 10, - token1_decimals - ), - 0 - ) AS tokens_owed1_adjusted, - COALESCE( - tokens_owed0_adjusted * p0.price, - 0 - ) AS tokens_owed0_usd, - COALESCE( - tokens_owed1_adjusted * p1.price, - 0 - ) AS tokens_owed1_usd, - A.token0_address, - A.token1_address, - token0_symbol, - token1_symbol, - event_index, - _log_id, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'event_index'] - ) }} AS univ3_positions_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - silver_positions A - LEFT JOIN pool_data p - ON A.pool_address = p.pool_address - LEFT JOIN token_prices p0 - ON p0.token_address = A.token0_address - AND p0.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) - LEFT JOIN token_prices p1 - ON p1.token_address = A.token1_address - AND p1.hour = DATE_TRUNC( - 'hour', - block_timestamp - ) diff --git a/models/silver/protocols/uniswapv3/silver__univ3_positions.yml b/models/silver/protocols/uniswapv3/silver__univ3_positions.yml deleted file mode 100644 index a3429c24..00000000 --- a/models/silver/protocols/uniswapv3/silver__univ3_positions.yml +++ /dev/null @@ -1,69 +0,0 @@ -version: 2 -models: - - name: silver__univ3_positions - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - _LOG_ID - columns: - - name: BLOCKCHAIN - tests: - - not_null - - name: BLOCK_NUMBER - tests: - - not_null - - name: BLOCK_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - name: FEE_PERCENT - tests: - - not_null: - enabled: True - - name: FEE_GROWTH_INSIDE0_LAST_x128 - tests: - - not_null: - enabled: True - - name: LIQUIDITY_ADJUSTED - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - decimal - - float - - name: FEE_GROWTH_INSIDE1_LAST_x128 - tests: - - not_null: - enabled: True - - name: IS_ACTIVE - tests: - - not_null - - name: LIQUIDITY_PROVIDER - tests: - - not_null - - name: NF_POSITION_MANAGER_ADDRESS - tests: - - not_null - - name: TOKENS_OWED0_USD - tests: - - not_null - - name: TOKENS_OWED1_USD - tests: - - not_null - - name: NF_TOKEN_ID - tests: - - not_null - - name: POOL_ADDRESS - tests: - - not_null - - name: TICK_LOWER - tests: - - not_null - - name: TICK_UPPER - tests: - - not_null - - name: TX_HASH - tests: - - not_null \ No newline at end of file diff --git a/models/silver/reads/silver__reads.sql b/models/silver/reads/silver__reads.sql deleted file mode 100644 index be87f116..00000000 --- a/models/silver/reads/silver__reads.sql +++ /dev/null @@ -1,56 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_reads') }} -{{ config( - materialized = 'incremental', - unique_key = 'reads_id', - cluster_by = ['_inserted_timestamp::date', 'function_signature'], - incremental_predicates = ["dynamic_range", "block_number"], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - merge_exclude_columns = ["inserted_timestamp"], - tags = ['silver','reads'] -) }} - -SELECT - COALESCE( - VALUE :"CONTRACT_ADDRESS" :: STRING, - VALUE :"contract_address" :: STRING - ) AS contract_address, - block_number, - COALESCE( - VALUE :"FUNCTION_SIGNATURE" :: STRING, - VALUE :"function_signature" :: STRING - ) AS function_signature, - COALESCE( - VALUE :"CALL_NAME" :: STRING, - VALUE :"call_name" :: STRING - ) AS call_name, - DATA :result :: STRING AS read_output, - COALESCE( - VALUE :"FUNCTION_INPUT" :: STRING, - VALUE :"function_input" :: STRING - ) AS function_input, - regexp_substr_all(SUBSTR(read_output, 3, len(read_output)), '.{64}') AS segmented_data, - TO_TIMESTAMP_NTZ(_inserted_timestamp) AS _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['contract_address', 'function_signature', 'call_name', 'function_input', 'block_number'] - ) }} AS id, - id AS reads_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - -{% if is_incremental() %} -{{ ref('bronze__streamline_reads') }} -WHERE - TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( - SELECT - DATEADD('hour', -3, MAX(_inserted_timestamp)) - FROM - {{ this }}) - {% else %} - {{ ref('bronze__streamline_fr_reads') }} - {% endif %} - - qualify(ROW_NUMBER() over (PARTITION BY id - ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/silver/reads/tests/test_silver__reads_full.sql b/models/silver/reads/tests/test_silver__reads_full.sql deleted file mode 100644 index 8662caf3..00000000 --- a/models/silver/reads/tests/test_silver__reads_full.sql +++ /dev/null @@ -1,9 +0,0 @@ -{{ config ( - materialized = 'view', - tags = ['test_silver','reads','full_test'] -) }} - -SELECT - * -FROM - {{ ref('silver__reads') }} diff --git a/models/silver/reads/tests/test_silver__reads_full.yml b/models/silver/reads/tests/test_silver__reads_full.yml deleted file mode 100644 index 1439e68e..00000000 --- a/models/silver/reads/tests/test_silver__reads_full.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2 -models: - - name: test_silver__reads_full - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - ID - - columns: - - name: _INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ diff --git a/models/silver/reads/tests/test_silver__reads_recent.sql b/models/silver/reads/tests/test_silver__reads_recent.sql deleted file mode 100644 index 887ef666..00000000 --- a/models/silver/reads/tests/test_silver__reads_recent.sql +++ /dev/null @@ -1,27 +0,0 @@ -{{ config ( - materialized = 'view', - tags = ['test_silver','reads','recent_test'] -) }} - -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_max_block_by_date") }} - qualify ROW_NUMBER() over ( - ORDER BY - block_number DESC - ) = 3 -) -SELECT - * -FROM - {{ ref('silver__reads') }} -WHERE - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) diff --git a/models/silver/reads/tests/test_silver__reads_recent.yml b/models/silver/reads/tests/test_silver__reads_recent.yml deleted file mode 100644 index 8d4e8ce8..00000000 --- a/models/silver/reads/tests/test_silver__reads_recent.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2 -models: - - name: test_silver__reads_recent - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - ID - - columns: - - name: _INSERTED_TIMESTAMP - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 2 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ diff --git a/models/streamline/silver/reads/complete/streamline__complete_reads.sql b/models/streamline/silver/reads/complete/streamline__complete_reads.sql deleted file mode 100644 index 3c7464dc..00000000 --- a/models/streamline/silver/reads/complete/streamline__complete_reads.sql +++ /dev/null @@ -1,50 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_reads') }} -{{ config ( - materialized = "incremental", - unique_key = "complete_reads_id", - cluster_by = "ROUND(block_number, -3)", - incremental_predicates = ["dynamic_range", "block_number"], - merge_update_columns = ["complete_reads_id"], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(complete_reads_id)", - tags = ['streamline_reads_complete'] -) }} - -SELECT - COALESCE( - VALUE :"CONTRACT_ADDRESS" :: STRING, - VALUE :"contract_address" :: STRING - ) AS contract_address, - block_number, - COALESCE( - VALUE :"FUNCTION_SIGNATURE" :: STRING, - VALUE :"function_signature" :: STRING - ) AS function_signature, - COALESCE( - VALUE :"CALL_NAME" :: STRING, - VALUE :"call_name" :: STRING - ) AS call_name, - COALESCE( - VALUE :"FUNCTION_INPUT" :: STRING, - VALUE :"function_input" :: STRING - ) AS function_input, - {{ dbt_utils.generate_surrogate_key( - ['contract_address', 'function_signature', 'call_name', 'function_input', 'block_number'] - ) }} AS complete_reads_id, - TO_TIMESTAMP_LTZ(_inserted_timestamp) AS _inserted_timestamp -FROM - -{% if is_incremental() %} -{{ ref('bronze__streamline_reads') }} -WHERE - TO_TIMESTAMP_LTZ(_inserted_timestamp) >= ( - SELECT - DATEADD('hour', -3, MAX(_inserted_timestamp)) - FROM - {{ this }}) - {% else %} - {{ ref('bronze__streamline_fr_reads') }} - {% endif %} - - qualify(ROW_NUMBER() over (PARTITION BY complete_reads_id - ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/reads/curated/streamline__chainlink_prices.sql b/models/streamline/silver/reads/curated/streamline__chainlink_prices.sql deleted file mode 100644 index 71669efb..00000000 --- a/models/streamline/silver/reads/curated/streamline__chainlink_prices.sql +++ /dev/null @@ -1,64 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"], - tags = ['stale'] -) }} - -WITH price_feeds AS ( - - SELECT - feed_address, - created_block_number - FROM - {{ ref('silver__chainlink_feeds_seed') }} -), -block_range AS ( - -- edit this range to use a different block range from the ephemeral table - SELECT - block_number_50 AS block_input, - _inserted_timestamp - FROM - {{ ref('_block_ranges') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - FROM - {{ this }} - ) -{% endif %} -), -feeds_range AS ( - SELECT - feed_address AS contract_address, - block_input AS block_number, - _inserted_timestamp, - '0x50d25bcd' AS function_signature, - 0 AS function_input_plug - FROM - price_feeds - JOIN block_range - WHERE - block_input IS NOT NULL - AND block_input >= created_block_number -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'contract_address', 'function_signature', 'function_input_plug'] - ) }} AS id, - NULL AS function_input, - function_signature, - block_number, - contract_address, - 'chainlink_price_feed' AS call_name, - _inserted_timestamp -FROM - feeds_range qualify(ROW_NUMBER() over(PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/reads/curated/streamline__dex_reserve_reads.sql b/models/streamline/silver/reads/curated/streamline__dex_reserve_reads.sql deleted file mode 100644 index baf16720..00000000 --- a/models/streamline/silver/reads/curated/streamline__dex_reserve_reads.sql +++ /dev/null @@ -1,91 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"], - tags = ['streamline_reads_curated'], - enabled = false -) }} - -WITH base AS ( - - SELECT - block_number, - contract_address, - _inserted_timestamp, - CASE - WHEN platform IN ( - 'uniswap-v2', - 'sushiswap' - ) THEN '0x0902f1ac' - ELSE 'missing' - END AS function_signature, - NULL AS function_input, - 0 AS function_input_plug - FROM - {{ ref('silver_dex__v2_swaps') }} - WHERE - platform IN ( - 'uniswap-v2', - 'sushiswap' - ) - AND origin_to_address = '0x00000000003b3cc22af3ae1eac0440bcee416b40' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - FROM - {{ this }} -) -{% endif %} -), -FINAL AS ( - SELECT - block_number, - contract_address, - function_signature, - _inserted_timestamp, - function_input, - function_input_plug - FROM - base - UNION - SELECT - block_number + 1 AS block_number, - contract_address, - function_signature, - _inserted_timestamp, - function_input, - function_input_plug - FROM - base - UNION - SELECT - block_number - 1 AS block_number, - contract_address, - function_signature, - _inserted_timestamp, - function_input, - function_input_plug - FROM - base -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'contract_address', 'function_signature', 'function_input_plug'] - ) }} AS id, - function_input, - function_signature, - block_number, - contract_address, - 'dex_reserve_reads' AS call_name, - _inserted_timestamp -FROM - FINAL -WHERE - function_signature <> 'missing' qualify(ROW_NUMBER() over(PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/reads/curated/streamline__price_api_token_reads.sql b/models/streamline/silver/reads/curated/streamline__price_api_token_reads.sql deleted file mode 100644 index 21480a48..00000000 --- a/models/streamline/silver/reads/curated/streamline__price_api_token_reads.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['streamline_reads_curated'] -) }} - -WITH base AS ( - - SELECT - DISTINCT token_address - FROM - {{ ref( - 'silver__complete_token_asset_metadata' - ) }} - WHERE - provider = 'coingecko' -), -missing_contracts AS ( - SELECT - * - FROM - base - EXCEPT - SELECT - address AS token_address - FROM - {{ ref('core__dim_contracts') }} - WHERE - symbol IS NOT NULL - AND decimals IS NOT NULL -) -SELECT - token_address, - block_number -FROM - missing_contracts - JOIN {{ ref('core__fact_traces') }} - ON token_address = to_address - AND TYPE ILIKE 'create' diff --git a/models/streamline/silver/reads/curated/streamline__uni_v3_pool_reads.sql b/models/streamline/silver/reads/curated/streamline__uni_v3_pool_reads.sql deleted file mode 100644 index e015c2a1..00000000 --- a/models/streamline/silver/reads/curated/streamline__uni_v3_pool_reads.sql +++ /dev/null @@ -1,107 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"], - tags = ['streamline_reads_curated'] -) }} --- this model looks at the following function for uni v3 pools every 25 blocks --- TEXT_SIGNATURE BYTES_SIGNATURE ID --- protocolFees() 0x1ad8b03b 181768 --- liquidity() 0x1a686502 174769 --- feeGrowthGlobal1X128() 0x46141319 178714 --- slot0() 0x3850c7bd 178654 --- feeGrowthGlobal0X128() 0xf3058399 178713 -WITH created_pools AS ( - - SELECT - created_block, - pool_address AS contract_address - FROM - {{ ref('silver__univ3_pools') }} -), -block_range AS ( - -- edit this range to use a different block range from the ephemeral table - SELECT - block_number_1000 AS block_input, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('_block_ranges') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - FROM - {{ this }} - ) -{% endif %} -), -pool_block_range AS ( - -- this only includes blocks after the creation of each asset - SELECT - contract_address, - block_input, - _inserted_timestamp - FROM - created_pools - JOIN block_range - WHERE - block_input IS NOT NULL - AND block_input >= created_block -), -function_sigs AS ( - SELECT - 'protocolFees()' AS text_signature, - '0x1ad8b03b' AS bytes_signature, - 181768 AS id - UNION ALL - SELECT - 'liquidity()', - '0x1a686502', - 174769 - UNION ALL - SELECT - 'feeGrowthGlobal1X128()', - '0x46141319', - 178714 - UNION ALL - SELECT - 'slot0()', - '0x3850c7bd', - 178654 - UNION ALL - SELECT - 'feeGrowthGlobal0X128()', - '0xf3058399', - 178713 -), -FINAL AS ( - SELECT - contract_address, - block_input AS block_number, - bytes_signature AS function_signature, - NULL AS function_input, - 0 AS function_input_plug, - _inserted_timestamp - FROM - pool_block_range - JOIN function_sigs -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'contract_address', 'function_signature', 'function_input_plug'] - ) }} AS id, - function_input, - function_signature, - block_number, - contract_address, - 'uni_v3_pool_reads' AS call_name, - _inserted_timestamp -FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/reads/curated/streamline__uni_v3_position_reads.sql b/models/streamline/silver/reads/curated/streamline__uni_v3_position_reads.sql deleted file mode 100644 index fcc6aa81..00000000 --- a/models/streamline/silver/reads/curated/streamline__uni_v3_position_reads.sql +++ /dev/null @@ -1,69 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"], - tags = ['streamline_reads_curated'] -) }} --- this model looks at the position function for uni v3 when there has been a liquidity action --- 0x99fbab88 -WITH liquidity_actions AS ( - - SELECT - block_number, - contract_address, - udf_hex_to_int( - topics [1] :: STRING - ) AS nf_position_id, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__fact_event_logs') }} - WHERE - contract_address = '0xc36442b4a4522e871399cd717abdd847ab11fe88' - AND topics [0] :: STRING IN ( - '0x3067048beee31b25b2f1681f88dac838c8bba36af25bfb2b7cf7473a5847e35f', - '0x26f6a048ee9138f2c0ce266f322cb99228e8d619ae2bff30c67f8dcf9d2377b4', - '0x40d0efd1a53d60ecbf40971b9daf7dc90178c3aadc7aab1765632738fa8b8f01' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX( - _inserted_timestamp - ) - FROM - {{ this }} -) -{% endif %} -), -FINAL AS ( - SELECT - block_number, - '0xc36442b4a4522e871399cd717abdd847ab11fe88' AS contract_address, - -- proxy 0xc36442b4a4522e871399cd717abdd847ab11fe88 - '0x99fbab88' AS function_signature, - TRIM( - to_char( - nf_position_id :: INTEGER, - 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' - ) - ) AS function_input, - _inserted_timestamp - FROM - liquidity_actions -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'contract_address', 'function_signature', 'function_input'] - ) }} AS id, - function_input, - function_signature, - block_number, - contract_address, - 'uni_v3_position_reads' AS call_name, - _inserted_timestamp -FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/reads/curated/streamline__uni_v3_token_reads.sql b/models/streamline/silver/reads/curated/streamline__uni_v3_token_reads.sql deleted file mode 100644 index 5a33cb62..00000000 --- a/models/streamline/silver/reads/curated/streamline__uni_v3_token_reads.sql +++ /dev/null @@ -1,90 +0,0 @@ -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["id"], - tags = ['streamline_reads_curated'] -) }} --- this looks at the decimals(), name(), and symbol() functions for all tokens used ina univ3 pool --- it will only look at each token once --- functions: decimals(0x313ce567), name(0x06fdde03), and symbol(0x95d89b41) -WITH all_tokens AS ( - - SELECT - created_block AS block_number, - token0_address AS contract_address - FROM - {{ ref('silver__univ3_pools') }} - UNION ALL - SELECT - created_block AS block_number, - token1_address AS contract_address - FROM - {{ ref('silver__univ3_pools') }} - UNION ALL - SELECT - block_number, - token_address AS contract_address - FROM - {{ ref('streamline__price_api_token_reads') }} -), -min_block AS ( - SELECT - contract_address, - MIN(block_number) + 1 AS block_number - FROM - all_tokens - -{% if is_incremental() %} -WHERE - contract_address NOT IN ( - SELECT - DISTINCT contract_address - FROM - {{ this }} - ) -{% endif %} -GROUP BY - contract_address -), -function_sigs AS ( - SELECT - 'decimals()' AS text_signature, - '0x313ce567' AS bytes_signature, - 4821 AS id - UNION ALL - SELECT - 'name()' AS text_signature, - '0x06fdde03' AS bytes_signature, - 4820 AS id - UNION ALL - SELECT - 'symbol()' AS text_signature, - '0x95d89b41' AS bytes_signature, - 4827 AS id -), -FINAL AS ( - SELECT - block_number, - contract_address, - bytes_signature AS function_signature, - NULL AS function_input, - 0 AS function_input_plug - FROM - min_block - JOIN function_sigs -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'contract_address', 'function_signature', 'function_input_plug'] - ) }} AS id, - function_input, - function_signature, - block_number, - contract_address, - 'uni_v3_token_reads' AS call_name, - SYSDATE() AS _inserted_timestamp -FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/reads/history/streamline__reads_history.sql b/models/streamline/silver/reads/history/streamline__reads_history.sql deleted file mode 100644 index a1e91c3e..00000000 --- a/models/streamline/silver/reads/history/streamline__reads_history.sql +++ /dev/null @@ -1,90 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"reads_v2", - "sql_limit" :"15000000", - "producer_batch_size" :"15000000", - "worker_batch_size" :"2500000", - "sql_source" :"{{this.identifier}}" } - ), - tags = ['streamline_reads_history'] -) }} - -WITH to_do AS ({% for item in range(17) %} - ( - - SELECT - contract_address, function_signature, call_name, function_input, block_number - FROM - {{ ref("streamline__contract_reads") }} - WHERE - block_number BETWEEN {{ item * 1000000 + 1 }} - AND {{(item + 1) * 1000000 }} - EXCEPT - SELECT - contract_address, function_signature, call_name, function_input, block_number - FROM - {{ ref("streamline__complete_reads") }} - WHERE - block_number BETWEEN {{ item * 1000000 + 1 }} - AND {{(item + 1) * 1000000 }} - ORDER BY - block_number) {% if not loop.last %} - UNION ALL - {% endif %} - {% endfor %}) -SELECT - contract_address, - function_signature, - call_name, - function_input, - block_number, - CASE - WHEN function_input IS NULL THEN function_signature - WHEN function_input ILIKE '0x%' THEN CONCAT( - function_signature, - LPAD(SUBSTR(function_input, 3), 64, 0) - ) - ELSE CONCAT( - function_signature, - LPAD( - function_input, - 64, - 0 - ) - ) - END AS DATA, - CONCAT((SYSDATE() :: DATE) :: STRING, '_', function_signature) AS partition_key, - {{ target.database }}.live.udf_api( - 'POST', - '{service}/{Authentication}', - OBJECT_CONSTRUCT( - 'Content-Type', - 'application/json' - ), - OBJECT_CONSTRUCT( - 'id', - block_number, - 'jsonrpc', - '2.0', - 'method', - 'eth_call', - 'params', - ARRAY_CONSTRUCT( - OBJECT_CONSTRUCT( - 'to', - contract_address, - 'data', - DATA - ), - utils.udf_int_to_hex(block_number) - ) - ), - 'vault/prod/ethereum/quicknode/mainnet' - ) AS request -FROM - to_do -ORDER BY - block_number DESC \ No newline at end of file diff --git a/models/streamline/silver/reads/realtime/streamline__reads_realtime.sql b/models/streamline/silver/reads/realtime/streamline__reads_realtime.sql deleted file mode 100644 index 321a2a4f..00000000 --- a/models/streamline/silver/reads/realtime/streamline__reads_realtime.sql +++ /dev/null @@ -1,114 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = fsc_utils.if_data_call_function_v2( - func = 'streamline.udf_bulk_rest_api_v2', - target = "{{this.schema}}.{{this.identifier}}", - params ={ "external_table" :"reads_v2", - "sql_limit" :"15000000", - "producer_batch_size" :"15000000", - "worker_batch_size" :"2500000", - "sql_source" :"{{this.identifier}}" } - ), - tags = ['streamline_reads_realtime'] -) }} - -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_max_block_by_date") }} - qualify ROW_NUMBER() over ( - ORDER BY - block_number DESC - ) = 3 -), -to_do AS ( - SELECT - contract_address, - function_signature, - call_name, - function_input, - block_number - FROM - {{ ref("streamline__contract_reads") }} - WHERE - ( - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) - AND block_number IS NOT NULL - EXCEPT - SELECT - contract_address, - function_signature, - call_name, - function_input, - block_number - FROM - {{ ref("streamline__complete_reads") }} - WHERE - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) -) -SELECT - contract_address, - function_signature, - call_name, - function_input, - block_number, - CASE - WHEN function_input IS NULL THEN function_signature - WHEN function_input ILIKE '0x%' THEN CONCAT( - function_signature, - LPAD(SUBSTR(function_input, 3), 64, 0) - ) - ELSE CONCAT( - function_signature, - LPAD( - function_input, - 64, - 0 - ) - ) - END AS DATA, - CONCAT((SYSDATE() :: DATE) :: STRING, '_', function_signature) AS partition_key, - {{ target.database }}.live.udf_api( - 'POST', - '{service}/{Authentication}', - OBJECT_CONSTRUCT( - 'Content-Type', - 'application/json' - ), - OBJECT_CONSTRUCT( - 'id', - block_number, - 'jsonrpc', - '2.0', - 'method', - 'eth_call', - 'params', - ARRAY_CONSTRUCT( - OBJECT_CONSTRUCT( - 'to', - contract_address, - 'data', - DATA - ), - utils.udf_int_to_hex(block_number) - ) - ), - 'vault/prod/ethereum/quicknode/mainnet' - ) AS request -FROM - to_do -ORDER BY - block_number DESC diff --git a/models/streamline/silver/reads/streamline__contract_reads.sql b/models/streamline/silver/reads/streamline__contract_reads.sql deleted file mode 100644 index f23ba2a0..00000000 --- a/models/streamline/silver/reads/streamline__contract_reads.sql +++ /dev/null @@ -1,36 +0,0 @@ -{{ config( - materialized = 'view' -) }} - -SELECT - id :: STRING AS id, - block_number :: INTEGER AS block_number, - contract_address :: STRING AS contract_address, - call_name :: STRING AS call_name, - function_signature :: STRING AS function_signature, - function_input :: STRING AS function_input, - _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp -FROM - {{ ref('streamline__uni_v3_pool_reads') }} -UNION ALL -SELECT - id :: STRING AS id, - block_number :: INTEGER AS block_number, - contract_address :: STRING AS contract_address, - call_name :: STRING AS call_name, - function_signature :: STRING AS function_signature, - function_input :: STRING AS function_input, - _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp -FROM - {{ ref('streamline__uni_v3_position_reads') }} -UNION ALL -SELECT - id :: STRING AS id, - block_number :: INTEGER AS block_number, - contract_address :: STRING AS contract_address, - call_name :: STRING AS call_name, - function_signature :: STRING AS function_signature, - function_input :: STRING AS function_input, - _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp -FROM - {{ ref('streamline__uni_v3_token_reads') }} \ No newline at end of file