diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a3b4500 --- /dev/null +++ b/Makefile @@ -0,0 +1,46 @@ +DBT_TARGET ?= dev + +deploy_streamline_functions: + rm -f package-lock.yml && dbt clean && dbt deps + dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) + dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) + +cleanup_time: + rm -f package-lock.yml && dbt clean && dbt deps + +deploy_streamline_tables: + rm -f package-lock.yml && dbt clean && dbt deps +ifeq ($(findstring dev,$(DBT_TARGET)),dev) + dbt run -m "fsc_evm,tag:bronze_external" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET) +else + dbt run -m "fsc_evm,tag:bronze_external" -t $(DBT_TARGET) +endif + dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET) + +deploy_streamline_requests: + rm -f package-lock.yml && dbt clean && dbt deps + dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) + +deploy_github_actions: + dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) + dbt seed -s github_actions__workflows -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) +ifeq ($(findstring dev,$(DBT_TARGET)),dev) + dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) +else + dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) +endif + +deploy_new_github_action: + dbt seed -s github_actions__workflows -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) +ifeq ($(findstring dev,$(DBT_TARGET)),dev) + dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) +else + dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) +endif + +regular_incremental: + dbt run -m "fsc_evm,tag:core" -t $(DBT_TARGET) + +.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time regular_incremental deploy_new_github_action \ No newline at end of file diff --git a/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql b/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql index 29f4ef3..cd6e573 100644 --- a/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql +++ b/models/silver/defi/bridge/allbridge/silver_bridge__allbridge_sent.sql @@ -46,8 +46,12 @@ WITH base_evt AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -141,7 +145,7 @@ SELECT END AS destination_chain_receiver, tokenSourceAddress AS token_address, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt WHERE diff --git a/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql b/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql index 782cc4e..8a21f12 100644 --- a/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql +++ b/models/silver/defi/bridge/axelar/silver_bridge__axelar_contractcallwithtoken.sql @@ -35,8 +35,12 @@ WITH base_evt AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -84,8 +88,12 @@ native_gas_paid AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -109,8 +117,12 @@ transfers AS ( tx_hash, event_index, contract_address AS token_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__transfers') }} WHERE diff --git a/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql b/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql index b006048..1eb3c37 100644 --- a/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql +++ b/models/silver/defi/bridge/celer/silver_bridge__celer_cbridge_send.sql @@ -39,8 +39,12 @@ WITH base_evt AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -87,6 +91,6 @@ SELECT token AS token_address, transferId AS transfer_id, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt diff --git a/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql b/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql index 124f611..03719b8 100644 --- a/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql +++ b/models/silver/defi/bridge/dln/silver_bridge__dln_debridge_createdorder.sql @@ -63,8 +63,12 @@ WITH base_evt AS ( DATA, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, CONCAT('0x', SUBSTR(segmented_data [24] :: STRING, 1, 40)) AS token_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -112,6 +116,6 @@ SELECT decoded_flat, order_obj, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt diff --git a/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql b/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql index 269d6da..74812ba 100644 --- a/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql +++ b/models/silver/defi/bridge/eywa/silver_bridge__eywa_requestsent.sql @@ -23,8 +23,12 @@ WITH base_evt AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -73,7 +77,7 @@ requestsent AS ( event_removed, tx_status, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt WHERE @@ -102,7 +106,7 @@ locked AS ( event_removed, tx_status, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt WHERE @@ -129,7 +133,7 @@ SELECT r.chainIdTo AS destination_chain_id, l.token AS token_address, _log_id, - _inserted_timestamp + modified_timestamp FROM requestsent r LEFT JOIN locked l USING( diff --git a/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql b/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql index 36d0578..4d65667 100644 --- a/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql +++ b/models/silver/defi/bridge/meson/silver_bridge__meson_transfers.sql @@ -20,8 +20,12 @@ WITH token_transfers AS ( from_address, to_address, raw_amount, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__transfers') }} WHERE @@ -120,8 +124,12 @@ dst_info AS ( tx_hash, topics [1] :: STRING AS encoded_data, SUBSTR(RIGHT(encoded_data, 12), 1, 4) AS destination_chain_id, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql index e177128..308af4f 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_swap.sql @@ -57,8 +57,12 @@ base_evt AS ( token_address, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} d @@ -105,7 +109,7 @@ SELECT protocolFee AS protocol_fee, token_address, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt b LEFT JOIN {{ ref('silver_bridge__stargate_chain_id_seed') }} diff --git a/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql b/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql index b1001a4..9fca14e 100644 --- a/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql +++ b/models/silver/defi/bridge/symbiosis/silver_bridge__symbiosis_synthesizerequest.sql @@ -34,8 +34,12 @@ WITH base_evt AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -79,6 +83,6 @@ SELECT revertableAddress AS revertable_address, token AS token_address, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt diff --git a/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql b/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql index 9d04164..2c3baa2 100644 --- a/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql +++ b/models/silver/defi/bridge/synapse/silver_bridge__synapse_token_bridge.sql @@ -31,8 +31,12 @@ WITH base_evt AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -79,6 +83,6 @@ SELECT chainId AS destination_chain_id, token AS token_address, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt diff --git a/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql b/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql index 5be8825..59f6d79 100644 --- a/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql +++ b/models/silver/defi/bridge/synapse/silver_bridge__synapse_tokenbridgeandswap.sql @@ -43,8 +43,12 @@ WITH base_evt AS ( decoded_flat, event_removed, tx_status, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__decoded_logs') }} WHERE @@ -90,6 +94,6 @@ SELECT tokenIndexFrom AS token_index_from, tokenIndexTo AS token_index_to, _log_id, - _inserted_timestamp + modified_timestamp FROM base_evt diff --git a/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql b/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql index 2f00d6c..905a375 100644 --- a/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql +++ b/models/silver/defi/dex/biswap/silver_dex__biswap_pools.sql @@ -20,8 +20,12 @@ WITH pool_creation AS ( utils.udf_hex_to_int( segmented_data [1] :: STRING ) :: INTEGER AS pool_id, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -50,7 +54,7 @@ SELECT pool_address, pool_id, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/biswap/silver_dex__biswap_swaps.sql b/models/silver/defi/dex/biswap/silver_dex__biswap_swaps.sql index 1447cd1..0fea19e 100644 --- a/models/silver/defi/dex/biswap/silver_dex__biswap_swaps.sql +++ b/models/silver/defi/dex/biswap/silver_dex__biswap_swaps.sql @@ -50,8 +50,12 @@ swaps_base AS ( CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS tx_to, token0, token1, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} l @@ -119,7 +123,7 @@ SELECT 'Swap' AS event_name, 'biswap' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql index 71f0a4c..af5f960 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_pools.sql @@ -17,8 +17,12 @@ WITH pool_events AS ( CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS newBorn, CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 25, 40)) AS baseToken, CONCAT('0x', SUBSTR(segmented_data [2] :: STRING, 25, 40)) AS quoteToken, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql index abab57a..9c2977b 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v1_swaps.sql @@ -56,8 +56,12 @@ sell_base_token AS ( base_token AS tokenOut, receiveQuote AS amountIn, payBase AS amountOut, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -111,8 +115,12 @@ buy_base_token AS ( base_token AS tokenOut, payQuote AS amountIn, receiveBase AS amountOut, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -156,7 +164,7 @@ SELECT 'SellBaseToken' AS event_name, 'dodo-v1' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM sell_base_token UNION ALL @@ -178,6 +186,6 @@ SELECT 'BuyBaseToken' AS event_name, 'dodo-v1' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM buy_base_token diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql index 7ad7c89..96d3925 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_pools.sql @@ -18,8 +18,12 @@ WITH pools AS ( CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 25, 40)) AS quoteToken, CONCAT('0x', SUBSTR(segmented_data [2] :: STRING, 25, 40)) AS creator, CONCAT('0x', SUBSTR(segmented_data [3] :: STRING, 25, 40)) AS pool_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -62,7 +66,7 @@ SELECT creator, pool_address, _log_id, - _inserted_timestamp + modified_timestamp FROM pools qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql index 6138c83..695d71b 100644 --- a/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql +++ b/models/silver/defi/dex/dodo/silver_dex__dodo_v2_swaps.sql @@ -79,8 +79,12 @@ swaps_base AS ( 40 ) ) AS receiver_address, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -124,6 +128,6 @@ SELECT 'DodoSwap' AS event_name, 'dodo-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base diff --git a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql index ca48445..1cfafba 100644 --- a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql +++ b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_pools.sql @@ -20,8 +20,12 @@ WITH pool_creation AS ( utils.udf_hex_to_int( segmented_data [1] :: STRING ) :: INTEGER AS pool_id, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -54,7 +58,7 @@ SELECT pool_address, pool_id, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql index 1f53ac7..de8335b 100644 --- a/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql +++ b/models/silver/defi/dex/fraxswap/silver_dex__fraxswap_swaps.sql @@ -50,8 +50,12 @@ swaps_base AS ( ) AS amount1Out, token0, token1, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -113,7 +117,7 @@ SELECT 'Swap' AS event_name, 'fraxswap' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE token_in <> token_out \ No newline at end of file diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql index 2ca4adc..7110f41 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_swaps.sql @@ -37,8 +37,12 @@ router_swaps_base AS ( segmented_data [6] :: STRING ) ) AS amountOut, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -82,8 +86,12 @@ swaps_base AS ( segmented_data [5] :: STRING ) ) AS amountOut, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -122,7 +130,7 @@ FINAL AS ( 'Swap' AS event_name, 'hashflow' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM router_swaps_base UNION ALL @@ -144,7 +152,7 @@ FINAL AS ( 'Swap' AS event_name, 'hashflow' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base ) @@ -172,6 +180,6 @@ SELECT event_name, platform, _log_id, - _inserted_timestamp + modified_timestamp FROM FINAL diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql index 4451bd3..0952dbc 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_pools.sql @@ -14,8 +14,12 @@ WITH contract_deployments AS ( regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, origin_from_address AS deployer_address, CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS pool_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -40,7 +44,7 @@ SELECT deployer_address, pool_address, _log_id, - _inserted_timestamp + modified_timestamp FROM contract_deployments qualify(ROW_NUMBER() OVER (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql index 635e879..cb55bba 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_v3_swaps.sql @@ -42,8 +42,12 @@ swaps AS ( segmented_data [6] :: STRING ) ) AS amountOut, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} l @@ -88,6 +92,6 @@ SELECT 'Trade' AS event_name, 'hashflow-v3' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql index 9bc9834..850320b 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql @@ -28,7 +28,7 @@ WITH pool_creation AS ( ) ) AS totalPool, _log_id, - _inserted_timestamp + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -58,7 +58,7 @@ SELECT ampBps AS amp_bps, totalPool AS total_pool, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql index 05bf829..bc340bb 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql @@ -55,8 +55,12 @@ swaps_base AS ( ) AS feeInPrecision, token0, token1, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -119,7 +123,7 @@ SELECT 'Dynamic Swap' AS event_name, 'kyberswap-v1' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql index 8acdb26..ebba448 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql @@ -32,8 +32,12 @@ WITH pool_creation AS ( segmented_data [3] :: STRING ) ) AS totalPool, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -64,7 +68,7 @@ SELECT feeUnits AS fee_units, totalPool AS total_pool, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql index 47a252a..2e5fe13 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql @@ -55,8 +55,12 @@ swaps_base AS ( ) AS feeInPrecision, token0, token1, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -119,7 +123,7 @@ SELECT 'Static Swap' AS event_name, 'kyberswap-v1' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql index f11dad5..a031c92 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql @@ -23,8 +23,12 @@ WITH pool_creation AS ( ) ) AS tickDistance, CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 25, 40)) AS pool_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -54,7 +58,7 @@ SELECT tickDistance AS tick_distance, pool_address, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql index ed3284c..6122b73 100644 --- a/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql +++ b/models/silver/defi/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql @@ -68,8 +68,12 @@ swaps_base AS ( WHEN deltaQty0 > 0 THEN token0 ELSE token1 END AS token_out, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -114,7 +118,7 @@ SELECT 'Elastic Swap' AS event_name, 'kyberswap-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE diff --git a/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql b/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql index b9a61f8..a15032a 100644 --- a/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql +++ b/models/silver/defi/dex/level/silver_dex__levelfi_swaps.sql @@ -66,8 +66,12 @@ WITH swaps_base AS ( ) ) END AS fee, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -105,6 +109,6 @@ SELECT 'Swap' AS event_name, 'level-finance' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql index 8e4dc57..78d95d2 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_pools.sql @@ -20,8 +20,12 @@ WITH pool_creation AS ( utils.udf_hex_to_int( segmented_data [1] :: STRING ) :: INTEGER AS pool_id, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -50,7 +54,7 @@ SELECT pool_address, pool_id, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_swaps.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_swaps.sql index 1b33be5..575d4ac 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_swaps.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v1_swaps.sql @@ -50,8 +50,12 @@ swaps_base AS ( CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS tx_to, token0, token1, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} INNER JOIN pools p @@ -112,7 +116,7 @@ SELECT 'Swap' AS event_name, 'pancakeswap-v1' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql index be3b6a5..a38bccb 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_pools.sql @@ -20,8 +20,12 @@ WITH pool_creation AS ( utils.udf_hex_to_int( segmented_data [1] :: STRING ) :: INTEGER AS pool_id, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -54,7 +58,7 @@ SELECT pool_address, pool_id, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_swaps.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_swaps.sql index 4de2ce2..f68b3d4 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_swaps.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_amm_swaps.sql @@ -50,8 +50,12 @@ swaps_base AS ( CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS tx_to, token0, token1, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} INNER JOIN pools p @@ -112,7 +116,7 @@ SELECT 'Swap' AS event_name, 'pancakeswap-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql index 4f930be..236ecbe 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_mm_swaps.sql @@ -50,8 +50,12 @@ WITH swaps_base AS ( quoteToken AS token_out, baseTokenAmount AS token_in_amount, quoteTokenAmount AS token_out_amount, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -95,6 +99,6 @@ SELECT 'Swap' AS event_name, 'pancakeswap-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql index 2626544..26ad024 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_pools.sql @@ -31,8 +31,12 @@ WITH pools AS ( WHEN contract_address = '0x36bbb126e75351c0dfb651e39b38fe0bc436ffd2' THEN NULL WHEN contract_address = '0x25a55f9f2279a54951133d503490342b50e5cd15' THEN CONCAT('0x', SUBSTR(segmented_data [3] :: STRING, 25, 40)) END AS lp, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref ('silver__logs') }} WHERE @@ -68,7 +72,7 @@ SELECT tokenC, lp, _log_id, - _inserted_timestamp + modified_timestamp FROM pools qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_swaps.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_swaps.sql index d782110..dced15a 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_swaps.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v2_ss_swaps.sql @@ -57,8 +57,12 @@ swaps_base AS ( WHEN bought_id = 0 THEN tokenB ELSE tokenA END AS tokenIn, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} INNER JOIN pools p @@ -99,6 +103,6 @@ SELECT 'TokenExchange' AS event_name, 'pancakeswap-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql index 77a3b54..f4914b4 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_pools.sql @@ -29,8 +29,12 @@ WITH created_pools AS ( ) ) AS tickSpacing, CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 25, 40)) AS pool_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -64,7 +68,7 @@ FINAL AS ( tickSpacing AS tick_spacing, pool_address, _log_id, - _inserted_timestamp + modified_timestamp FROM created_pools ) diff --git a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_swaps.sql b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_swaps.sql index 31137a4..602d817 100644 --- a/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_swaps.sql +++ b/models/silver/defi/dex/pancake_swap/silver_dex__pancakeswap_v3_swaps.sql @@ -93,8 +93,12 @@ base_swaps AS ( END AS token_out, fee, tick_spacing, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -143,6 +147,6 @@ SELECT token_in, token_out, _log_id, - _inserted_timestamp + modified_timestamp FROM base_swaps diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql index ee3b429..c1a3d53 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_pools.sql @@ -50,7 +50,7 @@ SELECT pool_address, pool_id, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql index 4b004ef..2e4ce29 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v1_swaps.sql @@ -50,8 +50,12 @@ swaps_base AS ( ) AS amount1Out, token0, token1, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -113,7 +117,7 @@ SELECT 'Swap' AS event_name, 'trader-joe-v1' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE token_in <> token_out diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql index 8a9a3c9..de33000 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_1_swaps.sql @@ -71,8 +71,12 @@ swaps_base AS ( ) AS protocolFee1, tokenX, tokenY, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -141,7 +145,7 @@ SELECT 'Swap' AS event_name, 'trader-joe-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE token_in <> token_out diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql index b361146..4cd5e66 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_pools.sql @@ -27,8 +27,12 @@ WITH pool_creation AS ( WHEN contract_address = '0x43646a8e839b2f2766392c1bf8f60f6e587b6960' THEN 'v2' WHEN contract_address = '0x8e42f2f4101563bf679975178e880fd87d3efd4e' THEN 'v2.1' END AS version, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -62,7 +66,7 @@ SELECT pool_id, version, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY lb_pair ORDER BY diff --git a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql index c429752..bc97378 100644 --- a/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql +++ b/models/silver/defi/dex/trader_joe/silver_dex__trader_joe_v2_swaps.sql @@ -70,8 +70,12 @@ swaps_base AS ( WHEN swapForY THEN tokenX ELSE tokenY END AS token_in_address, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -114,6 +118,6 @@ SELECT 'Swap' AS event_name, 'trader-joe-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql b/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql index e242bd2..c028043 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ2_pools.sql @@ -50,7 +50,7 @@ SELECT pool_address, pool_id, _log_id, - _inserted_timestamp + modified_timestamp FROM pool_creation qualify(ROW_NUMBER() over (PARTITION BY pool_address ORDER BY diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ2_swaps.sql b/models/silver/defi/dex/uniswap/silver_dex__univ2_swaps.sql index c2cc44c..11f1e47 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ2_swaps.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ2_swaps.sql @@ -50,8 +50,12 @@ swaps_base AS ( CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS tx_to, token0, token1, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} INNER JOIN pools p @@ -112,7 +116,7 @@ SELECT 'Swap' AS event_name, 'uniswap-v2' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base WHERE diff --git a/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql b/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql index 513ca81..72b37fd 100644 --- a/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql +++ b/models/silver/defi/dex/uniswap/silver_dex__univ3_pools.sql @@ -29,8 +29,12 @@ WITH created_pools AS ( ) ) AS tick_spacing, CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 25, 40)) AS pool_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE diff --git a/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql b/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql index 031ad9b..7469794 100644 --- a/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql +++ b/models/silver/defi/dex/woofi/silver_dex__woofi_swaps.sql @@ -52,8 +52,12 @@ WITH router_swaps_base AS ( 40 ) ) AS rebateTo, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -117,8 +121,12 @@ swaps_base AS ( 40 ) ) AS rebateTo, - l._log_id, - l._inserted_timestamp + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id, + l.modified_timestamp FROM {{ ref('silver__logs') }} l @@ -178,7 +186,7 @@ SELECT 'WooRouterSwap' AS event_name, 'woofi' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM router_swaps_base UNION ALL @@ -208,6 +216,6 @@ SELECT 'WooSwap' AS event_name, 'woofi' AS platform, _log_id, - _inserted_timestamp + modified_timestamp FROM swaps_base diff --git a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql index 0d3d37d..2e3656d 100644 --- a/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql +++ b/models/silver/defi/lending/complete_lending/silver__complete_lending_borrows.sql @@ -167,8 +167,12 @@ venus AS ( amount, platform, 'bsc' AS blockchain, - l._LOG_ID, - l._INSERTED_TIMESTAMP + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__venus_borrows') }} l diff --git a/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql b/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql index c34709a..2d1e609 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql @@ -13,8 +13,12 @@ WITH log_pull AS ( block_number, block_timestamp, contract_address, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -75,8 +79,8 @@ contract_pull AS ( WHEN l.contract_address = '0x390bf37355e9df6ea2e16eed5686886da6f47669' THEN '0x2170ed0880ac9a755fd29b2688956bd959f933f8' --WETH ELSE t.underlying_asset END AS underlying_asset, - l._inserted_timestamp, - l._log_id + l._log_id, + l.modified_timestamp FROM log_pull l LEFT JOIN traces_pull t diff --git a/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql b/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql index 75f24ee..5827c72 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_withdraws.sql @@ -40,8 +40,12 @@ dforce_redemptions AS ( ) :: INTEGER AS redeemed_token_raw, CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS redeemer, 'dForce' AS platform, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE diff --git a/models/silver/defi/lending/kinza/silver__kinza_borrows.sql b/models/silver/defi/lending/kinza/silver__kinza_borrows.sql index 68a441f..211013d 100644 --- a/models/silver/defi/lending/kinza/silver__kinza_borrows.sql +++ b/models/silver/defi/lending/kinza/silver__kinza_borrows.sql @@ -39,8 +39,12 @@ borrow AS ( origin_to_address, contract_address ) AS lending_pool_contract, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -107,7 +111,7 @@ SELECT 'Kinza' AS platform, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM borrow LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/kinza/silver__kinza_deposits.sql b/models/silver/defi/lending/kinza/silver__kinza_deposits.sql index 7bc0311..3a606c6 100644 --- a/models/silver/defi/lending/kinza/silver__kinza_deposits.sql +++ b/models/silver/defi/lending/kinza/silver__kinza_deposits.sql @@ -32,8 +32,12 @@ WITH deposits AS( origin_to_address, contract_address ) AS lending_pool_contract, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -95,7 +99,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM deposits INNER JOIN atoken_meta diff --git a/models/silver/defi/lending/kinza/silver__kinza_flashloans.sql b/models/silver/defi/lending/kinza/silver__kinza_flashloans.sql index b26e772..3a31fec 100644 --- a/models/silver/defi/lending/kinza/silver__kinza_flashloans.sql +++ b/models/silver/defi/lending/kinza/silver__kinza_flashloans.sql @@ -34,8 +34,12 @@ WITH flashloan AS ( origin_to_address, contract_address ) AS lending_pool_contract, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -102,7 +106,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM flashloan LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/kinza/silver__kinza_liquidations.sql b/models/silver/defi/lending/kinza/silver__kinza_liquidations.sql index 613f2f1..bf26414 100644 --- a/models/silver/defi/lending/kinza/silver__kinza_liquidations.sql +++ b/models/silver/defi/lending/kinza/silver__kinza_liquidations.sql @@ -32,8 +32,12 @@ WITH liquidation AS( origin_to_address, contract_address ) AS lending_pool_contract, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -93,7 +97,7 @@ SELECT amd.underlying_symbol AS debt_token_symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM liquidation LEFT JOIN atoken_meta amc diff --git a/models/silver/defi/lending/kinza/silver__kinza_repayments.sql b/models/silver/defi/lending/kinza/silver__kinza_repayments.sql index 1c0c366..db8b6d7 100644 --- a/models/silver/defi/lending/kinza/silver__kinza_repayments.sql +++ b/models/silver/defi/lending/kinza/silver__kinza_repayments.sql @@ -29,8 +29,12 @@ WITH repay AS( contract_address ) AS lending_pool_contract, origin_from_address AS repayer_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -93,7 +97,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM repay LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/kinza/silver__kinza_tokens.sql b/models/silver/defi/lending/kinza/silver__kinza_tokens.sql index 3515582..5c85e1f 100644 --- a/models/silver/defi/lending/kinza/silver__kinza_tokens.sql +++ b/models/silver/defi/lending/kinza/silver__kinza_tokens.sql @@ -24,8 +24,12 @@ WITH DECODE AS ( utils.udf_hex_to_string ( segmented_data [9] :: STRING ) :: STRING AS atoken_symbol, - l._inserted_timestamp, - l._log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} l @@ -78,8 +82,12 @@ debt_tokens AS ( CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS atoken_address, CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 27, 40)) :: STRING AS atoken_stable_debt_address, CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 27, 40)) :: STRING AS atoken_variable_debt_address, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE diff --git a/models/silver/defi/lending/kinza/silver__kinza_withdraws.sql b/models/silver/defi/lending/kinza/silver__kinza_withdraws.sql index 8b748d6..4ad32a2 100644 --- a/models/silver/defi/lending/kinza/silver__kinza_withdraws.sql +++ b/models/silver/defi/lending/kinza/silver__kinza_withdraws.sql @@ -28,8 +28,12 @@ WITH withdraw AS( origin_to_address, contract_address ) AS lending_pool_contract, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -90,7 +94,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM withdraw LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql b/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql index ad5e827..7f66ae0 100644 --- a/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql +++ b/models/silver/defi/lending/liqee/silver__liqee_withdraws.sql @@ -40,8 +40,12 @@ liqee_redemptions AS ( ) :: INTEGER AS redeemed_token_raw, CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS redeemer, 'Liqee'AS platform, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE diff --git a/models/silver/defi/lending/radiant/silver__radiant_borrows.sql b/models/silver/defi/lending/radiant/silver__radiant_borrows.sql index f6e3f4b..6fdb160 100644 --- a/models/silver/defi/lending/radiant/silver__radiant_borrows.sql +++ b/models/silver/defi/lending/radiant/silver__radiant_borrows.sql @@ -39,8 +39,12 @@ borrow AS ( origin_to_address, contract_address ) AS lending_pool_contract, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -102,7 +106,7 @@ SELECT 'Radiant V2' AS platform, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM borrow LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/radiant/silver__radiant_deposits.sql b/models/silver/defi/lending/radiant/silver__radiant_deposits.sql index c48c2ae..a18e025 100644 --- a/models/silver/defi/lending/radiant/silver__radiant_deposits.sql +++ b/models/silver/defi/lending/radiant/silver__radiant_deposits.sql @@ -32,8 +32,12 @@ WITH deposits AS( origin_to_address, contract_address ) AS lending_pool_contract, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -90,7 +94,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM deposits LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/radiant/silver__radiant_flashloans.sql b/models/silver/defi/lending/radiant/silver__radiant_flashloans.sql index 5958e9b..b77ff9d 100644 --- a/models/silver/defi/lending/radiant/silver__radiant_flashloans.sql +++ b/models/silver/defi/lending/radiant/silver__radiant_flashloans.sql @@ -34,8 +34,12 @@ WITH flashloan AS ( origin_to_address, contract_address ) AS lending_pool_contract, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -97,7 +101,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM flashloan LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/radiant/silver__radiant_liquidations.sql b/models/silver/defi/lending/radiant/silver__radiant_liquidations.sql index 6d707f5..6f8743f 100644 --- a/models/silver/defi/lending/radiant/silver__radiant_liquidations.sql +++ b/models/silver/defi/lending/radiant/silver__radiant_liquidations.sql @@ -32,8 +32,12 @@ WITH liquidation AS( origin_to_address, contract_address ) AS lending_pool_contract, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -93,7 +97,7 @@ SELECT amd.underlying_symbol AS debt_token_symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM liquidation LEFT JOIN atoken_meta amc diff --git a/models/silver/defi/lending/radiant/silver__radiant_repayments.sql b/models/silver/defi/lending/radiant/silver__radiant_repayments.sql index 14f5cbb..97d828d 100644 --- a/models/silver/defi/lending/radiant/silver__radiant_repayments.sql +++ b/models/silver/defi/lending/radiant/silver__radiant_repayments.sql @@ -29,8 +29,12 @@ WITH repay AS( contract_address ) AS lending_pool_contract, origin_from_address AS repayer_address, - _log_id, - _inserted_timestamp + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -88,7 +92,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM repay LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/radiant/silver__radiant_tokens.sql b/models/silver/defi/lending/radiant/silver__radiant_tokens.sql index 83b29d2..c93fa67 100644 --- a/models/silver/defi/lending/radiant/silver__radiant_tokens.sql +++ b/models/silver/defi/lending/radiant/silver__radiant_tokens.sql @@ -24,8 +24,12 @@ WITH DECODE AS ( utils.udf_hex_to_string ( segmented_data [9] :: STRING ) :: STRING AS atoken_symbol, - l._inserted_timestamp, - l._log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} l @@ -76,8 +80,12 @@ debt_tokens AS ( CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS atoken_address, CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 27, 40)) :: STRING AS atoken_stable_debt_address, CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 27, 40)) :: STRING AS atoken_variable_debt_address, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE diff --git a/models/silver/defi/lending/radiant/silver__radiant_withdraws.sql b/models/silver/defi/lending/radiant/silver__radiant_withdraws.sql index 5d6cfdb..c50a7e9 100644 --- a/models/silver/defi/lending/radiant/silver__radiant_withdraws.sql +++ b/models/silver/defi/lending/radiant/silver__radiant_withdraws.sql @@ -28,8 +28,12 @@ WITH withdraw AS( origin_to_address, contract_address ) AS lending_pool_contract, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE @@ -85,7 +89,7 @@ SELECT atoken_meta.underlying_symbol AS symbol, 'bsc' AS blockchain, _log_id, - _inserted_timestamp + modified_timestamp FROM withdraw LEFT JOIN atoken_meta diff --git a/models/silver/defi/lending/venus/silver__venus_withdraws.sql b/models/silver/defi/lending/venus/silver__venus_withdraws.sql index 629a5c3..b5e7c29 100644 --- a/models/silver/defi/lending/venus/silver__venus_withdraws.sql +++ b/models/silver/defi/lending/venus/silver__venus_withdraws.sql @@ -40,8 +40,12 @@ venus_redemptions AS ( ) :: INTEGER AS redeemed_token_raw, CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS redeemer, 'Venus' AS platform, - _inserted_timestamp, - _log_id + CONCAT( + tx_hash :: STRING, + '-', + event_index :: STRING + ) AS _log_id, + modified_timestamp FROM {{ ref('silver__logs') }} WHERE diff --git a/standardization_updates.sql b/standardization_updates.sql new file mode 100755 index 0000000..e81be4a --- /dev/null +++ b/standardization_updates.sql @@ -0,0 +1,21 @@ +#!/bin/bash + +if [ -z "$1" ]; then + echo "Please provide a directory path" + exit 1 +fi + +find "$1" -type f -name "*.sql" | while read -r file; do + # Replace the _log_id pattern in files referencing silver__logs + sed -i ' + /{{ ref (.silver__logs.|.silver__decoded_logs.)/{ + N;N;N; + s/_log_id,\n.*_inserted_timestamp\n.*FROM/CONCAT(\ntx_hash :: STRING,\n'\''-'\'',\nevent_index :: STRING\n) AS _log_id,\nmodified_timestamp\nFROM/g + } + ' "$file" + + # Replace remaining _inserted_timestamp with modified_timestamp + sed -i 's/_inserted_timestamp/modified_timestamp/g' "$file" + + echo "Processed: $file" +done \ No newline at end of file