diff --git a/.github/workflows/dbt_run_template_docs_update.yml b/.github/workflows/dbt_run_template_docs_update.yml index 27833777..cdaf728a 100644 --- a/.github/workflows/dbt_run_template_docs_update.yml +++ b/.github/workflows/dbt_run_template_docs_update.yml @@ -58,7 +58,7 @@ jobs: git checkout -B docs origin/main - name: generate dbt docs - run: dbt docs generate -t prod --exclude "fsc_evm,tag:streamline" "fsc_evm,tag:scores" + run: dbt docs generate -t prod --select "$PROJECT_NAME.*" "fsc_evm,tag:core" "fsc_evm,tag:prices" "fsc_evm,tag:labels" "fsc_evm,tag:balances" "fsc_evm,tag:decoded_logs" "fsc_evm,tag:abis" "fsc_evm,tag:defi" "fsc_evm,tag:stats" - name: move files to docs directory run: | diff --git a/macros/global/variables/project_vars/arbitrum_vars.sql b/macros/global/variables/project_vars/arbitrum_vars.sql index 8f676508..2006f491 100644 --- a/macros/global/variables/project_vars/arbitrum_vars.sql +++ b/macros/global/variables/project_vars/arbitrum_vars.sql @@ -23,7 +23,7 @@ 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['radiant-v2','radiant-v1'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','gmx-v1','woofi-v3','woofi-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['trader_joe-v2','kyberswap-v1','kyberswap-v2','zyberswap-v2','fraxswap-v1'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['hyperliquid-v1','across-v2','multichain-v7','symbiosis-v1','hop-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['hyperliquid-v1','across-v2','multichain-v7','symbiosis-v1','hop-v1','everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v2': { diff --git a/macros/global/variables/project_vars/avalanche_vars.sql b/macros/global/variables/project_vars/avalanche_vars.sql index 280cecbd..fded5e3d 100644 --- a/macros/global/variables/project_vars/avalanche_vars.sql +++ b/macros/global/variables/project_vars/avalanche_vars.sql @@ -22,8 +22,8 @@ 'CUSTOM_GHA_SCHEDULED_DEXALOT_MAIN_CRON': '5 * * * *', 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['aave-v2'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['gmx-v1','hashflow-v1','woofi-v1'], - 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['pharaoh-v1','kyberswap-v2','curve-v1','uniswap-v2','kyberswap-v1','fraxswap-v1'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['symbiosis-v1','multichain-v7','allbridge-v1'], + 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['pharaoh-v1','kyberswap-v2','curve-v1','uniswap-v2','kyberswap-v1','fraxswap-v1','pharaoh-v2'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['symbiosis-v1','multichain-v7','allbridge-v1','everclear-v1','eywa-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'arena_trade': { 'v1': { diff --git a/macros/global/variables/project_vars/ethereum_vars.sql b/macros/global/variables/project_vars/ethereum_vars.sql index c7ac3a24..20d0653c 100644 --- a/macros/global/variables/project_vars/ethereum_vars.sql +++ b/macros/global/variables/project_vars/ethereum_vars.sql @@ -28,7 +28,7 @@ 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['silo-v1','aave-v1','flux-v1','aave-v2','aave-v2.1','sturdy-v1','silo-v2'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','pancakeswap-v2','synthetix-v1','uniswap-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['kyberswap-v2','uniswap-v1'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v1','near_rainbow_bridge-v1','hop-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v1','near_rainbow_bridge-v1','hop-v1','everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v1': { diff --git a/macros/global/variables/project_vars/ink_vars.sql b/macros/global/variables/project_vars/ink_vars.sql index 0774c79b..b9fbea85 100644 --- a/macros/global/variables/project_vars/ink_vars.sql +++ b/macros/global/variables/project_vars/ink_vars.sql @@ -18,6 +18,7 @@ 'MAIN_GHA_SCHEDULED_SCORES_CRON': '50 5 * * *', 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['morpho-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['squidswap-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v2': { diff --git a/macros/global/variables/project_vars/optimism_vars.sql b/macros/global/variables/project_vars/optimism_vars.sql index 5a0bf41d..4fca6e5a 100644 --- a/macros/global/variables/project_vars/optimism_vars.sql +++ b/macros/global/variables/project_vars/optimism_vars.sql @@ -22,7 +22,7 @@ 'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['granary-v1','sonne-v1'], 'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['woofi-v1','hashflow-v1'], 'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['fraxswap-v1','kyberswap-v2','sushiswap-v1','uniswap-v2'], - 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['eywa-v1','across-v2','meson-v1','symbiosis-v1','multichain-v7','hop-v1'], + 'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['eywa-v1','across-v2','meson-v1','symbiosis-v1','multichain-v7','hop-v1','everclear-v1'], 'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': { 'uniswap': { 'v2': { diff --git a/models/balances_package/native/streamline/streamline__balances_native_daily_records.sql b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql similarity index 75% rename from models/balances_package/native/streamline/streamline__balances_native_daily_records.sql rename to models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql index e6f0463a..d8fb71c6 100644 --- a/models/balances_package/native/streamline/streamline__balances_native_daily_records.sql +++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql @@ -51,7 +51,7 @@ tx_fees AS ( ) {% endif %} ), -native_transfers_snapshot AS ( +native_records_snapshot AS ( SELECT DISTINCT ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE AS block_date, @@ -80,8 +80,20 @@ native_transfers_snapshot AS ( tx_fees WHERE block_timestamp :: DATE <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + {% if not is_incremental() %} + UNION + SELECT + DISTINCT + ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE AS block_date, + address + FROM + {{ ref('silver__balances_validator_addresses_daily')}} b + GROUP BY + address + HAVING MIN(b.block_date) <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + {% endif %} ), -native_transfers_history AS ( +native_records_history AS ( SELECT DISTINCT block_timestamp :: DATE AS block_date, @@ -110,11 +122,28 @@ native_transfers_history AS ( tx_fees WHERE block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE -), -all_transfers AS ( - SELECT * FROM native_transfers_snapshot UNION - SELECT * FROM native_transfers_history + SELECT + DISTINCT + block_date, + address + FROM + {{ ref('silver__balances_validator_addresses_daily')}} + WHERE + block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE + {% if is_incremental() %} + AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} +), +all_records AS ( + SELECT * FROM native_records_snapshot + UNION + SELECT * FROM native_records_history ) SELECT block_date, @@ -124,6 +153,6 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - all_transfers qualify (ROW_NUMBER() over (PARTITION BY balances_native_daily_records_id + all_records qualify (ROW_NUMBER() over (PARTITION BY balances_native_daily_records_id ORDER BY modified_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql new file mode 100644 index 00000000..4f504d1a --- /dev/null +++ b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql @@ -0,0 +1,41 @@ +{# Get variables #} +{% set vars = return_vars() %} +{# Log configuration details #} +{{ log_model_details() }} +{# Set up dbt configuration #} +{{ config ( + materialized = "incremental", + unique_key = "balances_validator_addresses_daily_id", + cluster_by = "block_date", + tags = ['silver','streamline','balances','records','native','phase_4'] +) }} + +WITH miner_addresses AS ( + SELECT + DISTINCT block_timestamp :: DATE AS block_date, + miner AS address + FROM + {{ ref('core__fact_blocks') }} + WHERE miner <> '0x0000000000000000000000000000000000000000' + {% if is_incremental() %} + AND + modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + ) +SELECT + block_date, + address, + {{ dbt_utils.generate_surrogate_key(['block_date', 'address']) }} AS balances_validator_addresses_daily_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + miner_addresses +QUALIFY(ROW_NUMBER() over (PARTITION BY balances_validator_addresses_daily_id +ORDER BY + modified_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql b/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql index c24119b5..713490c0 100644 --- a/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql +++ b/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql @@ -33,9 +33,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_native_prices.sql b/models/main_package/prices/silver/silver__complete_native_prices.sql index c128229f..f7a0be57 100644 --- a/models/main_package/prices/silver/silver__complete_native_prices.sql +++ b/models/main_package/prices/silver/silver__complete_native_prices.sql @@ -37,9 +37,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql b/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql index dccedac2..5289f3d7 100644 --- a/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql +++ b/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql @@ -33,9 +33,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_provider_prices.sql b/models/main_package/prices/silver/silver__complete_provider_prices.sql index 7de9677c..cb20efa6 100644 --- a/models/main_package/prices/silver/silver__complete_provider_prices.sql +++ b/models/main_package/prices/silver/silver__complete_provider_prices.sql @@ -38,9 +38,9 @@ FROM WHERE p.modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql b/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql index 3571415e..05e47be6 100644 --- a/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql +++ b/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql @@ -40,9 +40,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/models/main_package/prices/silver/silver__complete_token_prices.sql b/models/main_package/prices/silver/silver__complete_token_prices.sql index 4dead8dc..8ddc83fc 100644 --- a/models/main_package/prices/silver/silver__complete_token_prices.sql +++ b/models/main_package/prices/silver/silver__complete_token_prices.sql @@ -44,9 +44,9 @@ FROM WHERE modified_timestamp >= ( SELECT - MAX( + coalesce(MAX( modified_timestamp - ) + ), '1970-01-01') FROM {{ this }} ) diff --git a/packages.yml b/packages.yml index f3c70125..28ef3f2b 100644 --- a/packages.yml +++ b/packages.yml @@ -1,11 +1,7 @@ packages: - - package: calogica/dbt_expectations - version: 0.8.2 - package: dbt-labs/dbt_external_tables version: 0.8.2 - - package: dbt-labs/dbt_utils - version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.39.1 + revision: v1.41.1 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"]