Merge branch 'main' into DAT2-135/generic-reads

2026-02-06 15:51:48 +00:00 · 2026-01-06 09:28:04 -07:00 · 2026-01-06 09:28:04 -07:00 · 1c2abb10ee
commit 1c2abb10ee
parent 13a658a672 1839229281
15 changed files with 97 additions and 30 deletions
--- a/.github/workflows/dbt_run_template_docs_update.yml
+++ b/.github/workflows/dbt_run_template_docs_update.yml
@ -58,7 +58,7 @@ jobs:
          git checkout -B docs origin/main

      - name: generate dbt docs
-        run: dbt docs generate -t prod --exclude "fsc_evm,tag:streamline" "fsc_evm,tag:scores"
+        run: dbt docs generate -t prod --select "$PROJECT_NAME.*" "fsc_evm,tag:core" "fsc_evm,tag:prices" "fsc_evm,tag:labels" "fsc_evm,tag:balances" "fsc_evm,tag:decoded_logs" "fsc_evm,tag:abis" "fsc_evm,tag:defi" "fsc_evm,tag:stats"

      - name: move files to docs directory
        run: |
--- a/macros/global/variables/project_vars/arbitrum_vars.sql
+++ b/macros/global/variables/project_vars/arbitrum_vars.sql
@ -23,7 +23,7 @@
        'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['radiant-v2','radiant-v1'],
        'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','gmx-v1','woofi-v3','woofi-v1'],
        'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['trader_joe-v2','kyberswap-v1','kyberswap-v2','zyberswap-v2','fraxswap-v1'],
-        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['hyperliquid-v1','across-v2','multichain-v7','symbiosis-v1','hop-v1'],
+        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['hyperliquid-v1','across-v2','multichain-v7','symbiosis-v1','hop-v1','everclear-v1'],
        'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': {
            'uniswap': {
                'v2': {
--- a/macros/global/variables/project_vars/avalanche_vars.sql
+++ b/macros/global/variables/project_vars/avalanche_vars.sql
@ -22,8 +22,8 @@
        'CUSTOM_GHA_SCHEDULED_DEXALOT_MAIN_CRON': '5 * * * *',
        'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['aave-v2'],
        'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['gmx-v1','hashflow-v1','woofi-v1'],
-        'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['pharaoh-v1','kyberswap-v2','curve-v1','uniswap-v2','kyberswap-v1','fraxswap-v1'],
-        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['symbiosis-v1','multichain-v7','allbridge-v1'],
+        'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['pharaoh-v1','kyberswap-v2','curve-v1','uniswap-v2','kyberswap-v1','fraxswap-v1','pharaoh-v2'],
+        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['symbiosis-v1','multichain-v7','allbridge-v1','everclear-v1','eywa-v1'],
        'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': {
            'arena_trade': {
                'v1': {
--- a/macros/global/variables/project_vars/ethereum_vars.sql
+++ b/macros/global/variables/project_vars/ethereum_vars.sql
@ -28,7 +28,7 @@
        'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['silo-v1','aave-v1','flux-v1','aave-v2','aave-v2.1','sturdy-v1','silo-v2'],
        'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['hashflow-v1','pancakeswap-v2','synthetix-v1','uniswap-v1'],
        'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['kyberswap-v2','uniswap-v1'],
-        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v1','near_rainbow_bridge-v1','hop-v1'],
+        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['ronin_axie_bridge-v1','multichain-v7','symbiosis-v1','axie_infinity-v2','across-v1','near_rainbow_bridge-v1','hop-v1','everclear-v1'],
        'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': {
            'uniswap': {
                'v1': {
--- a/macros/global/variables/project_vars/ink_vars.sql
+++ b/macros/global/variables/project_vars/ink_vars.sql
@ -18,6 +18,7 @@
        'MAIN_GHA_SCHEDULED_SCORES_CRON': '50 5 * * *',
        'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['morpho-v1'],
        'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['squidswap-v1'],
+        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['everclear-v1'],
        'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': {
            'uniswap': {
                'v2': {
--- a/macros/global/variables/project_vars/optimism_vars.sql
+++ b/macros/global/variables/project_vars/optimism_vars.sql
@ -22,7 +22,7 @@
        'CURATED_DEFI_LENDING_RECENCY_EXCLUSION_LIST': ['granary-v1','sonne-v1'],
        'CURATED_DEFI_DEX_SWAPS_RECENCY_EXCLUSION_LIST': ['woofi-v1','hashflow-v1'],
        'CURATED_DEFI_DEX_LP_ACTIONS_RECENCY_EXCLUSION_LIST': ['fraxswap-v1','kyberswap-v2','sushiswap-v1','uniswap-v2'],
-        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['eywa-v1','across-v2','meson-v1','symbiosis-v1','multichain-v7','hop-v1'],
+        'CURATED_DEFI_BRIDGE_RECENCY_EXCLUSION_LIST': ['eywa-v1','across-v2','meson-v1','symbiosis-v1','multichain-v7','hop-v1','everclear-v1'],
        'CURATED_DEFI_DEX_SWAPS_CONTRACT_MAPPING': {
            'uniswap': {
                'v2': {
--- a/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql
+++ b/models/balances_package/native/streamline/records/streamline__balances_native_daily_records.sql
@ -51,7 +51,7 @@ tx_fees AS (
        )
    {% endif %}
 ),
-native_transfers_snapshot AS (
+native_records_snapshot AS (
    SELECT
        DISTINCT 
        ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE AS block_date,
@ -80,8 +80,20 @@ native_transfers_snapshot AS (
        tx_fees
    WHERE
        block_timestamp :: DATE <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE
+    {% if not is_incremental() %}
+    UNION
+    SELECT
+        DISTINCT 
+        ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE AS block_date,
+        address
+    FROM
+        {{ ref('silver__balances_validator_addresses_daily')}} b
+    GROUP BY
+        address
+    HAVING MIN(b.block_date) <= ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE
+    {% endif %}
 ),
-native_transfers_history AS (
+native_records_history AS (
    SELECT
        DISTINCT 
        block_timestamp :: DATE AS block_date,
@ -110,11 +122,28 @@ native_transfers_history AS (
        tx_fees
    WHERE
        block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE
-),
-all_transfers AS (
-    SELECT * FROM native_transfers_snapshot
    UNION
-    SELECT * FROM native_transfers_history
+    SELECT
+        DISTINCT
+        block_date,
+        address
+    FROM
+        {{ ref('silver__balances_validator_addresses_daily')}}
+    WHERE 
+        block_date > ('{{ vars.BALANCES_SL_START_DATE }}' :: TIMESTAMP) :: DATE
+    {% if is_incremental() %}
+    AND modified_timestamp > (
+        SELECT
+            MAX(modified_timestamp)
+        FROM
+            {{ this }}
+    )
+    {% endif %}
+),
+all_records AS (
+    SELECT * FROM native_records_snapshot
+    UNION
+    SELECT * FROM native_records_history
 )
 SELECT
    block_date,
@ -124,6 +153,6 @@ SELECT
    SYSDATE() AS modified_timestamp,
    '{{ invocation_id }}' AS _invocation_id
 FROM
-    all_transfers qualify (ROW_NUMBER() over (PARTITION BY balances_native_daily_records_id
+    all_records qualify (ROW_NUMBER() over (PARTITION BY balances_native_daily_records_id
 ORDER BY
    modified_timestamp DESC)) = 1
--- a/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql
+++ b/models/balances_package/native/streamline/records/validators/silver__balances_validator_addresses_daily.sql
@ -0,0 +1,41 @@
+{# Get variables #}
+{% set vars = return_vars() %}
+{# Log configuration details #}
+{{ log_model_details() }}
+{# Set up dbt configuration #}
+{{ config (
+    materialized = "incremental",
+    unique_key = "balances_validator_addresses_daily_id",
+    cluster_by = "block_date",
+    tags = ['silver','streamline','balances','records','native','phase_4']
+) }}
+
+WITH miner_addresses AS (
+        SELECT
+            DISTINCT block_timestamp :: DATE AS block_date,
+            miner AS address
+        FROM
+            {{ ref('core__fact_blocks') }}
+        WHERE miner <> '0x0000000000000000000000000000000000000000'
+        {% if is_incremental() %}
+        AND 
+            modified_timestamp > (
+                SELECT
+                    MAX(modified_timestamp)
+                FROM
+                    {{ this }}
+            )
+        {% endif %}
+    )
+SELECT
+    block_date,
+    address,
+    {{ dbt_utils.generate_surrogate_key(['block_date', 'address']) }} AS balances_validator_addresses_daily_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    miner_addresses
+QUALIFY(ROW_NUMBER() over (PARTITION BY balances_validator_addresses_daily_id
+ORDER BY
+    modified_timestamp DESC)) = 1
--- a/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql
+++ b/models/main_package/prices/silver/silver__complete_native_asset_metadata.sql
@ -33,9 +33,9 @@ FROM
 WHERE
    modified_timestamp >= (
        SELECT
-            MAX(
+            coalesce(MAX(
                modified_timestamp
-            )
+            ), '1970-01-01')
        FROM
            {{ this }}
    )
--- a/models/main_package/prices/silver/silver__complete_native_prices.sql
+++ b/models/main_package/prices/silver/silver__complete_native_prices.sql
@ -37,9 +37,9 @@ FROM
 WHERE
    modified_timestamp >= (
        SELECT
-            MAX(
+            coalesce(MAX(
                modified_timestamp
-            )
+            ), '1970-01-01')
        FROM
            {{ this }}
    )
--- a/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql
+++ b/models/main_package/prices/silver/silver__complete_provider_asset_metadata.sql
@ -33,9 +33,9 @@ FROM
 WHERE
    modified_timestamp >= (
        SELECT
-            MAX(
+            coalesce(MAX(
                modified_timestamp
-            )
+            ), '1970-01-01')
        FROM
            {{ this }}
    )
--- a/models/main_package/prices/silver/silver__complete_provider_prices.sql
+++ b/models/main_package/prices/silver/silver__complete_provider_prices.sql
@ -38,9 +38,9 @@ FROM
 WHERE
    p.modified_timestamp >= (
        SELECT
-            MAX(
+            coalesce(MAX(
                modified_timestamp
-            )
+            ), '1970-01-01')
        FROM
            {{ this }}
    )
--- a/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql
+++ b/models/main_package/prices/silver/silver__complete_token_asset_metadata.sql
@ -40,9 +40,9 @@ FROM
 WHERE
    modified_timestamp >= (
        SELECT
-            MAX(
+            coalesce(MAX(
                modified_timestamp
-            )
+            ), '1970-01-01')
        FROM
            {{ this }}
    )
--- a/models/main_package/prices/silver/silver__complete_token_prices.sql
+++ b/models/main_package/prices/silver/silver__complete_token_prices.sql
@ -44,9 +44,9 @@ FROM
 WHERE
    modified_timestamp >= (
        SELECT
-            MAX(
+            coalesce(MAX(
                modified_timestamp
-            )
+            ), '1970-01-01')
        FROM
            {{ this }}
    )
--- a/packages.yml
+++ b/packages.yml
@ -1,11 +1,7 @@
 packages:
-  - package: calogica/dbt_expectations
-    version: 0.8.2
  - package: dbt-labs/dbt_external_tables
    version: 0.8.2
-  - package: dbt-labs/dbt_utils
-    version: 1.0.0
  - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: v1.39.1
+    revision: v1.41.1
  - package: get-select/dbt_snowflake_query_tags
    version: [">=2.0.0", "<3.0.0"]