diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql index 1b71bae3..9d7745e9 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH all_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql index 49843d45..0a241909 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v2_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v2_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH all_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql index e2f4a963..1a183b4e 100644 --- a/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aave/silver_reads__aave_v3_reads.sql @@ -6,7 +6,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aave_v3_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH all_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql index bed2aade..1d4e2f90 100644 --- a/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/aerodrome/silver_reads__aerodrome_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'aerodrome_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH liquidity_pools AS ( diff --git a/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql index 7e48c75e..ab803acf 100644 --- a/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/binance/silver_reads__binance_v1_reads.sql @@ -7,7 +7,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'binance_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH contracts AS ( diff --git a/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v1_reads.sql index c2c34027..aad35681 100644 --- a/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'compound_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} -- Compound V1 TVL: balanceOf(MoneyMarket) for each collateral token diff --git a/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v2_reads.sql index 0231a231..981fb0b4 100644 --- a/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v2_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'compound_v2_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH all_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v3_reads.sql index 25a6f85e..129d10d8 100644 --- a/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/compound/silver_reads__compound_v3_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'compound_v3_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH base_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_v1_reads.sql index f24ff4e2..91a5096a 100644 --- a/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/curve/silver_reads__curve_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'curve_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH liquidity_pools AS ( diff --git a/models/curated_package/contract_reads/streamline/records/eigenlayer/silver_reads__eigenlayer_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/eigenlayer/silver_reads__eigenlayer_v1_reads.sql index 0c29e59e..66865685 100644 --- a/models/curated_package/contract_reads/streamline/records/eigenlayer/silver_reads__eigenlayer_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/eigenlayer/silver_reads__eigenlayer_v1_reads.sql @@ -6,7 +6,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'eigenlayer_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH strategy_events AS ( diff --git a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql index 9e51616f..98b41f25 100644 --- a/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/etherfi/silver_reads__etherfi_v1_reads.sql @@ -7,7 +7,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'etherfi_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH contracts AS ( diff --git a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql index b6a40d31..7c9f95b0 100644 --- a/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/lido/silver_reads__lido_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'lido_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH pooled_assets AS ( diff --git a/models/curated_package/contract_reads/streamline/records/morpho/silver_reads__morpho_blue_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/morpho/silver_reads__morpho_blue_v1_reads.sql index c5a77dc6..c4fbbdd2 100644 --- a/models/curated_package/contract_reads/streamline/records/morpho/silver_reads__morpho_blue_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/morpho/silver_reads__morpho_blue_v1_reads.sql @@ -6,7 +6,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'morpho_blue_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH market_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql index 357c0d04..14f30085 100644 --- a/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/polymarket/silver_reads__polymarket_v1_reads.sql @@ -7,7 +7,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'polymarket_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH contracts AS ( diff --git a/models/curated_package/contract_reads/streamline/records/rocketpool/silver_reads__rocketpool_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/rocketpool/silver_reads__rocketpool_v1_reads.sql index 02957595..ea436272 100644 --- a/models/curated_package/contract_reads/streamline/records/rocketpool/silver_reads__rocketpool_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/rocketpool/silver_reads__rocketpool_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'rocketpool_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} SELECT diff --git a/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_collateral_joins.sql b/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_collateral_joins.sql index 4a13011b..5d9a1863 100644 --- a/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_collateral_joins.sql +++ b/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_collateral_joins.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'sky_v1_collateral_joins_id', - tags = ['silver','contract_reads','sky'] + tags = ['streamline','contract_reads','records','sky'] ) }} -- Sky Protocol (MakerDAO) collateral join discovery diff --git a/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_reads.sql index 7fc9558d..d60db6e8 100644 --- a/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/sky/silver_reads__sky_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'sky_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH collateral_joins AS ( diff --git a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql index 41b8531a..f06560d0 100644 --- a/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/stablecoins/silver_reads__stablecoins_reads.sql @@ -10,7 +10,7 @@ incremental_strategy = 'delete+insert', unique_key = 'stablecoins_reads_id', post_hook = '{{ unverify_stablecoins() }}', - tags = ['silver','contract_reads','heal'] + tags = ['streamline','contract_reads','records','heal'] ) }} WITH verified_stablecoins AS ( diff --git a/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql index 55a319f9..2a077383 100644 --- a/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/superchain/silver_reads__superchain_slipstream_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'superchain_slipstream_v1_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH blacklisted_tokens AS ( diff --git a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_v1_reads.sql b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_v1_reads.sql index 7c384883..d55b3838 100644 --- a/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_v1_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/tornado_cash/silver_reads__tornado_cash_v1_reads.sql @@ -8,7 +8,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'tornado_cash_reads_id', - tags = ['silver','contract_reads'] + tags = ['streamline','contract_reads','records'] ) }} WITH mixers AS ( diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql index 88d828a4..c38a758e 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v2_reads.sql @@ -7,7 +7,7 @@ incremental_strategy = 'delete+insert', unique_key = 'uniswap_v2_reads_id', post_hook = '{{ unverify_contract_reads() }}', - tags = ['silver','contract_reads','heal'] + tags = ['streamline','contract_reads','records','heal'] ) }} WITH verified_contracts AS ( diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql index 31de424a..dab2230a 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v3_reads.sql @@ -9,7 +9,7 @@ incremental_strategy = 'delete+insert', unique_key = 'uniswap_v3_reads_id', post_hook = '{{ unverify_contract_reads() }}', - tags = ['silver','contract_reads','heal'] + tags = ['streamline','contract_reads','records','heal'] ) }} WITH verified_contracts AS ( diff --git a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql index 24812a7a..c0b2371b 100644 --- a/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql +++ b/models/curated_package/contract_reads/streamline/records/uniswap/silver_reads__uniswap_v4_reads.sql @@ -9,7 +9,7 @@ incremental_strategy = 'delete+insert', unique_key = 'uniswap_v4_reads_id', post_hook = '{{ unverify_contract_reads() }}', - tags = ['silver','contract_reads','heal'] + tags = ['streamline','contract_reads','records','heal'] ) }} WITH verified_contracts AS ( @@ -20,29 +20,38 @@ WITH verified_contracts AS ( WHERE is_verified AND token_address IS NOT NULL -), +), +{% if is_incremental() %} +existing_pairs AS ( + SELECT DISTINCT + metadata:token0::STRING AS token0, + metadata:token1::STRING AS token1 + FROM {{ this }} +), +{% endif %} liquidity_pools AS ( SELECT - token0, - token1, - pool_address AS factory_address, - hook_address, - protocol, - version, - platform + pools.token0, + pools.token1, + pools.pool_address AS factory_address, + pools.hook_address, + pools.protocol, + pools.version, + pools.platform FROM - {{ ref('silver_dex__uniswap_v4_pools') }} - WHERE token0 IN (SELECT token_address FROM verified_contracts) - AND token1 IN (SELECT token_address FROM verified_contracts) + {{ ref('silver_dex__uniswap_v4_pools') }} pools + {% if is_incremental() %} + LEFT JOIN existing_pairs ep + ON pools.token0 = ep.token0 + AND pools.token1 = ep.token1 + {% endif %} + WHERE pools.token0 IN (SELECT token_address FROM verified_contracts) + AND pools.token1 IN (SELECT token_address FROM verified_contracts) {% if is_incremental() %} AND ( - modified_timestamp > (SELECT MAX(modified_timestamp) FROM {{ this }}) - OR CONCAT(token0,'-',token1) NOT IN ( - SELECT CONCAT(metadata:token0::STRING,'-',metadata:token1::STRING) - FROM {{ this }} - ) - -- pull in pools with newly verified tokens + pools.modified_timestamp > (SELECT MAX(modified_timestamp) FROM {{ this }}) + OR ep.token0 IS NULL -- pull in pools with newly verified tokens ) {% endif %} ),