AN-3041/defillama-request-fail (#26)

models/defillama/bronze/bronze__defillama_bridges.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'bridge_id',
+    full_refresh = false,
     tags = ['defillama']
 ) }}
 
@@ -11,8 +12,9 @@ SELECT
         'GET','https://bridges.llama.fi/bridges?includeChains=true',{},{}
     ) AS read,
     SYSDATE() AS _inserted_timestamp
-)
+),
 
+FINAL AS (
 SELECT
     VALUE:id::STRING AS bridge_id,
     VALUE:name::STRING AS bridge,
@@ -22,7 +24,6 @@ SELECT
         WHEN VALUE:destinationChain::STRING ilike 'false' OR VALUE:destinationChain::STRING = '-' THEN NULL 
         ELSE VALUE:destinationChain::STRING 
     END AS destination_chain,
-    ROW_NUMBER() OVER (ORDER BY bridge) AS row_num,
     _inserted_timestamp
 FROM bridge_base,
     LATERAL FLATTEN (input=> read:data:bridges)
@@ -34,4 +35,34 @@ WHERE bridge_id NOT IN (
     FROM
         {{ this }}
 )
+)
+
+SELECT
+    bridge_id,
+    bridge,
+    bridge_name,
+    chains,
+    destination_chain,
+    m.row_num + ROW_NUMBER() OVER (ORDER BY bridge) AS row_num,
+    _inserted_timestamp
+FROM FINAL
+JOIN (
+    SELECT
+        MAX(row_num) AS row_num
+    FROM
+        {{ this }}
+) m ON 1=1
+
+{% else %}
+)
+
+SELECT
+    bridge_id,
+    bridge,
+    bridge_name,
+    chains,
+    destination_chain,
+    ROW_NUMBER() OVER (ORDER BY bridge) AS row_num,
+    _inserted_timestamp
+FROM FINAL
 {% endif %}

models/defillama/bronze/bronze__defillama_chains.sql

@@ -11,13 +11,13 @@ SELECT
         'GET','https://api.llama.fi/chains',{},{}
     ) AS read,
     SYSDATE() AS _inserted_timestamp
-)
+),
 
+FINAL AS (
 SELECT
     VALUE:chainId::STRING AS chain_id,
     VALUE:name::STRING AS chain,
     VALUE:tokenSymbol::STRING AS token_symbol,
-    ROW_NUMBER() OVER (ORDER BY chain) AS row_num,
     _inserted_timestamp
 FROM chain_base,
     LATERAL FLATTEN (input=> read:data)
@@ -29,4 +29,29 @@ WHERE chain NOT IN (
     FROM
         {{ this }}
 )
+)
+
+SELECT
+    chain_id,
+    chain,
+    token_symbol,
+    m.row_num + ROW_NUMBER() OVER (ORDER BY chain) AS row_num,
+    _inserted_timestamp
+FROM FINAL
+JOIN (
+    SELECT
+        MAX(row_num) AS row_num
+    FROM
+        {{ this }}
+) m ON 1=1
+
+{% else %}
+)
+SELECT
+    chain_id,
+    chain,
+    token_symbol,
+    ROW_NUMBER() OVER (ORDER BY chain) AS row_num,
+    _inserted_timestamp
+FROM FINAL
 {% endif %}

models/defillama/bronze/bronze__defillama_stablecoins.sql

@@ -11,8 +11,9 @@ SELECT
         'GET','https://stablecoins.llama.fi/stablecoins?includePrices=false',{},{}
     ) AS read,
     SYSDATE() AS _inserted_timestamp
-)
+),
 
+FINAL AS (
 SELECT
     VALUE:id::STRING AS stablecoin_id,
     VALUE:name::STRING AS stablecoin,
@@ -21,7 +22,6 @@ SELECT
     VALUE:pegMechanism::STRING AS peg_mechanism,
     VALUE:priceSource::STRING AS price_source,
     VALUE:chains AS chains,
-    ROW_NUMBER() OVER (ORDER BY stablecoin) AS row_num,
     _inserted_timestamp
 FROM stablecoin_base,
     LATERAL FLATTEN (input=> read:data:peggedAssets)
@@ -33,4 +33,38 @@ WHERE stablecoin_id NOT IN (
     FROM
         {{ this }}
 )
-{% endif %}
+)
+
+SELECT
+    stablecoin_id,
+    stablecoin,
+    symbol,
+    peg_type,
+    peg_mechanism,
+    price_source,
+    chains,
+    m.row_num + ROW_NUMBER() OVER (ORDER BY stablecoin) AS row_num,
+    _inserted_timestamp
+FROM FINAL
+JOIN (
+    SELECT
+        MAX(row_num) AS row_num
+    FROM
+        {{ this }}
+) m ON 1=1
+
+{% else %}
+)
+
+SELECT
+    stablecoin_id,
+    stablecoin,
+    symbol,
+    peg_type,
+    peg_mechanism,
+    price_source,
+    chains,
+    ROW_NUMBER() OVER (ORDER BY stablecoin) AS row_num,
+    _inserted_timestamp
+FROM FINAL
+{% endif %}

models/defillama/silver/silver__defillama_bridge_volume.sql

@@ -33,7 +33,7 @@ WHERE bridge_id NOT IN (
     FROM (
         SELECT 
             DISTINCT bridge_id,
-            MAX(COALESCE(timestamp::DATE,CURRENT_DATE::TIMESTAMP)) AS max_timestamp
+            MAX(timestamp::DATE) AS max_timestamp
         FROM {{ this }}
         GROUP BY 1
         HAVING CURRENT_DATE = max_timestamp
@@ -50,7 +50,7 @@ SELECT
     bridge_id,
     bridge,
     bridge_name,
-    COALESCE(TO_TIMESTAMP(VALUE:date::INTEGER),CURRENT_DATE::TIMESTAMP) AS timestamp,
+    TO_TIMESTAMP(VALUE:date::INTEGER) AS timestamp,
     VALUE:depositTxs::INTEGER AS deposit_txs,
     VALUE:depositUSD::INTEGER AS deposit_usd,
     VALUE:withdrawTxs::INTEGER AS withdraw_txs,
@@ -58,6 +58,4 @@ SELECT
     _inserted_timestamp,
     CONCAT(bridge_id,'-',bridge,'-',timestamp) AS id
 FROM bridge_base,
-    LATERAL FLATTEN (input=> read:data)
-WHERE deposit_txs IS NOT NULL
-    OR withdraw_txs IS NOT NULL
+    LATERAL FLATTEN (input=> read:data)

models/defillama/silver/silver__defillama_chains_tvl.sql

@@ -23,6 +23,8 @@ FROM (
         row_num
     FROM {{ ref('bronze__defillama_chains') }}
     WHERE row_num BETWEEN {{ item * 60 + 1 }} AND {{ (item + 1) * 60 }}
+        AND chain NOT IN ('Regen')
+        --exclude chains with response size > 6mb
     )
 {% if is_incremental() %}
 WHERE chain NOT IN (