retry abis (#160)

models/bronze/api_udf/_retry_abis.sql

@@ -0,0 +1,82 @@
+{{ config (
+    materialized = "ephemeral"
+) }}
+
+WITH retry AS (
+
+    SELECT
+        contract_address,
+        MAX(block_number) AS block_number,
+        COUNT(*) AS events
+    FROM
+        {{ ref("silver__logs") }}
+        l
+        LEFT JOIN {{ source(
+            'optimism_silver',
+            'verified_abis'
+        ) }}
+        v USING (contract_address)
+    WHERE
+        l.block_timestamp >= CURRENT_DATE - INTERVAL '30 days' -- recent activity
+        AND v.contract_address IS NULL -- no verified abi
+        AND l.contract_address NOT IN (
+            SELECT
+                contract_address
+            FROM
+                {{ source(
+                    'optimism_bronze_api',
+                    'contract_abis'
+                ) }}
+            WHERE
+                _inserted_timestamp >= CURRENT_DATE - INTERVAL '30 days' -- this won't let us retry the same contract within 30 days
+                AND abi_data :data :result :: STRING <> 'Max rate limit reached'
+        )
+    GROUP BY
+        contract_address
+    ORDER BY
+        events DESC
+    LIMIT
+        25
+), FINAL AS (
+    SELECT
+        proxy_address AS contract_address,
+        start_block AS block_number
+    FROM
+        {{ ref("silver__proxies") }}
+        p
+        JOIN retry r USING (contract_address)
+        LEFT JOIN {{ source(
+            'optimism_silver',
+            'verified_abis'
+        ) }}
+        v
+        ON v.contract_address = p.proxy_address
+    WHERE
+        v.contract_address IS NULL
+        AND p.contract_address NOT IN (
+            SELECT
+                contract_address
+            FROM
+                {{ source(
+                    'optimism_bronze_api',
+                    'contract_abis'
+                ) }}
+            WHERE
+                _inserted_timestamp >= CURRENT_DATE - INTERVAL '30 days' -- this won't let us retry the same contract within 30 days
+                AND abi_data :data :result :: STRING <> 'Max rate limit reached'
+        )
+    UNION ALL
+    SELECT
+        contract_address,
+        block_number
+    FROM
+        retry
+)
+SELECT
+    *
+FROM
+    FINAL qualify ROW_NUMBER() over (
+        PARTITION BY contract_address
+        ORDER BY
+            block_number DESC
+    ) = 1

models/bronze/api_udf/bronze_api__contract_abis.sql

@@ -33,20 +33,28 @@ WHERE
     abi_data :data :result :: STRING <> 'Max rate limit reached'
 {% endif %}
 LIMIT
-    300
-), row_nos AS (
+    50
+), all_contracts AS (
+    SELECT
+        contract_address
+    FROM
+        base
+    UNION
+    SELECT
+        contract_address
+    FROM
+        {{ ref('_retry_abis') }}
+),
+row_nos AS (
     SELECT
         contract_address,
         ROW_NUMBER() over (
             ORDER BY
                 contract_address
         ) AS row_no,
-        FLOOR(
-            row_no / 2
-        ) + 1 AS batch_no,
         api_key
     FROM
-        base
+        all_contracts
         JOIN api_keys
         ON 1 = 1
 ),
@@ -56,14 +64,9 @@ SELECT
 FROM
     row_nos rn
 WHERE
-    batch_no = {{ item }} + 1
-    AND EXISTS (
-SELECT
-    1
-FROM
-    row_nos
-LIMIT
-    1) {% if not loop.last %}
+    row_no = {{ item }}
+
+    {% if not loop.last %}
     UNION ALL
     {% endif %}
 {% endfor %})

models/sources.yml

@@ -62,3 +62,8 @@ sources:
     schema: silver
     tables:
       - name: verified_abis
+  - name: optimism_bronze_api
+    database: optimism
+    schema: bronze_api
+    tables:
+      - name: contract_abis