An 3706/abi retry (#55)

* retry abis

* test

models/bronze/api_udf/_retry_abis.sql

@@ -0,0 +1,82 @@
+{{ config (
+    materialized = "ephemeral"
+) }}
+
+WITH retry AS (
+
+    SELECT
+        contract_address,
+        MAX(block_number) AS block_number,
+        COUNT(*) AS events
+    FROM
+        {{ ref("silver__logs") }}
+        l
+        LEFT JOIN {{ source(
+            'base_silver',
+            'verified_abis'
+        ) }}
+        v USING (contract_address)
+    WHERE
+        l.block_timestamp >= CURRENT_DATE - INTERVAL '30 days' -- recent activity
+        AND v.contract_address IS NULL -- no verified abi
+        AND l.contract_address NOT IN (
+            SELECT
+                contract_address
+            FROM
+                {{ source(
+                    'base_bronze_api',
+                    'contract_abis'
+                ) }}
+            WHERE
+                _inserted_timestamp >= CURRENT_DATE - INTERVAL '30 days' -- this won't let us retry the same contract within 30 days
+                AND abi_data :data :result :: STRING <> 'Max rate limit reached'
+        )
+    GROUP BY
+        contract_address
+    ORDER BY
+        events DESC
+    LIMIT
+        5
+), FINAL AS (
+    SELECT
+        proxy_address AS contract_address,
+        start_block AS block_number
+    FROM
+        {{ ref("silver__proxies") }}
+        p
+        JOIN retry r USING (contract_address)
+        LEFT JOIN {{ source(
+            'base_silver',
+            'verified_abis'
+        ) }}
+        v
+        ON v.contract_address = p.proxy_address
+    WHERE
+        v.contract_address IS NULL
+        AND p.contract_address NOT IN (
+            SELECT
+                contract_address
+            FROM
+                {{ source(
+                    'base_bronze_api',
+                    'contract_abis'
+                ) }}
+            WHERE
+                _inserted_timestamp >= CURRENT_DATE - INTERVAL '30 days' -- this won't let us retry the same contract within 30 days
+                AND abi_data :data :result :: STRING <> 'Max rate limit reached'
+        )
+    UNION ALL
+    SELECT
+        contract_address,
+        block_number
+    FROM
+        retry
+)
+SELECT
+    *
+FROM
+    FINAL qualify ROW_NUMBER() over (
+        PARTITION BY contract_address
+        ORDER BY
+            block_number DESC
+    ) = 1

models/bronze/api_udf/bronze_api__contract_abis.sql

@@ -34,38 +34,39 @@ WHERE
 {% endif %}
 LIMIT
     5
-), row_nos AS (
+), all_contracts AS (
+    SELECT
+        contract_address
+    FROM
+        base
+    UNION
+    SELECT
+        contract_address
+    FROM
+        {{ ref('_retry_abis') }}
+),
+row_nos AS (
     SELECT
         contract_address,
         ROW_NUMBER() over (
             ORDER BY
                 contract_address
         ) AS row_no,
-        CEIL(
-            row_no / 1
-        ) AS batch_no,
         api_key
     FROM
-        base
+        all_contracts
         JOIN api_keys
         ON 1 = 1
 ),
-batched AS ({% for item in range(6) %}
+batched AS ({% for item in range(15) %}
 SELECT
     rn.contract_address, ethereum.streamline.udf_api('GET', CONCAT('https://api.basescan.org/api?module=contract&action=getabi&address=', rn.contract_address, '&apikey=', api_key),{ 'User-Agent': 'FlipsideStreamline' },{}) AS abi_data, SYSDATE() AS _inserted_timestamp
 FROM
     row_nos rn
 WHERE
-    batch_no = {{ item }}
-    AND EXISTS (
-SELECT
-    1
-FROM
-    row_nos
-WHERE
-    batch_no = {{ item }}
-LIMIT
-    1) {% if not loop.last %}
+    row_no = {{ item }}
+
+    {% if not loop.last %}
     UNION ALL
     {% endif %}
 {% endfor %})

models/bronze/api_udf/bronze_api__contract_abis.yml

@@ -20,6 +20,4 @@ models:
               column_type_list:
                 - VARCHAR
           - dbt_expectations.expect_column_values_to_match_regex:
-              regex: "^(0x)[0-9a-fA-F]{40}$"
-          - unique:
-              where: "abi_data :data :result :: STRING <> 'Max rate limit reached'"
+              regex: "^(0x)[0-9a-fA-F]{40}$"

models/sources.yml

@@ -47,3 +47,13 @@ sources:
     schema: crosschain
     tables:
       - name: node_mapping
+  - name: base_silver
+    database: base
+    schema: silver
+    tables:
+      - name: verified_abis
+  - name: base_bronze_api
+    database: base
+    schema: bronze_api
+    tables:
+      - name: contract_abis