abis and token reads (#15)

* abis and token reads

* abi refresh

.github/workflows/dbt_run_abi_refresh.yml

@@ -0,0 +1,46 @@
+name: dbt_run_abi_refresh
+run-name: dbt_run_abi_refresh
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs “At minute 10 past every 12th hour.” (see https://crontab.guru)
+    - cron: '10 */12 * * *'
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "blast_models,tag:abis"
+  

models/bronze/api_udf/bronze_api__contract_abis.sql

@@ -0,0 +1,59 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = "contract_address",
+    full_refresh = false,
+    tags = ['non_realtime']
+) }}
+
+WITH base AS (
+
+    SELECT
+        contract_address
+    FROM
+        {{ ref('silver__relevant_abi_contracts') }}
+
+{% if is_incremental() %}
+EXCEPT
+SELECT
+    contract_address
+FROM
+    {{ this }}
+WHERE
+    abi_data :data :result :: STRING <> 'Max rate limit reached'
+{% endif %}
+LIMIT
+    50
+), all_contracts AS (
+    SELECT
+        contract_address
+    FROM
+        base -- to do: add back retry logic
+),
+row_nos AS (
+    SELECT
+        contract_address,
+        ROW_NUMBER() over (
+            ORDER BY
+                contract_address
+        ) AS row_no
+    FROM
+        all_contracts
+),
+batched AS ({% for item in range(101) %}
+SELECT
+    rn.contract_address, live.udf_api('GET', CONCAT('https://api.blastscan.io/api?module=contract&action=getabi&address=', rn.contract_address, '&apikey={key}'),{ 'User-Agent': 'FlipsideStreamline' },{}, 'Vault/prod/block_explorers/blast_scan') AS abi_data, SYSDATE() AS _inserted_timestamp
+FROM
+    row_nos rn
+WHERE
+    row_no = {{ item }}
+
+    {% if not loop.last %}
+    UNION ALL
+    {% endif %}
+{% endfor %})
+SELECT
+    contract_address,
+    abi_data,
+    _inserted_timestamp
+FROM
+    batched

models/bronze/api_udf/bronze_api__contract_abis.yml

@@ -0,0 +1,22 @@
+version: 2
+models:
+  - name: bronze_api__contract_abis
+
+    columns:
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: CONTRACT_ADDRESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - VARCHAR
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: "^(0x)[0-9a-fA-F]{40}$"

models/bronze/api_udf/bronze_api__token_reads.sql

@@ -0,0 +1,127 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = "contract_address",
+    full_refresh = false,
+    tags = ['non_realtime']
+) }}
+
+WITH base AS (
+
+    SELECT
+        contract_address,
+        latest_block
+    FROM
+        {{ ref('silver__relevant_contracts') }}
+
+{% if is_incremental() %}
+WHERE
+    contract_address NOT IN (
+        SELECT
+            contract_address
+        FROM
+            {{ this }}
+    )
+{% endif %}
+LIMIT
+    200
+), function_sigs AS (
+    SELECT
+        '0x313ce567' AS function_sig,
+        'decimals' AS function_name
+    UNION
+    SELECT
+        '0x06fdde03',
+        'name'
+    UNION
+    SELECT
+        '0x95d89b41',
+        'symbol'
+),
+all_reads AS (
+    SELECT
+        *
+    FROM
+        base
+        JOIN function_sigs
+        ON 1 = 1
+),
+ready_reads AS (
+    SELECT
+        contract_address,
+        latest_block,
+        function_sig,
+        RPAD(
+            function_sig,
+            64,
+            '0'
+        ) AS input,
+        utils.udf_json_rpc_call(
+            'eth_call',
+            [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(latest_block)],
+            concat_ws(
+                '-',
+                contract_address,
+                input,
+                latest_block
+            )
+        ) AS rpc_request
+    FROM
+        all_reads
+),
+batch_reads AS (
+    SELECT
+        ARRAY_AGG(rpc_request) AS batch_rpc_request
+    FROM
+        ready_reads
+),
+node_call AS (
+    SELECT
+        *,
+        live.udf_api(
+            'POST',
+            CONCAT(
+                '{service}',
+                '/',
+                '{Authentication}'
+            ),{},
+            batch_rpc_request,
+            'Vault/prod/blast/mainnet'
+        ) AS response
+    FROM
+        batch_reads
+    WHERE
+        EXISTS (
+            SELECT
+                1
+            FROM
+                ready_reads
+            LIMIT
+                1
+        )
+), flat_responses AS (
+    SELECT
+        VALUE :id :: STRING AS call_id,
+        VALUE :result :: STRING AS read_result
+    FROM
+        node_call,
+        LATERAL FLATTEN (
+            input => response :data
+        )
+)
+SELECT
+    SPLIT_PART(
+        call_id,
+        '-',
+        1
+    ) AS contract_address,
+    SPLIT_PART(
+        call_id,
+        '-',
+        3
+    ) AS block_number,
+    LEFT(SPLIT_PART(call_id, '-', 2), 10) AS function_sig,
+    NULL AS function_input,
+    read_result,
+    SYSDATE() :: TIMESTAMP AS _inserted_timestamp
+FROM
+    flat_responses

models/bronze/api_udf/bronze_api__token_reads.yml

@@ -0,0 +1,18 @@
+version: 2
+models:
+  - name: bronze_api__token_reads
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - CONTRACT_ADDRESS
+            - FUNCTION_SIG
+    columns:
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ

models/silver/abis/silver__proxies.sql

@@ -0,0 +1,104 @@
+{{ config (
+    materialized = 'incremental',
+    unique_key = ['contract_address','proxy_address'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
+    tags = ['abis']
+) }}
+
+WITH base AS (
+
+    SELECT
+        from_address,
+        to_address,
+        MIN(block_number) AS start_block,
+        MAX(_inserted_timestamp) AS _inserted_timestamp
+    FROM
+        {{ ref('silver__traces') }}
+    WHERE
+        TYPE = 'DELEGATECALL'
+        AND trace_status = 'SUCCESS'
+        AND tx_status = 'SUCCESS'
+        AND from_address != to_address -- exclude self-calls
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        MAX(_inserted_timestamp) - INTERVAL '24 hours'
+    FROM
+        {{ this }}
+)
+{% endif %}
+GROUP BY
+    from_address,
+    to_address
+),
+create_id AS (
+    SELECT
+        from_address AS contract_address,
+        to_address AS proxy_address,
+        start_block,
+        CONCAT(
+            from_address,
+            '-',
+            to_address
+        ) AS _id,
+        _inserted_timestamp
+    FROM
+        base
+),
+heal AS (
+    SELECT
+        contract_address,
+        proxy_address,
+        start_block,
+        _id,
+        _inserted_timestamp
+    FROM
+        create_id
+
+{% if is_incremental() %}
+UNION ALL
+SELECT
+    contract_address,
+    proxy_address,
+    start_block,
+    _id,
+    _inserted_timestamp
+FROM
+    {{ this }}
+    JOIN create_id USING (
+        contract_address,
+        proxy_address
+    )
+{% endif %}
+),
+FINAL AS (
+    SELECT
+        contract_address,
+        proxy_address,
+        start_block,
+        _id,
+        _inserted_timestamp
+    FROM
+        heal qualify ROW_NUMBER() over (
+            PARTITION BY contract_address,
+            proxy_address
+            ORDER BY
+                start_block ASC
+        ) = 1
+)
+SELECT
+    f.contract_address,
+    f.proxy_address,
+    f.start_block,
+    f._id,
+    f._inserted_timestamp,
+    C.block_number AS created_block,
+    p.block_number AS proxy_created_block
+FROM
+    FINAL f
+    JOIN {{ ref('silver__created_contracts') }} C
+    ON f.contract_address = C.created_contract_address
+    JOIN {{ ref('silver__created_contracts') }}
+    p
+    ON f.proxy_address = p.created_contract_address

models/silver/abis/silver__proxies.yml

@@ -0,0 +1,7 @@
+version: 2
+models:
+  - name: silver__proxies
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - _ID

models/silver/abis/silver__relevant_abi_contracts.sql

@@ -0,0 +1,29 @@
+{{ config(
+    materialized = 'table',
+    unique_key = "contract_address",
+    tags = ['abis']
+) }}
+
+WITH base AS (
+
+    SELECT
+        contract_address
+    FROM
+        {{ ref('silver__relevant_contracts') }}
+),
+proxies AS (
+    SELECT
+        proxy_address
+    FROM
+        {{ ref('silver__proxies') }}
+        JOIN base USING (contract_address)
+)
+SELECT
+    contract_address
+FROM
+    base
+UNION
+SELECT
+    proxy_address AS contract_address
+FROM
+    proxies

models/silver/abis/silver__relevant_abi_contracts.yml

@@ -0,0 +1,7 @@
+version: 2
+models:
+  - name: silver__relevant_abi_contracts
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - CONTRACT_ADDRESS

models/silver/core/silver__relevant_contracts.sql

@@ -0,0 +1,18 @@
+{{ config(
+    materialized = 'table',
+    unique_key = "contract_address",
+    tags = ['non_realtime']
+) }}
+
+SELECT
+    contract_address,
+    'blast' AS blockchain,
+    COUNT(*) AS transfers,
+    MAX(block_number) AS latest_block
+FROM
+    {{ ref('silver__logs') }}
+GROUP BY
+    1,
+    2
+HAVING
+    COUNT(*) > 25

models/silver/core/silver__relevant_contracts.yml

@@ -0,0 +1,7 @@
+version: 2
+models:
+  - name: silver__relevant_contracts
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - CONTRACT_ADDRESS

package-lock.yml

@@ -6,11 +6,11 @@ packages:
 - package: dbt-labs/dbt_utils
   version: 1.0.0
 - git: https://github.com/FlipsideCrypto/fsc-utils.git
-  revision: b0b5e615143f736d3de249152cda509e47fd21fd
+  revision: 436e7a90ccb1bfa8d566ab99a807f38ec53f74b7
 - package: get-select/dbt_snowflake_query_tags
   version: 2.3.3
 - package: calogica/dbt_date
   version: 0.7.2
 - git: https://github.com/FlipsideCrypto/livequery-models.git
-  revision: bca494102fbd2d621d32746e9a7fe780678044f8
-sha1_hash: 31b810d35e40945328ca2d93e7c9e190a8901ecf
+  revision: 992947a4eaa8fccdf2cfcd2cb73a470ff5e89fa2
+sha1_hash: 342c7081a105a2da3cd7f77edf07124c8f0f1c25

packages.yml

@@ -6,6 +6,6 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 1.0.0
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: v1.16.1
+    revision: v1.18.0
   - package: get-select/dbt_snowflake_query_tags
     version: [">=2.0.0", "<3.0.0"]