diff --git a/.github/workflows/dbt_run_abi_refresh.yml b/.github/workflows/dbt_run_abi_refresh.yml new file mode 100644 index 0000000..e31ae38 --- /dev/null +++ b/.github/workflows/dbt_run_abi_refresh.yml @@ -0,0 +1,46 @@ +name: dbt_run_abi_refresh +run-name: dbt_run_abi_refresh + +on: + workflow_dispatch: + schedule: + # Runs “At minute 10 past every 12th hour.” (see https://crontab.guru) + - cron: '10 */12 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "blast_models,tag:abis" + \ No newline at end of file diff --git a/models/bronze/api_udf/bronze_api__contract_abis.sql b/models/bronze/api_udf/bronze_api__contract_abis.sql new file mode 100644 index 0000000..9dbf21b --- /dev/null +++ b/models/bronze/api_udf/bronze_api__contract_abis.sql @@ -0,0 +1,59 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + full_refresh = false, + tags = ['non_realtime'] +) }} + +WITH base AS ( + + SELECT + contract_address + FROM + {{ ref('silver__relevant_abi_contracts') }} + +{% if is_incremental() %} +EXCEPT +SELECT + contract_address +FROM + {{ this }} +WHERE + abi_data :data :result :: STRING <> 'Max rate limit reached' +{% endif %} +LIMIT + 50 +), all_contracts AS ( + SELECT + contract_address + FROM + base -- to do: add back retry logic +), +row_nos AS ( + SELECT + contract_address, + ROW_NUMBER() over ( + ORDER BY + contract_address + ) AS row_no + FROM + all_contracts +), +batched AS ({% for item in range(101) %} +SELECT + rn.contract_address, live.udf_api('GET', CONCAT('https://api.blastscan.io/api?module=contract&action=getabi&address=', rn.contract_address, '&apikey={key}'),{ 'User-Agent': 'FlipsideStreamline' },{}, 'Vault/prod/block_explorers/blast_scan') AS abi_data, SYSDATE() AS _inserted_timestamp +FROM + row_nos rn +WHERE + row_no = {{ item }} + + {% if not loop.last %} + UNION ALL + {% endif %} +{% endfor %}) +SELECT + contract_address, + abi_data, + _inserted_timestamp +FROM + batched diff --git a/models/bronze/api_udf/bronze_api__contract_abis.yml b/models/bronze/api_udf/bronze_api__contract_abis.yml new file mode 100644 index 0000000..5785b15 --- /dev/null +++ b/models/bronze/api_udf/bronze_api__contract_abis.yml @@ -0,0 +1,22 @@ +version: 2 +models: + - name: bronze_api__contract_abis + + columns: + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: "^(0x)[0-9a-fA-F]{40}$" \ No newline at end of file diff --git a/models/bronze/api_udf/bronze_api__token_reads.sql b/models/bronze/api_udf/bronze_api__token_reads.sql new file mode 100644 index 0000000..151de68 --- /dev/null +++ b/models/bronze/api_udf/bronze_api__token_reads.sql @@ -0,0 +1,127 @@ +{{ config( + materialized = 'incremental', + unique_key = "contract_address", + full_refresh = false, + tags = ['non_realtime'] +) }} + +WITH base AS ( + + SELECT + contract_address, + latest_block + FROM + {{ ref('silver__relevant_contracts') }} + +{% if is_incremental() %} +WHERE + contract_address NOT IN ( + SELECT + contract_address + FROM + {{ this }} + ) +{% endif %} +LIMIT + 200 +), function_sigs AS ( + SELECT + '0x313ce567' AS function_sig, + 'decimals' AS function_name + UNION + SELECT + '0x06fdde03', + 'name' + UNION + SELECT + '0x95d89b41', + 'symbol' +), +all_reads AS ( + SELECT + * + FROM + base + JOIN function_sigs + ON 1 = 1 +), +ready_reads AS ( + SELECT + contract_address, + latest_block, + function_sig, + RPAD( + function_sig, + 64, + '0' + ) AS input, + utils.udf_json_rpc_call( + 'eth_call', + [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(latest_block)], + concat_ws( + '-', + contract_address, + input, + latest_block + ) + ) AS rpc_request + FROM + all_reads +), +batch_reads AS ( + SELECT + ARRAY_AGG(rpc_request) AS batch_rpc_request + FROM + ready_reads +), +node_call AS ( + SELECT + *, + live.udf_api( + 'POST', + CONCAT( + '{service}', + '/', + '{Authentication}' + ),{}, + batch_rpc_request, + 'Vault/prod/blast/mainnet' + ) AS response + FROM + batch_reads + WHERE + EXISTS ( + SELECT + 1 + FROM + ready_reads + LIMIT + 1 + ) +), flat_responses AS ( + SELECT + VALUE :id :: STRING AS call_id, + VALUE :result :: STRING AS read_result + FROM + node_call, + LATERAL FLATTEN ( + input => response :data + ) +) +SELECT + SPLIT_PART( + call_id, + '-', + 1 + ) AS contract_address, + SPLIT_PART( + call_id, + '-', + 3 + ) AS block_number, + LEFT(SPLIT_PART(call_id, '-', 2), 10) AS function_sig, + NULL AS function_input, + read_result, + SYSDATE() :: TIMESTAMP AS _inserted_timestamp +FROM + flat_responses diff --git a/models/bronze/api_udf/bronze_api__token_reads.yml b/models/bronze/api_udf/bronze_api__token_reads.yml new file mode 100644 index 0000000..cfbf685 --- /dev/null +++ b/models/bronze/api_udf/bronze_api__token_reads.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: bronze_api__token_reads + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS + - FUNCTION_SIG + columns: + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/abis/silver__proxies.sql b/models/silver/abis/silver__proxies.sql new file mode 100644 index 0000000..409c1ab --- /dev/null +++ b/models/silver/abis/silver__proxies.sql @@ -0,0 +1,104 @@ +{{ config ( + materialized = 'incremental', + unique_key = ['contract_address','proxy_address'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['abis'] +) }} + +WITH base AS ( + + SELECT + from_address, + to_address, + MIN(block_number) AS start_block, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('silver__traces') }} + WHERE + TYPE = 'DELEGATECALL' + AND trace_status = 'SUCCESS' + AND tx_status = 'SUCCESS' + AND from_address != to_address -- exclude self-calls + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '24 hours' + FROM + {{ this }} +) +{% endif %} +GROUP BY + from_address, + to_address +), +create_id AS ( + SELECT + from_address AS contract_address, + to_address AS proxy_address, + start_block, + CONCAT( + from_address, + '-', + to_address + ) AS _id, + _inserted_timestamp + FROM + base +), +heal AS ( + SELECT + contract_address, + proxy_address, + start_block, + _id, + _inserted_timestamp + FROM + create_id + +{% if is_incremental() %} +UNION ALL +SELECT + contract_address, + proxy_address, + start_block, + _id, + _inserted_timestamp +FROM + {{ this }} + JOIN create_id USING ( + contract_address, + proxy_address + ) +{% endif %} +), +FINAL AS ( + SELECT + contract_address, + proxy_address, + start_block, + _id, + _inserted_timestamp + FROM + heal qualify ROW_NUMBER() over ( + PARTITION BY contract_address, + proxy_address + ORDER BY + start_block ASC + ) = 1 +) +SELECT + f.contract_address, + f.proxy_address, + f.start_block, + f._id, + f._inserted_timestamp, + C.block_number AS created_block, + p.block_number AS proxy_created_block +FROM + FINAL f + JOIN {{ ref('silver__created_contracts') }} C + ON f.contract_address = C.created_contract_address + JOIN {{ ref('silver__created_contracts') }} + p + ON f.proxy_address = p.created_contract_address diff --git a/models/silver/abis/silver__proxies.yml b/models/silver/abis/silver__proxies.yml new file mode 100644 index 0000000..da5fa8f --- /dev/null +++ b/models/silver/abis/silver__proxies.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver__proxies + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _ID \ No newline at end of file diff --git a/models/silver/abis/silver__relevant_abi_contracts.sql b/models/silver/abis/silver__relevant_abi_contracts.sql new file mode 100644 index 0000000..3c0b8a5 --- /dev/null +++ b/models/silver/abis/silver__relevant_abi_contracts.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'table', + unique_key = "contract_address", + tags = ['abis'] +) }} + +WITH base AS ( + + SELECT + contract_address + FROM + {{ ref('silver__relevant_contracts') }} +), +proxies AS ( + SELECT + proxy_address + FROM + {{ ref('silver__proxies') }} + JOIN base USING (contract_address) +) +SELECT + contract_address +FROM + base +UNION +SELECT + proxy_address AS contract_address +FROM + proxies diff --git a/models/silver/abis/silver__relevant_abi_contracts.yml b/models/silver/abis/silver__relevant_abi_contracts.yml new file mode 100644 index 0000000..e3aa647 --- /dev/null +++ b/models/silver/abis/silver__relevant_abi_contracts.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver__relevant_abi_contracts + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS \ No newline at end of file diff --git a/models/silver/core/silver__relevant_contracts.sql b/models/silver/core/silver__relevant_contracts.sql new file mode 100644 index 0000000..acf7bac --- /dev/null +++ b/models/silver/core/silver__relevant_contracts.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'table', + unique_key = "contract_address", + tags = ['non_realtime'] +) }} + +SELECT + contract_address, + 'blast' AS blockchain, + COUNT(*) AS transfers, + MAX(block_number) AS latest_block +FROM + {{ ref('silver__logs') }} +GROUP BY + 1, + 2 +HAVING + COUNT(*) > 25 diff --git a/models/silver/core/silver__relevant_contracts.yml b/models/silver/core/silver__relevant_contracts.yml new file mode 100644 index 0000000..41b1888 --- /dev/null +++ b/models/silver/core/silver__relevant_contracts.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver__relevant_contracts + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - CONTRACT_ADDRESS \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml index df90127..84cd1c1 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -6,11 +6,11 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: b0b5e615143f736d3de249152cda509e47fd21fd + revision: 436e7a90ccb1bfa8d566ab99a807f38ec53f74b7 - package: get-select/dbt_snowflake_query_tags version: 2.3.3 - package: calogica/dbt_date version: 0.7.2 - git: https://github.com/FlipsideCrypto/livequery-models.git - revision: bca494102fbd2d621d32746e9a7fe780678044f8 -sha1_hash: 31b810d35e40945328ca2d93e7c9e190a8901ecf + revision: 992947a4eaa8fccdf2cfcd2cb73a470ff5e89fa2 +sha1_hash: 342c7081a105a2da3cd7f77edf07124c8f0f1c25 diff --git a/packages.yml b/packages.yml index 0c2ea4f..544b4dd 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.16.1 + revision: v1.18.0 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] \ No newline at end of file