mirror of
https://github.com/FlipsideCrypto/sei-models.git
synced 2026-02-06 15:21:52 +00:00
upgrades (#26)
* upgrades * updates * tests * sizes * internal columns * c
This commit is contained in:
Austin
GitHub
parent
33425abedc
commit
bfe1f68341
45
.github/workflows/dbt_test_evm_monthly.yml
vendored
Normal file
45
.github/workflows/dbt_test_evm_monthly.yml
vendored
Normal file
@ -0,0 +1,45 @@
|
||||
name: dbt_test_evm_monthly
|
||||
run-name: dbt_test_evm_monthly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Runs “28th of month at 10AM” (see https://crontab.guru)
|
||||
- cron: '0 10 28 * *'
|
||||
|
||||
env:
|
||||
DBT_PROFILES_DIR: ./
|
||||
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt test -m "sei_models,tag:full_evm_test"
|
||||
46
.github/workflows/dbt_test_evm_recent.yml
vendored
Normal file
46
.github/workflows/dbt_test_evm_recent.yml
vendored
Normal file
@ -0,0 +1,46 @@
|
||||
name: dbt_test_evm_recent
|
||||
run-name: dbt_test_evm_recent
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Daily at 10:30am UTC
|
||||
- cron: '30 10 * * *'
|
||||
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.TEST_WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt test -m models/bronze/core "sei_models,tag:recent_evm_test"
|
||||
121
macros/tests/missing_txs.sql
Normal file
121
macros/tests/missing_txs.sql
Normal file
@ -0,0 +1,121 @@
|
||||
{% macro missing_txs(
|
||||
model
|
||||
) %}
|
||||
WITH txs_base AS (
|
||||
SELECT
|
||||
block_number AS base_block_number,
|
||||
tx_hash AS base_tx_hash
|
||||
FROM
|
||||
{{ ref('test_silver_evm__transactions_full') }}
|
||||
),
|
||||
model_name AS (
|
||||
SELECT
|
||||
block_number AS model_block_number,
|
||||
tx_hash AS model_tx_hash
|
||||
FROM
|
||||
{{ model }}
|
||||
)
|
||||
SELECT
|
||||
base_block_number,
|
||||
base_tx_hash,
|
||||
model_block_number,
|
||||
model_tx_hash
|
||||
FROM
|
||||
txs_base
|
||||
LEFT JOIN model_name
|
||||
ON base_block_number = model_block_number
|
||||
AND base_tx_hash = model_tx_hash
|
||||
WHERE
|
||||
(
|
||||
model_tx_hash IS NULL
|
||||
OR model_block_number IS NULL
|
||||
)
|
||||
{% endmacro %}
|
||||
|
||||
{% macro recent_missing_txs(
|
||||
model
|
||||
) %}
|
||||
WITH txs_base AS (
|
||||
SELECT
|
||||
block_number AS base_block_number,
|
||||
tx_hash AS base_tx_hash
|
||||
FROM
|
||||
{{ ref('test_silver_evm__transactions_recent') }}
|
||||
),
|
||||
model_name AS (
|
||||
SELECT
|
||||
block_number AS model_block_number,
|
||||
tx_hash AS model_tx_hash
|
||||
FROM
|
||||
{{ model }}
|
||||
)
|
||||
SELECT
|
||||
base_block_number,
|
||||
base_tx_hash,
|
||||
model_block_number,
|
||||
model_tx_hash
|
||||
FROM
|
||||
txs_base
|
||||
LEFT JOIN model_name
|
||||
ON base_block_number = model_block_number
|
||||
AND base_tx_hash = model_tx_hash
|
||||
WHERE
|
||||
model_tx_hash IS NULL
|
||||
OR model_block_number IS NULL
|
||||
{% endmacro %}
|
||||
|
||||
{% macro missing_confirmed_txs(
|
||||
model1,
|
||||
model2
|
||||
) %}
|
||||
WITH txs_base AS (
|
||||
SELECT
|
||||
block_number AS base_block_number,
|
||||
block_hash AS base_block_hash,
|
||||
tx_hash AS base_tx_hash
|
||||
FROM
|
||||
{{ model1 }}
|
||||
),
|
||||
model_name AS (
|
||||
SELECT
|
||||
block_number AS model_block_number,
|
||||
block_hash AS model_block_hash,
|
||||
tx_hash AS model_tx_hash
|
||||
FROM
|
||||
{{ model2 }}
|
||||
)
|
||||
SELECT
|
||||
DISTINCT base_block_number AS block_number
|
||||
FROM
|
||||
txs_base
|
||||
LEFT JOIN model_name
|
||||
ON base_block_number = model_block_number
|
||||
AND base_tx_hash = model_tx_hash
|
||||
AND base_block_hash = model_block_hash
|
||||
WHERE
|
||||
model_tx_hash IS NULL
|
||||
AND model_block_number <= (
|
||||
SELECT
|
||||
MAX(base_block_number)
|
||||
FROM
|
||||
txs_base
|
||||
)
|
||||
{% endmacro %}
|
||||
|
||||
{% macro missing_traces(
|
||||
model1,
|
||||
model2
|
||||
) %}
|
||||
SELECT
|
||||
DISTINCT block_number
|
||||
FROM
|
||||
{{ model1 }}
|
||||
tx
|
||||
LEFT JOIN {{ model2 }}
|
||||
tr USING (
|
||||
block_number,
|
||||
tx_hash
|
||||
)
|
||||
WHERE
|
||||
tr.tx_hash IS NULL
|
||||
{% endmacro %}
|
||||
126
models/evm/bronze/api_udf/bronze_evm_api__token_reads.sql
Normal file
126
models/evm/bronze/api_udf/bronze_evm_api__token_reads.sql
Normal file
@ -0,0 +1,126 @@
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
unique_key = "contract_address",
|
||||
full_refresh = false,
|
||||
tags = ['core','recent_evm_test']
|
||||
) }}
|
||||
|
||||
WITH base AS (
|
||||
|
||||
SELECT
|
||||
contract_address,
|
||||
latest_event_block AS latest_block
|
||||
FROM
|
||||
{{ ref('silver_evm__relevant_contracts') }}
|
||||
WHERE
|
||||
total_event_count >= 25
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND contract_address NOT IN (
|
||||
SELECT
|
||||
contract_address
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% endif %}
|
||||
ORDER BY
|
||||
total_event_count DESC
|
||||
LIMIT
|
||||
200
|
||||
), function_sigs AS (
|
||||
SELECT
|
||||
'0x313ce567' AS function_sig,
|
||||
'decimals' AS function_name
|
||||
UNION ALL
|
||||
SELECT
|
||||
'0x06fdde03',
|
||||
'name'
|
||||
UNION ALL
|
||||
SELECT
|
||||
'0x95d89b41',
|
||||
'symbol'
|
||||
),
|
||||
all_reads AS (
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
base
|
||||
JOIN function_sigs
|
||||
ON 1 = 1
|
||||
),
|
||||
ready_reads AS (
|
||||
SELECT
|
||||
contract_address,
|
||||
latest_block,
|
||||
function_sig,
|
||||
RPAD(
|
||||
function_sig,
|
||||
64,
|
||||
'0'
|
||||
) AS input,
|
||||
utils.udf_json_rpc_call(
|
||||
'eth_call',
|
||||
[{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(latest_block)],
|
||||
concat_ws(
|
||||
'-',
|
||||
contract_address,
|
||||
input,
|
||||
latest_block
|
||||
)
|
||||
) AS rpc_request
|
||||
FROM
|
||||
all_reads
|
||||
),
|
||||
batch_reads AS (
|
||||
SELECT
|
||||
ARRAY_AGG(rpc_request) AS batch_rpc_request
|
||||
FROM
|
||||
ready_reads
|
||||
),
|
||||
node_call AS (
|
||||
SELECT
|
||||
*,
|
||||
live.udf_api(
|
||||
'POST',
|
||||
'{Service}/{Authentication}',{},
|
||||
batch_rpc_request,
|
||||
'Vault/prod/sei/quicknode/mainnet'
|
||||
) AS response
|
||||
FROM
|
||||
batch_reads
|
||||
WHERE
|
||||
EXISTS (
|
||||
SELECT
|
||||
1
|
||||
FROM
|
||||
ready_reads
|
||||
LIMIT
|
||||
1
|
||||
)
|
||||
), flat_responses AS (
|
||||
SELECT
|
||||
VALUE :id :: STRING AS call_id,
|
||||
VALUE :result :: STRING AS read_result
|
||||
FROM
|
||||
node_call,
|
||||
LATERAL FLATTEN (
|
||||
input => response :data
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
SPLIT_PART(
|
||||
call_id,
|
||||
'-',
|
||||
1
|
||||
) AS contract_address,
|
||||
SPLIT_PART(
|
||||
call_id,
|
||||
'-',
|
||||
3
|
||||
) AS block_number,
|
||||
LEFT(SPLIT_PART(call_id, '-', 2), 10) AS function_sig,
|
||||
NULL AS function_input,
|
||||
read_result,
|
||||
SYSDATE() :: TIMESTAMP AS _inserted_timestamp
|
||||
FROM
|
||||
flat_responses
|
||||
18
models/evm/bronze/api_udf/bronze_evm_api__token_reads.yml
Normal file
18
models/evm/bronze/api_udf/bronze_evm_api__token_reads.yml
Normal file
@ -0,0 +1,18 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: bronze_evm_api__token_reads
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- CONTRACT_ADDRESS
|
||||
- FUNCTION_SIG
|
||||
columns:
|
||||
- name: _INSERTED_TIMESTAMP
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 1
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
56
models/evm/silver/core/silver_evm__confirmed_blocks.sql
Normal file
56
models/evm/silver/core/silver_evm__confirmed_blocks.sql
Normal file
@ -0,0 +1,56 @@
|
||||
-- depends_on: {{ ref('bronze_evm__streamline_confirm_blocks') }}
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
incremental_strategy = 'delete+insert',
|
||||
unique_key = "block_number",
|
||||
cluster_by = "round(block_number,-3)",
|
||||
tags = ['core']
|
||||
) }}
|
||||
|
||||
WITH base AS (
|
||||
|
||||
SELECT
|
||||
VALUE :BLOCK_NUMBER :: INT AS block_number,
|
||||
DATA :result :hash :: STRING AS block_hash,
|
||||
DATA :result :transactions txs,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
|
||||
{% if is_incremental() %}
|
||||
{{ ref('bronze_evm__streamline_confirm_blocks') }}
|
||||
WHERE
|
||||
_inserted_timestamp >= (
|
||||
SELECT
|
||||
IFNULL(
|
||||
MAX(
|
||||
_inserted_timestamp
|
||||
),
|
||||
'1970-01-01' :: TIMESTAMP
|
||||
) _inserted_timestamp
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% else %}
|
||||
{{ ref('bronze_evm__streamline_FR_confirm_blocks') }}
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY block_number
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
)
|
||||
SELECT
|
||||
block_number,
|
||||
block_hash,
|
||||
VALUE :: STRING AS tx_hash,
|
||||
_inserted_timestamp,
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['tx_hash']
|
||||
) }} AS confirmed_blocks_id,
|
||||
SYSDATE() AS inserted_timestamp,
|
||||
SYSDATE() AS modified_timestamp,
|
||||
'{{ invocation_id }}' AS _invocation_id
|
||||
FROM
|
||||
base,
|
||||
LATERAL FLATTEN (
|
||||
input => txs
|
||||
)
|
||||
102
models/evm/silver/core/silver_evm__contracts.sql
Normal file
102
models/evm/silver/core/silver_evm__contracts.sql
Normal file
@ -0,0 +1,102 @@
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
unique_key = 'contract_address',
|
||||
merge_exclude_columns = ["inserted_timestamp"],
|
||||
tags = ['core','recent_evm_test']
|
||||
) }}
|
||||
|
||||
WITH base_metadata AS (
|
||||
|
||||
SELECT
|
||||
contract_address,
|
||||
block_number,
|
||||
function_sig AS function_signature,
|
||||
read_result AS read_output,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
{{ ref('bronze_evm_api__token_reads') }}
|
||||
WHERE
|
||||
read_result IS NOT NULL
|
||||
AND read_result <> '0x'
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND _inserted_timestamp >= (
|
||||
SELECT
|
||||
MAX(
|
||||
_inserted_timestamp
|
||||
)
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% endif %}
|
||||
),
|
||||
token_names AS (
|
||||
SELECT
|
||||
contract_address,
|
||||
block_number,
|
||||
function_signature,
|
||||
read_output,
|
||||
utils.udf_hex_to_string(SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_name
|
||||
FROM
|
||||
base_metadata
|
||||
WHERE
|
||||
function_signature = '0x06fdde03'
|
||||
AND token_name IS NOT NULL),
|
||||
token_symbols AS (
|
||||
SELECT
|
||||
contract_address,
|
||||
block_number,
|
||||
function_signature,
|
||||
read_output,
|
||||
utils.udf_hex_to_string(SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_symbol
|
||||
FROM
|
||||
base_metadata
|
||||
WHERE
|
||||
function_signature = '0x95d89b41'
|
||||
AND token_symbol IS NOT NULL),
|
||||
token_decimals AS (
|
||||
SELECT
|
||||
contract_address,
|
||||
utils.udf_hex_to_int(
|
||||
read_output :: STRING
|
||||
) AS token_decimals,
|
||||
LENGTH(token_decimals) AS dec_length
|
||||
FROM
|
||||
base_metadata
|
||||
WHERE
|
||||
function_signature = '0x313ce567'
|
||||
AND read_output IS NOT NULL
|
||||
AND read_output <> '0x'
|
||||
),
|
||||
contracts AS (
|
||||
SELECT
|
||||
contract_address,
|
||||
MAX(_inserted_timestamp) AS _inserted_timestamp
|
||||
FROM
|
||||
base_metadata
|
||||
GROUP BY
|
||||
1
|
||||
)
|
||||
SELECT
|
||||
c1.contract_address AS contract_address,
|
||||
token_name,
|
||||
token_decimals :: INTEGER AS token_decimals,
|
||||
token_symbol,
|
||||
_inserted_timestamp,
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['c1.contract_address']
|
||||
) }} AS contracts_id,
|
||||
SYSDATE() AS inserted_timestamp,
|
||||
SYSDATE() AS modified_timestamp,
|
||||
'{{ invocation_id }}' AS _invocation_id
|
||||
FROM
|
||||
contracts c1
|
||||
LEFT JOIN token_names
|
||||
ON c1.contract_address = token_names.contract_address
|
||||
LEFT JOIN token_symbols
|
||||
ON c1.contract_address = token_symbols.contract_address
|
||||
LEFT JOIN token_decimals
|
||||
ON c1.contract_address = token_decimals.contract_address
|
||||
AND dec_length < 3 qualify(ROW_NUMBER() over(PARTITION BY c1.contract_address
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
24
models/evm/silver/core/silver_evm__contracts.yml
Normal file
24
models/evm/silver/core/silver_evm__contracts.yml
Normal file
@ -0,0 +1,24 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: silver_evm__contracts
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- CONTRACT_ADDRESS
|
||||
columns:
|
||||
- name: CONTRACT_ADDRESS
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- STRING
|
||||
- VARCHAR
|
||||
- name: _INSERTED_TIMESTAMP
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
43
models/evm/silver/core/silver_evm__created_contracts.sql
Normal file
43
models/evm/silver/core/silver_evm__created_contracts.sql
Normal file
@ -0,0 +1,43 @@
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "created_contract_address",
|
||||
merge_exclude_columns = ["inserted_timestamp"],
|
||||
tags = ['recent_evm_test']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
block_number,
|
||||
block_timestamp,
|
||||
tx_hash,
|
||||
to_address AS created_contract_address,
|
||||
from_address AS creator_address,
|
||||
input AS created_contract_input,
|
||||
_inserted_timestamp,
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['to_address']
|
||||
) }} AS created_contracts_id,
|
||||
SYSDATE() AS inserted_timestamp,
|
||||
SYSDATE() AS modified_timestamp,
|
||||
'{{ invocation_id }}' AS _invocation_id
|
||||
FROM
|
||||
{{ ref('silver_evm__traces') }}
|
||||
WHERE
|
||||
TYPE ILIKE 'create%'
|
||||
AND to_address IS NOT NULL
|
||||
AND input IS NOT NULL
|
||||
AND input != '0x'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND trace_status = 'SUCCESS'
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND _inserted_timestamp >= (
|
||||
SELECT
|
||||
MAX(_inserted_timestamp) - INTERVAL '24 hours'
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over(PARTITION BY created_contract_address
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
29
models/evm/silver/core/silver_evm__created_contracts.yml
Normal file
29
models/evm/silver/core/silver_evm__created_contracts.yml
Normal file
@ -0,0 +1,29 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: silver_evm__created_contracts
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- created_contract_address
|
||||
columns:
|
||||
- name: BLOCK_TIMESTAMP
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 3
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- TIMESTAMP_LTZ
|
||||
- name: _INSERTED_TIMESTAMP
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 3
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- TIMESTAMP_LTZ
|
||||
|
||||
145
models/evm/silver/core/silver_evm__relevant_contracts.sql
Normal file
145
models/evm/silver/core/silver_evm__relevant_contracts.sql
Normal file
@ -0,0 +1,145 @@
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
unique_key = "contract_address",
|
||||
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)",
|
||||
tags = ['recent_evm_test', 'core'],
|
||||
merge_exclude_columns = ["inserted_timestamp"]
|
||||
) }}
|
||||
|
||||
WITH emitted_events AS (
|
||||
|
||||
SELECT
|
||||
contract_address,
|
||||
COUNT(*) AS event_count,
|
||||
MAX(_inserted_timestamp) AS max_inserted_timestamp_logs,
|
||||
MAX(block_number) AS latest_event_block
|
||||
FROM
|
||||
{{ ref('silver_evm__logs') }}
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
_inserted_timestamp > (
|
||||
SELECT
|
||||
MAX(max_inserted_timestamp_logs)
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% endif %}
|
||||
GROUP BY
|
||||
contract_address
|
||||
),
|
||||
function_calls AS (
|
||||
SELECT
|
||||
IFF(
|
||||
TYPE = 'DELEGATECALL',
|
||||
from_address,
|
||||
to_address
|
||||
) AS contract_address,
|
||||
COUNT(*) AS function_call_count,
|
||||
MAX(_inserted_timestamp) AS max_inserted_timestamp_traces,
|
||||
MAX(block_number) AS latest_call_block
|
||||
FROM
|
||||
{{ ref('silver_evm__traces') }}
|
||||
WHERE
|
||||
tx_status = 'SUCCESS'
|
||||
AND trace_status = 'SUCCESS'
|
||||
AND to_address IS NOT NULL
|
||||
AND input IS NOT NULL
|
||||
AND input <> '0x'
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND _inserted_timestamp > (
|
||||
SELECT
|
||||
MAX(max_inserted_timestamp_traces)
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% endif %}
|
||||
GROUP BY
|
||||
1
|
||||
),
|
||||
active_contracts AS (
|
||||
SELECT
|
||||
contract_address
|
||||
FROM
|
||||
emitted_events
|
||||
UNION
|
||||
SELECT
|
||||
contract_address
|
||||
FROM
|
||||
function_calls
|
||||
),
|
||||
previous_totals AS (
|
||||
|
||||
{% if is_incremental() %}
|
||||
SELECT
|
||||
contract_address, total_event_count, total_call_count, max_inserted_timestamp_logs, latest_event_block, max_inserted_timestamp_traces, latest_call_block
|
||||
FROM
|
||||
{{ this }}
|
||||
{% else %}
|
||||
SELECT
|
||||
NULL AS contract_address, 0 AS total_event_count, 0 AS total_call_count, '1970-01-01 00:00:00' AS max_inserted_timestamp_logs, 0 AS latest_event_block, '1970-01-01 00:00:00' AS max_inserted_timestamp_traces, 0 AS latest_call_block
|
||||
{% endif %})
|
||||
SELECT
|
||||
C.contract_address,
|
||||
COALESCE(
|
||||
p.total_event_count,
|
||||
0
|
||||
) + COALESCE(
|
||||
e.event_count,
|
||||
0
|
||||
) AS total_event_count,
|
||||
COALESCE(
|
||||
p.total_call_count,
|
||||
0
|
||||
) + COALESCE(
|
||||
f.function_call_count,
|
||||
0
|
||||
) AS total_call_count,
|
||||
COALESCE(
|
||||
p.total_event_count,
|
||||
0
|
||||
) + COALESCE(
|
||||
e.event_count,
|
||||
0
|
||||
) + COALESCE(
|
||||
p.total_call_count,
|
||||
0
|
||||
) + COALESCE(
|
||||
f.function_call_count,
|
||||
0
|
||||
) AS total_interaction_count,
|
||||
COALESCE(
|
||||
e.max_inserted_timestamp_logs,
|
||||
p.max_inserted_timestamp_logs,
|
||||
'1970-01-01 00:00:00'
|
||||
) AS max_inserted_timestamp_logs,
|
||||
COALESCE(
|
||||
f.max_inserted_timestamp_traces,
|
||||
p.max_inserted_timestamp_traces,
|
||||
'1970-01-01 00:00:00'
|
||||
) AS max_inserted_timestamp_traces,
|
||||
COALESCE(
|
||||
e.latest_event_block,
|
||||
p.latest_event_block,
|
||||
0
|
||||
) AS latest_event_block,
|
||||
COALESCE(
|
||||
f.latest_call_block,
|
||||
p.latest_call_block,
|
||||
0
|
||||
) AS latest_call_block,
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['c.contract_address']
|
||||
) }} AS relevant_contracts_id,
|
||||
SYSDATE() AS inserted_timestamp,
|
||||
SYSDATE() AS modified_timestamp,
|
||||
'{{ invocation_id }}' AS _invocation_id
|
||||
FROM
|
||||
active_contracts C
|
||||
LEFT JOIN emitted_events e
|
||||
ON C.contract_address = e.contract_address
|
||||
LEFT JOIN function_calls f
|
||||
ON C.contract_address = f.contract_address
|
||||
LEFT JOIN previous_totals p
|
||||
ON C.contract_address = p.contract_address
|
||||
@ -0,0 +1,7 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: silver_evm__relevant_contracts
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- CONTRACT_ADDRESS
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['full_test']
|
||||
tags = ['full_evm_test']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['recent_test']
|
||||
tags = ['recent_evm_test']
|
||||
) }}
|
||||
|
||||
WITH last_3_days AS (
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['full_test']
|
||||
tags = ['full_evm_test']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['recent_test']
|
||||
tags = ['recent_evm_test']
|
||||
) }}
|
||||
|
||||
WITH last_3_days AS (
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['full_test']
|
||||
tags = ['full_evm_test']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['recent_test']
|
||||
tags = ['recent_evm_test']
|
||||
) }}
|
||||
|
||||
WITH last_3_days AS (
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['full_test']
|
||||
tags = ['full_evm_test']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['recent_test']
|
||||
tags = ['recent_evm_test']
|
||||
) }}
|
||||
|
||||
WITH last_3_days AS (
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['full_test']
|
||||
tags = ['full_evm_test']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{{ config (
|
||||
materialized = 'view',
|
||||
tags = ['recent_test']
|
||||
tags = ['recent_evm_test']
|
||||
) }}
|
||||
|
||||
WITH last_3_days AS (
|
||||
|
||||
@ -1,7 +1,11 @@
|
||||
{{ config (
|
||||
materialized = "ephemeral"
|
||||
) }}
|
||||
{# replace this with regular logic after silver is built#}
|
||||
|
||||
SELECT
|
||||
79123881 AS block_number
|
||||
MIN(block_number) AS block_number
|
||||
FROM
|
||||
{{ ref("silver_evm__blocks") }}
|
||||
WHERE
|
||||
block_timestamp >= DATEADD('hour', -72, TRUNCATE(SYSDATE(), 'HOUR'))
|
||||
AND block_timestamp < DATEADD('hour', -71, TRUNCATE(SYSDATE(), 'HOUR'))
|
||||
|
||||
37
models/evm/streamline/_max_evm_block_by_hour.sql
Normal file
37
models/evm/streamline/_max_evm_block_by_hour.sql
Normal file
@ -0,0 +1,37 @@
|
||||
{{ config (
|
||||
materialized = "ephemeral"
|
||||
) }}
|
||||
|
||||
WITH base AS (
|
||||
|
||||
SELECT
|
||||
DATE_TRUNC(
|
||||
'hour',
|
||||
block_timestamp
|
||||
) AS block_hour,
|
||||
MAX(block_number) block_number
|
||||
FROM
|
||||
{{ ref("silver_evm__blocks") }}
|
||||
WHERE
|
||||
block_timestamp > DATEADD(
|
||||
'day',
|
||||
-5,
|
||||
CURRENT_DATE
|
||||
)
|
||||
GROUP BY
|
||||
1
|
||||
)
|
||||
SELECT
|
||||
block_hour,
|
||||
block_number
|
||||
FROM
|
||||
base
|
||||
WHERE
|
||||
block_hour <> (
|
||||
SELECT
|
||||
MAX(
|
||||
block_hour
|
||||
)
|
||||
FROM
|
||||
base
|
||||
)
|
||||
@ -0,0 +1,35 @@
|
||||
-- depends_on: {{ ref('bronze_evm__streamline_confirm_blocks') }}
|
||||
{{ config (
|
||||
materialized = "incremental",
|
||||
unique_key = "block_number",
|
||||
cluster_by = "ROUND(block_number, -3)",
|
||||
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)",
|
||||
tags = ['streamline_core_evm_complete']
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
VALUE :BLOCK_NUMBER :: INT AS block_number,
|
||||
{{ dbt_utils.generate_surrogate_key(
|
||||
['block_number']
|
||||
) }} AS complete_blocks_evm_id,
|
||||
SYSDATE() AS inserted_timestamp,
|
||||
SYSDATE() AS modified_timestamp,
|
||||
_inserted_timestamp,
|
||||
'{{ invocation_id }}' AS _invocation_id
|
||||
FROM
|
||||
|
||||
{% if is_incremental() %}
|
||||
{{ ref('bronze_evm__streamline_confirm_blocks') }}
|
||||
WHERE
|
||||
_inserted_timestamp >= (
|
||||
SELECT
|
||||
COALESCE(MAX(_inserted_timestamp), '1970-01-01' :: TIMESTAMP) _inserted_timestamp
|
||||
FROM
|
||||
{{ this }})
|
||||
{% else %}
|
||||
{{ ref('bronze_evm__streamline_FR_confirm_blocks') }}
|
||||
{% endif %}
|
||||
|
||||
qualify(ROW_NUMBER() over (PARTITION BY block_number
|
||||
ORDER BY
|
||||
_inserted_timestamp DESC)) = 1
|
||||
@ -0,0 +1,108 @@
|
||||
{{ config (
|
||||
materialized = "view",
|
||||
post_hook = fsc_utils.if_data_call_function_v2(
|
||||
func = 'streamline.udf_bulk_rest_api_v2',
|
||||
target = "{{this.schema}}.{{this.identifier}}",
|
||||
params ={ "external_table" :"evm_confirm_blocks",
|
||||
"sql_limit" :"25000",
|
||||
"producer_batch_size" :"10000",
|
||||
"worker_batch_size" :"5000",
|
||||
"sql_source" :"{{this.identifier}}" }
|
||||
),
|
||||
tags = ['streamline_core_evm_realtime']
|
||||
) }}
|
||||
|
||||
WITH last_3_days AS (
|
||||
{#
|
||||
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("_evm_block_lookback") }}
|
||||
#}
|
||||
SELECT
|
||||
79123881 AS block_number
|
||||
),
|
||||
look_back AS (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("_max_evm_block_by_hour") }}
|
||||
qualify ROW_NUMBER() over (
|
||||
ORDER BY
|
||||
block_number DESC
|
||||
) = 6
|
||||
),
|
||||
to_do AS (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__evm_blocks") }}
|
||||
WHERE
|
||||
block_number IS NOT NULL
|
||||
AND block_number <= (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
look_back
|
||||
)
|
||||
AND block_number >= (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
last_3_days
|
||||
)
|
||||
EXCEPT
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("streamline__complete_evm_confirmed_blocks") }}
|
||||
WHERE
|
||||
block_number IS NOT NULL
|
||||
AND block_number <= (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
look_back
|
||||
)
|
||||
AND block_number >= (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
last_3_days
|
||||
)
|
||||
),
|
||||
ready_blocks AS (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
to_do
|
||||
)
|
||||
SELECT
|
||||
block_number,
|
||||
ROUND(
|
||||
block_number,
|
||||
-3
|
||||
) :: INT AS partition_key,
|
||||
{{ target.database }}.live.udf_api(
|
||||
'POST',
|
||||
'{Service}/{Authentication}',
|
||||
OBJECT_CONSTRUCT(
|
||||
'Content-Type',
|
||||
'application/json'
|
||||
),
|
||||
OBJECT_CONSTRUCT(
|
||||
'id',
|
||||
block_number :: STRING,
|
||||
'jsonrpc',
|
||||
'2.0',
|
||||
'method',
|
||||
'eth_getBlockByNumber',
|
||||
'params',
|
||||
ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)),
|
||||
'Vault/prod/sei/quicknode/mainnet'
|
||||
) AS request
|
||||
FROM
|
||||
ready_blocks
|
||||
ORDER BY
|
||||
block_number ASC
|
||||
@ -57,7 +57,12 @@ ready_blocks AS (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
to_do {# add retry here #}
|
||||
to_do
|
||||
UNION
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("_missing_receipts") }}
|
||||
)
|
||||
SELECT
|
||||
block_number,
|
||||
|
||||
@ -58,6 +58,11 @@ ready_blocks AS (
|
||||
block_number
|
||||
FROM
|
||||
to_do
|
||||
UNION
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("_missing_traces") }}
|
||||
)
|
||||
SELECT
|
||||
block_number,
|
||||
|
||||
@ -57,7 +57,12 @@ ready_blocks AS (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
to_do {# add retry here #}
|
||||
to_do
|
||||
UNION
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("_missing_receipts") }}
|
||||
)
|
||||
SELECT
|
||||
block_number,
|
||||
|
||||
34
models/evm/streamline/silver/retry/_missing_receipts.sql
Normal file
34
models/evm/streamline/silver/retry/_missing_receipts.sql
Normal file
@ -0,0 +1,34 @@
|
||||
{{ config (
|
||||
materialized = "ephemeral"
|
||||
) }}
|
||||
|
||||
WITH lookback AS (
|
||||
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("_evm_block_lookback") }}
|
||||
)
|
||||
SELECT
|
||||
DISTINCT t.block_number AS block_number
|
||||
FROM
|
||||
{{ ref("silver_evm__transactions") }}
|
||||
t
|
||||
LEFT JOIN {{ ref("silver_evm__receipts") }}
|
||||
r USING (
|
||||
block_number,
|
||||
block_hash,
|
||||
tx_hash
|
||||
)
|
||||
WHERE
|
||||
r.tx_hash IS NULL
|
||||
AND t.block_number >= (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
lookback
|
||||
)
|
||||
AND t.block_timestamp >= DATEADD('hour', -84, SYSDATE())
|
||||
AND (
|
||||
r._inserted_timestamp >= DATEADD('hour', -84, SYSDATE())
|
||||
OR r._inserted_timestamp IS NULL)
|
||||
30
models/evm/streamline/silver/retry/_missing_traces.sql
Normal file
30
models/evm/streamline/silver/retry/_missing_traces.sql
Normal file
@ -0,0 +1,30 @@
|
||||
{{ config (
|
||||
materialized = "ephemeral"
|
||||
) }}
|
||||
|
||||
WITH lookback AS (
|
||||
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
{{ ref("_evm_block_lookback") }}
|
||||
)
|
||||
SELECT
|
||||
DISTINCT tx.block_number block_number
|
||||
FROM
|
||||
{{ ref("silver_evm__transactions") }}
|
||||
tx
|
||||
LEFT JOIN {{ ref("silver_evm__traces") }}
|
||||
tr
|
||||
ON tx.block_number = tr.block_number
|
||||
AND tx.tx_hash = tr.tx_hash
|
||||
AND tr.block_timestamp >= DATEADD('hour', -84, SYSDATE())
|
||||
WHERE
|
||||
tx.block_timestamp >= DATEADD('hour', -84, SYSDATE())
|
||||
AND tr.tx_hash IS NULL
|
||||
AND tx.block_number >= (
|
||||
SELECT
|
||||
block_number
|
||||
FROM
|
||||
lookback
|
||||
)
|
||||
@ -0,0 +1,2 @@
|
||||
-- depends_on: {{ ref('test_silver_evm__transactions_recent') }}
|
||||
{{ recent_missing_txs(ref("test_silver_evm__receipts_recent")) }}
|
||||
1
tests/sei_evm/test_silver_evm__recent_missing_traces.sql
Normal file
1
tests/sei_evm/test_silver_evm__recent_missing_traces.sql
Normal file
@ -0,0 +1 @@
|
||||
{{ missing_traces(ref("test_silver_evm__transactions_recent"), ref("test_silver_evm__traces_recent")) }}
|
||||
Loading…
Reference in New Issue
Block a user