diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml new file mode 100644 index 0000000..da996d8 --- /dev/null +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -0,0 +1,46 @@ +name: dbt_alter_gha_task +run-name: dbt_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + with: + workflow_name: | + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} + environment: workflow_prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml new file mode 100644 index 0000000..addc968 --- /dev/null +++ b/.github/workflows/dbt_docs_update.yml @@ -0,0 +1,27 @@ +name: docs_update + +on: + push: + branches: + - "main" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 757ff1c..4fca74c 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -1,5 +1,5 @@ name: dbt_run_adhoc -run-name: dbt_run_adhoc +run-name: ${{ inputs.dbt_command }} on: workflow_dispatch: diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index 77ba297..c28db06 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -1,5 +1,5 @@ name: dbt_run_deployment -run-name: dbt_run_deployment +run-name: ${{ inputs.dbt_command }} on: workflow_dispatch: diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml new file mode 100644 index 0000000..5e08079 --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -0,0 +1,45 @@ +name: dbt_run_dev_refresh +run-name: dbt_run_dev_refresh + +on: + workflow_dispatch: + schedule: + # Runs "at 9:00 UTC" (see https://crontab.guru) + - cron: '0 9 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone \ No newline at end of file diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml new file mode 100644 index 0000000..a3b1c0a --- /dev/null +++ b/.github/workflows/dbt_run_full_observability.yml @@ -0,0 +1,48 @@ +name: dbt_run_full_observability +run-name: dbt_run_full_observability + +on: + workflow_dispatch: + schedule: + # Runs “At 16:00 on day-of-month 1.” (see https://crontab.guru) + - cron: '0 16 1 * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_2xl + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "blast_models,tag:observability" + + + diff --git a/.github/workflows/dbt_run_scheduled_non_realtime.yml b/.github/workflows/dbt_run_scheduled_non_realtime.yml new file mode 100644 index 0000000..8579184 --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_non_realtime.yml @@ -0,0 +1,44 @@ +name: dbt_run_scheduled_non_realtime +run-name: dbt_run_scheduled_non_realtime + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "blast_models,tag:non_realtime" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml new file mode 100644 index 0000000..1034e9c --- /dev/null +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -0,0 +1,44 @@ +name: dbt_run_streamline_chainhead +run-name: dbt_run_streamline_chainhead + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "blast_models,tag:streamline_core_complete" "blast_models,tag:streamline_core_realtime" diff --git a/.github/workflows/dbt_run_streamline_history_adhoc.yml b/.github/workflows/dbt_run_streamline_history_adhoc.yml new file mode 100644 index 0000000..9b60d31 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_history_adhoc.yml @@ -0,0 +1,69 @@ +name: dbt_run_streamline_history_adhoc +run-name: dbt_run_streamline_history_adhoc + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + - prod_backfill + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: choice + description: 'DBT Run Command' + required: true + options: + - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "blast_models,tag:streamline_core_history" "blast_models,tag:streamline_core_complete" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.warehouse }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} \ No newline at end of file diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml new file mode 100644 index 0000000..31683b1 --- /dev/null +++ b/.github/workflows/dbt_test_daily.yml @@ -0,0 +1,48 @@ +name: dbt_test_daily +run-name: dbt_test_daily + +on: + workflow_dispatch: + schedule: + # Runs "at 9:00 UTC" (see https://crontab.guru) + - cron: '0 9 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_test + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test --exclude "blast_models,tag:full_test" "blast_models,tag:recent_test" "blast_models,tag:gha_tasks" livequery_models + + + diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml new file mode 100644 index 0000000..5a33e97 --- /dev/null +++ b/.github/workflows/dbt_test_intraday.yml @@ -0,0 +1,49 @@ +name: dbt_test_intraday +run-name: dbt_test_intraday + +on: + workflow_dispatch: + schedule: + # Runs “At minute 5 past every 4th hour.” (see https://crontab.guru) + - cron: '5 */4 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_test + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "blast_models,tag:observability" + dbt test -m "blast_models,tag:recent_test" + + + diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml new file mode 100644 index 0000000..dce2945 --- /dev/null +++ b/.github/workflows/dbt_test_monthly.yml @@ -0,0 +1,48 @@ +name: dbt_test_monthly +run-name: dbt_test_monthly + +on: + workflow_dispatch: + schedule: + # Runs “At 7pm on the 28th of month.” (see https://crontab.guru) + - cron: '0 19 28 * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_test + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m "blast_models,tag:full_test" + + + diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml new file mode 100644 index 0000000..976440b --- /dev/null +++ b/.github/workflows/dbt_test_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_test_tasks +run-name: dbt_test_tasks + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index b17ab47..c7fccc6 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,6 +1,4 @@ workflow_name,workflow_schedule -dbt_run_scheduled_non_realtime,"22,52 * * * *" -dbt_run_streamline_chainhead,"15,45 * * * *" -dbt_run_streamline_decoder,"0,30 * * * *" -dbt_run_scheduled_curated,"40 * * * *" -dbt_test_tasks,"15 * * * *" \ No newline at end of file +dbt_run_scheduled_non_realtime,"17,47 * * * *" +dbt_run_streamline_chainhead,"10,40 * * * *" +dbt_test_tasks,"10 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index d977862..373b852 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -28,7 +28,7 @@ tests: +store_failures: true # all tests on-run-start: - # - "{{ create_sps() }}" + - "{{ create_sps() }}" - "{{ create_udfs() }}" on-run-end: diff --git a/macros/create_sps.sql b/macros/create_sps.sql index fe4a5cf..2f5e3db 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,8 +1,8 @@ -{# {% macro create_sps() %} +{% macro create_sps() %} {% if var("UPDATE_UDFS_AND_SPS") %} {% if target.database == 'BLAST' %} CREATE schema IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; {% endif %} {% endif %} -{% endmacro %} #} +{% endmacro %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 82313e6..be8e2af 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,5 +1,15 @@ {% macro create_udfs() %} {% if var("UPDATE_UDFS_AND_SPS") %} + {% set sql %} + CREATE schema if NOT EXISTS silver; + + {{ create_udtf_get_base_table( + schema = "streamline" + ) }} + {{ create_udf_rest_api() }} + + {% endset %} + {% do run_query(sql) %} {{- fsc_utils.create_udfs() -}} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql new file mode 100644 index 0000000..ca08ae5 --- /dev/null +++ b/macros/streamline/api_integrations.sql @@ -0,0 +1,22 @@ +{% macro create_aws_base_api() %} + {{ log( + "Creating integration for target:" ~ target + ) }} + {# {% if target.name == "prod" %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_blast_api api_provider = aws_api_gateway api_aws_role_arn = 'insert-prod-arn-here' api_allowed_prefixes = ( + 'insert-prod-url-here' + ) enabled = TRUE; +{% endset %} + {% do run_query(sql) %} + #} + {% if target.name == "dev" %} + --replace if with elif after prod is deployed + {% set sql %} + CREATE api integration IF NOT EXISTS aws_blast_api_dev api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::704693948482:role/blast-api-dev-rolesnowflakeudfsAF733095-Wtkj0DGJ7lOQ' api_allowed_prefixes = ( + 'https://05340o05al.execute-api.us-east-1.amazonaws.com/dev/' + ) enabled = TRUE; +{% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} diff --git a/macros/streamline/get_base_table_udtf.sql b/macros/streamline/get_base_table_udtf.sql new file mode 100644 index 0000000..a488d14 --- /dev/null +++ b/macros/streamline/get_base_table_udtf.sql @@ -0,0 +1,24 @@ +{% macro create_udtf_get_base_table(schema) %} +create or replace function {{ schema }}.udtf_get_base_table(max_height integer) +returns table (height number) +as +$$ + with base as ( + select + row_number() over ( + order by + seq4() + ) as id + from + table(generator(rowcount => 100000000)) + ) +select + id as height +from + base +where + id <= max_height +$$ +; + +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql new file mode 100644 index 0000000..e2d2444 --- /dev/null +++ b/macros/streamline/models.sql @@ -0,0 +1,117 @@ +{% macro streamline_external_table_query( + model, + partition_function, + partition_name, + unique_key + ) %} + WITH meta AS ( + SELECT + last_modified AS _inserted_timestamp, + file_name, + {{ partition_function }} AS {{ partition_name }} + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", model) }}') + ) A + ) + SELECT + {{ unique_key }} AS block_number, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text + ) + ) AS id, + s.{{ partition_name }}, + s.value AS VALUE + FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.{{ partition_name }} = s.{{ partition_name }} + WHERE + b.{{ partition_name }} = s.{{ partition_name }} + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010', + '-32608' + ) + ) +{% endmacro %} + +{% macro streamline_external_table_FR_query( + model, + partition_function, + partition_name, + unique_key + ) %} + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + {{ partition_function }} AS {{ partition_name }} + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", model) }}' + ) + ) A + ) +SELECT + {{ unique_key }} AS block_number, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text + ) + ) AS id, + s.{{ partition_name }}, + s.value AS VALUE +FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.{{ partition_name }} = s.{{ partition_name }} +WHERE + b.{{ partition_name }} = s.{{ partition_name }} + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010', + '-32608' + ) + ) +{% endmacro %} diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql new file mode 100644 index 0000000..9067a80 --- /dev/null +++ b/macros/streamline/streamline_udfs.sql @@ -0,0 +1,11 @@ +{% macro create_udf_rest_api() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_rest_api( + json OBJECT + ) returns ARRAY api_integration = + {% if target.name == "prod" %} + aws_blast_api AS 'insert-prod-url-here' + {% else %} + aws_blast_api_dev AS 'https://05340o05al.execute-api.us-east-1.amazonaws.com/dev/udf_bulk_rest_api' + {%- endif %}; +{% endmacro %} \ No newline at end of file diff --git a/macros/utils.sql b/macros/utils.sql index a75daac..0b800b8 100644 --- a/macros/utils.sql +++ b/macros/utils.sql @@ -1,4 +1,4 @@ -{# {% macro if_data_call_function( +{% macro if_data_call_function( func, target ) %} @@ -75,4 +75,4 @@ {% endif %} {% endif %} {% endif %} -{% endmacro %} #} +{% endmacro %} diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql new file mode 100644 index 0000000..a2071b7 --- /dev/null +++ b/models/gold/core/core__fact_blocks.sql @@ -0,0 +1,78 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + A.block_number AS block_number, + block_timestamp, + 'testnet' AS network, + 'blast' AS blockchain, + d.tx_count, + difficulty, + total_difficulty, + extra_data, + gas_limit, + gas_used, + HASH, + parent_hash, + receipts_root, + sha3_uncles, + SIZE, + uncles AS uncle_blocks, + withdrawals_root, + OBJECT_CONSTRUCT( + 'baseFeePerGas', + base_fee_per_gas, + 'difficulty', + difficulty, + 'extraData', + extra_data, + 'gasLimit', + gas_limit, + 'gasUsed', + gas_used, + 'hash', + HASH, + 'logsBloom', + logs_bloom, + 'miner', + miner, + 'nonce', + nonce, + 'number', + NUMBER, + 'parentHash', + parent_hash, + 'receiptsRoot', + receipts_root, + 'sha3Uncles', + sha3_uncles, + 'size', + SIZE, + 'stateRoot', + state_root, + 'timestamp', + block_timestamp, + 'totalDifficulty', + total_difficulty, + 'transactionsRoot', + transactions_root, + 'uncles', + uncles + ) AS block_header_json, + blocks_id AS fact_blocks_id, + GREATEST( + A.inserted_timestamp, + d.inserted_timestamp + ) AS inserted_timestamp, + GREATEST( + A.modified_timestamp, + d.modified_timestamp + ) AS modified_timestamp +FROM + {{ ref('silver__blocks') }} A + LEFT JOIN {{ ref('silver__tx_count') }} + d + ON A.block_number = d.block_number diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml new file mode 100644 index 0000000..d59e122 --- /dev/null +++ b/models/gold/core/core__fact_blocks.yml @@ -0,0 +1,48 @@ +version: 2 +models: + - name: core__fact_blocks + description: '{{ doc("blast_blocks_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("blast_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("blast_block_timestamp") }}' + - name: NETWORK + description: '{{ doc("blast_network") }}' + - name: BLOCKCHAIN + description: '{{ doc("blast_blockchain") }}' + - name: TX_COUNT + description: '{{ doc("blast_tx_count") }}' + - name: DIFFICULTY + description: '{{ doc("blast_difficulty") }}' + - name: TOTAL_DIFFICULTY + description: '{{ doc("blast_total_difficulty") }}' + - name: EXTRA_DATA + description: '{{ doc("blast_extra_data") }}' + - name: GAS_LIMIT + description: '{{ doc("blast_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("blast_gas_used") }}' + - name: HASH + description: '{{ doc("blast_blocks_hash") }}' + - name: PARENT_HASH + description: '{{ doc("blast_parent_hash") }}' + - name: RECEIPTS_ROOT + description: '{{ doc("blast_receipts_root") }}' + - name: SHA3_UNCLES + description: '{{ doc("blast_sha3_uncles") }}' + - name: SIZE + description: '{{ doc("blast_size") }}' + - name: UNCLE_BLOCKS + description: '{{ doc("blast_uncle_blocks") }}' + - name: BLOCK_HEADER_JSON + description: '{{ doc("blast_block_header_json") }}' + - name: FACT_BLOCKS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + - name: WITHDRAWALS_ROOT + description: A hash of the withdrawals that occurred in this block. \ No newline at end of file diff --git a/models/gold/core/core__fact_event_logs.sql b/models/gold/core/core__fact_event_logs.sql new file mode 100644 index 0000000..c0fcf94 --- /dev/null +++ b/models/gold/core/core__fact_event_logs.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + contract_address, + topics, + DATA, + event_removed, + tx_status, + _log_id, + logs_id AS fact_event_logs_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__logs') }} diff --git a/models/gold/core/core__fact_event_logs.yml b/models/gold/core/core__fact_event_logs.yml new file mode 100644 index 0000000..0131cad --- /dev/null +++ b/models/gold/core/core__fact_event_logs.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: core__fact_event_logs + description: '{{ doc("blast_logs_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("blast_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("blast_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("blast_logs_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("blast_event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("blast_logs_contract_address") }}' + - name: TOPICS + description: '{{ doc("blast_topics") }}' + - name: DATA + description: '{{ doc("blast_logs_data") }}' + - name: EVENT_REMOVED + description: '{{ doc("blast_event_removed") }}' + - name: _LOG_ID + description: '{{ doc("internal_column") }}' + - name: TX_STATUS + description: '{{ doc("blast_tx_status") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("blast_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("blast_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("blast_origin_to") }}' + - name: FACT_EVENT_LOGS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql new file mode 100644 index 0000000..eeca29e --- /dev/null +++ b/models/gold/core/core__fact_traces.sql @@ -0,0 +1,32 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + eth_value AS VALUE, + eth_value_precise_raw AS value_precise_raw, + eth_value_precise AS value_precise, + gas, + gas_used, + input, + output, + TYPE, + identifier, + DATA, + tx_status, + sub_traces, + trace_status, + error_reason, + trace_index, + traces_id AS fact_traces_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__traces') }} diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml new file mode 100644 index 0000000..9504e4a --- /dev/null +++ b/models/gold/core/core__fact_traces.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: core__fact_traces + description: '{{ doc("blast_traces_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("blast_traces_block_no") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("blast_traces_blocktime") }}' + - name: TX_HASH + description: '{{ doc("blast_traces_tx_hash") }}' + - name: FROM_ADDRESS + description: '{{ doc("blast_traces_from") }}' + - name: TO_ADDRESS + description: '{{ doc("blast_traces_to") }}' + - name: VALUE + description: '{{ doc("blast_traces_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: GAS + description: '{{ doc("blast_traces_gas") }}' + - name: GAS_USED + description: '{{ doc("blast_traces_gas_used") }}' + - name: INPUT + description: '{{ doc("blast_traces_input") }}' + - name: OUTPUT + description: '{{ doc("blast_traces_output") }}' + - name: TYPE + description: '{{ doc("blast_traces_type") }}' + - name: IDENTIFIER + description: '{{ doc("blast_traces_identifier") }}' + - name: DATA + description: '{{ doc("blast_traces_call_data") }}' + - name: TX_STATUS + description: '{{ doc("blast_tx_status") }}' + - name: SUB_TRACES + description: '{{ doc("blast_traces_sub") }}' + - name: TRACE_STATUS + description: The status of the trace, either `SUCCESS` or `FAIL` + - name: ERROR_REASON + description: The reason for the trace failure, if any. + - name: TRACE_INDEX + description: The index of the trace within the transaction. + - name: FACT_TRACES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + + + + \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql new file mode 100644 index 0000000..12400fe --- /dev/null +++ b/models/gold/core/core__fact_transactions.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + block_hash, + tx_hash, + nonce, + POSITION, + origin_function_signature, + from_address, + to_address, + VALUE, + value_precise_raw, + value_precise, + tx_fee, + tx_fee_precise, + gas_price, + effective_gas_price, + gas AS gas_limit, + gas_used, + l1_gas_price, + l1_gas_used, + l1_fee_scalar, + l1_fee, + l1_fee_precise, + cumulative_gas_used, + max_fee_per_gas, + max_priority_fee_per_gas, + input_data, + tx_status AS status, + r, + s, + v, + deposit_nonce, + deposit_receipt_version, + transactions_id AS fact_transactions_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__transactions') }} diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml new file mode 100644 index 0000000..26f25d5 --- /dev/null +++ b/models/gold/core/core__fact_transactions.yml @@ -0,0 +1,78 @@ +version: 2 +models: + - name: core__fact_transactions + description: '{{ doc("blast_tx_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("blast_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("blast_block_timestamp") }}' + - name: BLOCK_HASH + description: '{{ doc("blast_tx_block_hash") }}' + - name: TX_HASH + description: '{{ doc("blast_tx_hash") }}' + - name: NONCE + description: '{{ doc("blast_tx_nonce") }}' + - name: POSITION + description: '{{ doc("blast_tx_position") }}' + - name: FROM_ADDRESS + description: '{{ doc("blast_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("blast_to_address") }}' + - name: VALUE + description: '{{ doc("blast_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: TX_FEE + description: '{{ doc("blast_tx_fee") }}' + - name: TX_FEE_PRECISE + description: '{{ doc("tx_fee_precise") }}' + - name: GAS_PRICE + description: '{{ doc("blast_tx_gas_price") }}' + - name: EFFECTIVE_GAS_PRICE + description: The total base charge plus tip paid for each unit of gas, in Gwei. + - name: GAS_LIMIT + description: '{{ doc("blast_tx_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("blast_tx_gas_used") }}' + - name: L1_GAS_PRICE + description: '{{ doc("blast_l1_gas_price") }}' + - name: L1_GAS_USED + description: '{{ doc("blast_l1_gas_used") }}' + - name: L1_FEE_SCALAR + description: '{{ doc("blast_l1_fee_scalar") }}' + - name: L1_FEE + description: The L1 portion of fees paid. + - name: L1_FEE_PRECISE + description: '{{ doc("tx_fee_precise") }}' + - name: CUMULATIVE_GAS_USED + description: '{{ doc("blast_cumulative_gas_used") }}' + - name: MAX_FEE_PER_GAS + description: The maximum fee per gas of the transaction, in Gwei. + - name: MAX_PRIORITY_FEE_PER_GAS + description: The maximum priority fee per gas of the transaction, in Gwei. + - name: STATUS + description: '{{ doc("blast_tx_status") }}' + - name: INPUT_DATA + description: '{{ doc("blast_tx_input_data") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("blast_tx_origin_sig") }}' + - name: R + description: The r value of the transaction signature. + - name: S + description: The s value of the transaction signature. + - name: V + description: The v value of the transaction signature. + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + - name: DEPOSIT_NONCE + description: The nonce of the deposit transaction. + - name: DEPOSIT_RECEIPT_VERSION + description: The version of the deposit receipt. \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql new file mode 100644 index 0000000..c5c625e --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -0,0 +1,161 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +blocks AS ( + SELECT + l.block_number, + block_timestamp, + LAG( + l.block_number, + 1 + ) over ( + ORDER BY + l.block_number ASC + ) AS prev_BLOCK_NUMBER + FROM + {{ ref("silver__blocks") }} + l + INNER JOIN block_range b + ON l.block_number = b.block_number + AND l.block_number >= ( + SELECT + MIN(block_number) + FROM + block_range + ) +), +block_gen AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + MIN(block_number) + FROM + blocks + ) + AND ( + SELECT + MAX(block_number) + FROM + blocks + ) +) +SELECT + 'blocks' AS test_name, + MIN( + b.block_number + ) AS min_block, + MAX( + b.block_number + ) AS max_block, + MIN( + b.block_timestamp + ) AS min_block_timestamp, + MAX( + b.block_timestamp + ) AS max_block_timestamp, + COUNT(1) AS blocks_tested, + COUNT( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) AS blocks_impacted_count, + ARRAY_AGG( + CASE + WHEN C.block_number IS NOT NULL THEN A.block_number + END + ) within GROUP ( + ORDER BY + A.block_number + ) AS blocks_impacted_array, + CURRENT_TIMESTAMP AS test_timestamp +FROM + block_gen A + LEFT JOIN blocks b + ON A.block_number = b.block_number + LEFT JOIN blocks C + ON A.block_number > C.prev_block_number + AND A.block_number < C.block_number + AND C.block_number - C.prev_block_number <> 1 +WHERE + COALESCE( + b.block_number, + C.block_number + ) IS NOT NULL diff --git a/models/silver/_observability/silver_observability__logs_completeness.sql b/models/silver/_observability/silver_observability__logs_completeness.sql new file mode 100644 index 0000000..bf88e6f --- /dev/null +++ b/models/silver/_observability/silver_observability__logs_completeness.sql @@ -0,0 +1,119 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__receipts") }} + r + LEFT JOIN {{ ref("silver__logs") }} + l USING ( + block_number, + tx_hash + ) + JOIN block_range USING (block_number) + WHERE + l.tx_hash IS NULL + AND ARRAY_SIZE( + r.logs + ) > 0 +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'event_logs' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__receipts_completeness.sql b/models/silver/_observability/silver_observability__receipts_completeness.sql new file mode 100644 index 0000000..c491166 --- /dev/null +++ b/models/silver/_observability/silver_observability__receipts_completeness.sql @@ -0,0 +1,117 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__transactions") }} + t + LEFT JOIN {{ ref("silver__receipts") }} + r USING ( + block_number, + tx_hash, + block_hash + ) + JOIN block_range USING (block_number) + WHERE + r.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'receipts' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__traces_completeness.sql b/models/silver/_observability/silver_observability__traces_completeness.sql new file mode 100644 index 0000000..a4ca260 --- /dev/null +++ b/models/silver/_observability/silver_observability__traces_completeness.sql @@ -0,0 +1,116 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__transactions") }} + tx + LEFT JOIN {{ ref("silver__traces") }} + tr USING ( + block_number, + tx_hash + ) + JOIN block_range USING (block_number) + WHERE + tr.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'traces' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql new file mode 100644 index 0000000..d755db3 --- /dev/null +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -0,0 +1,117 @@ +{{ config( + materialized = 'incremental', + unique_key = 'test_timestamp', + full_refresh = false, + tags = ['observability'] +) }} + +WITH summary_stats AS ( + + SELECT + MIN(block_number) AS min_block, + MAX(block_number) AS max_block, + MIN(block_timestamp) AS min_block_timestamp, + MAX(block_timestamp) AS max_block_timestamp, + COUNT(1) AS blocks_tested + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + +{% if is_incremental() %} +AND ( + block_number >= ( + SELECT + MIN(block_number) + FROM + ( + SELECT + MIN(block_number) AS block_number + FROM + {{ ref('silver__blocks') }} + WHERE + block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) + AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + UNION + SELECT + MIN(VALUE) - 1 AS block_number + FROM + ( + SELECT + blocks_impacted_array + FROM + {{ this }} + qualify ROW_NUMBER() over ( + ORDER BY + test_timestamp DESC + ) = 1 + ), + LATERAL FLATTEN( + input => blocks_impacted_array + ) + ) + ) {% if var('OBSERV_FULL_TEST') %} + OR block_number >= 0 + {% endif %} +) +{% endif %} +), +block_range AS ( + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id BETWEEN ( + SELECT + min_block + FROM + summary_stats + ) + AND ( + SELECT + max_block + FROM + summary_stats + ) +), +broken_blocks AS ( + SELECT + DISTINCT block_number + FROM + {{ ref("silver__confirmed_blocks") }} + b + LEFT JOIN {{ ref("silver__transactions") }} + t USING ( + block_number, + tx_hash, + block_hash + ) + JOIN block_range USING (block_number) + WHERE + t.tx_hash IS NULL +), +impacted_blocks AS ( + SELECT + COUNT(1) AS blocks_impacted_count, + ARRAY_AGG(block_number) within GROUP ( + ORDER BY + block_number + ) AS blocks_impacted_array + FROM + broken_blocks +) +SELECT + 'transactions' AS test_name, + min_block, + max_block, + min_block_timestamp, + max_block_timestamp, + blocks_tested, + blocks_impacted_count, + blocks_impacted_array, + CURRENT_TIMESTAMP() AS test_timestamp +FROM + summary_stats + JOIN impacted_blocks + ON 1 = 1 diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index c4b4749..f47c400 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_blocks') }} +-- depends_on: {{ ref('bronze__streamline_blocks_testnet') }} {{ config( materialized = 'incremental', unique_key = "block_number", @@ -12,48 +12,48 @@ SELECT DATA, block_number, utils.udf_hex_to_int( - DATA :baseFeePerGas :: STRING + DATA :result :baseFeePerGas :: STRING ) :: INT AS base_fee_per_gas, utils.udf_hex_to_int( - DATA :difficulty :: STRING + DATA :result :difficulty :: STRING ) :: INT AS difficulty, - DATA :extraData :: STRING AS extra_data, + DATA :result :extraData :: STRING AS extra_data, utils.udf_hex_to_int( - DATA :gasLimit :: STRING + DATA :result :gasLimit :: STRING ) :: INT AS gas_limit, utils.udf_hex_to_int( - DATA :gasUsed :: STRING + DATA :result :gasUsed :: STRING ) :: INT AS gas_used, - DATA :hash :: STRING AS HASH, - DATA :logsBloom :: STRING AS logs_bloom, - DATA :miner :: STRING AS miner, - DATA :mixHash :: STRING AS mixHash, + DATA :result :hash :: STRING AS HASH, + DATA :result :logsBloom :: STRING AS logs_bloom, + DATA :result :miner :: STRING AS miner, + DATA :result :mixHash :: STRING AS mixHash, utils.udf_hex_to_int( - DATA :nonce :: STRING + DATA :result :nonce :: STRING ) :: INT AS nonce, utils.udf_hex_to_int( - DATA :number :: STRING + DATA :result :number :: STRING ) :: INT AS NUMBER, - DATA :parentHash :: STRING AS parent_hash, - DATA :receiptsRoot :: STRING AS receipts_root, - DATA :sha3Uncles :: STRING AS sha3_uncles, + DATA :result :parentHash :: STRING AS parent_hash, + DATA :result :receiptsRoot :: STRING AS receipts_root, + DATA :result :sha3Uncles :: STRING AS sha3_uncles, utils.udf_hex_to_int( - DATA :size :: STRING + DATA :result :size :: STRING ) :: INT AS SIZE, - DATA :stateRoot :: STRING AS state_root, + DATA :result :stateRoot :: STRING AS state_root, utils.udf_hex_to_int( - DATA :timestamp :: STRING + DATA :result :timestamp :: STRING ) :: TIMESTAMP AS block_timestamp, utils.udf_hex_to_int( - DATA :totalDifficulty :: STRING + DATA :result :totalDifficulty :: STRING ) :: INT AS total_difficulty, ARRAY_SIZE( - DATA :transactions + DATA :result :transactions ) AS tx_count, - DATA :transactionsRoot :: STRING AS transactions_root, - DATA :uncles AS uncles, - DATA :withdrawals AS withdrawals, - DATA :withdrawalsRoot :: STRING AS withdrawals_root, + DATA :result :transactionsRoot :: STRING AS transactions_root, + DATA :result :uncles AS uncles, + DATA :result :withdrawals AS withdrawals, + DATA :result :withdrawalsRoot :: STRING AS withdrawals_root, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_number'] @@ -64,7 +64,7 @@ SELECT FROM {% if is_incremental() %} -{{ ref('bronze__streamline_blocks') }} +{{ ref('bronze__streamline_blocks_testnet') }} WHERE _inserted_timestamp >= ( SELECT @@ -73,7 +73,7 @@ WHERE {{ this }} ) {% else %} - {{ ref('bronze__streamline_FR_blocks') }} + {{ ref('bronze__streamline_FR_blocks_testnet') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/silver/core/silver__confirmed_blocks.sql b/models/silver/core/silver__confirmed_blocks.sql new file mode 100644 index 0000000..0753f1f --- /dev/null +++ b/models/silver/core/silver__confirmed_blocks.sql @@ -0,0 +1,50 @@ +-- depends_on: {{ ref('bronze__streamline_confirm_blocks_testnet') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "round(block_number,-3)", + tags = ['realtime'] +) }} + +WITH base AS ( + + SELECT + block_number, + DATA :result :hash :: STRING AS block_hash, + DATA :result :transactions txs, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_confirm_blocks_testnet') }} +WHERE + _inserted_timestamp >= ( + SELECT + IFNULL( + MAX( + _inserted_timestamp + ), + '1970-01-01' :: TIMESTAMP + ) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_confirm_blocks_testnet') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 +) +SELECT + block_number, + block_hash, + VALUE :: STRING AS tx_hash, + _inserted_timestamp +FROM + base, + LATERAL FLATTEN ( + input => txs + ) diff --git a/models/silver/core/silver__receipts.sql b/models/silver/core/silver__receipts.sql index 14e8bdf..a0595b6 100644 --- a/models/silver/core/silver__receipts.sql +++ b/models/silver/core/silver__receipts.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_receipts') }} +-- depends_on: {{ ref('bronze__streamline_receipts_testnet') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', @@ -17,7 +17,7 @@ WITH base AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_receipts') }} +{{ ref('bronze__streamline_receipts_testnet') }} WHERE _inserted_timestamp >= ( SELECT @@ -27,7 +27,7 @@ WHERE ) AND IS_OBJECT(DATA) {% else %} - {{ ref('bronze__streamline_FR_receipts') }} + {{ ref('bronze__streamline_FR_receipts_testnet') }} WHERE IS_OBJECT(DATA) {% endif %} diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index 30f2679..ad1b519 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_traces') }} +-- depends_on: {{ ref('bronze__streamline_traces_testnet') }} {{ config ( materialized = "incremental", incremental_strategy = 'delete+insert', @@ -12,15 +12,13 @@ WITH bronze_traces AS ( SELECT block_number, - tx_position, - full_traces, - {# VALUE :array_index :: INT AS tx_position, - DATA :result AS full_traces, #} + VALUE :array_index :: INT AS tx_position, + DATA :result AS full_traces, _inserted_timestamp FROM {% if is_incremental() %} -{{ ref('bronze__streamline_traces') }} +{{ ref('bronze__streamline_traces_testnet') }} WHERE _inserted_timestamp >= ( SELECT @@ -28,13 +26,13 @@ WHERE FROM {{ this }} ) - {# AND DATA :result IS NOT NULL #} + AND DATA :result IS NOT NULL {% else %} - {{ ref('bronze__streamline_FR_traces') }} -{# WHERE - _partition_by_block_id <= 2300000 - AND - DATA :result IS NOT NULL #} + {{ ref('bronze__streamline_FR_traces_testnet') }} +WHERE + {# _partition_by_block_id <= 2300000 + AND #} + DATA :result IS NOT NULL {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 31680c1..c4a6a0e 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_transactions') }} +-- depends_on: {{ ref('bronze__streamline_transactions_testnet') }} {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', @@ -17,7 +17,7 @@ WITH base AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_transactions') }} +{{ ref('bronze__streamline_transactions_testnet') }} WHERE _inserted_timestamp >= ( SELECT @@ -27,7 +27,7 @@ WHERE ) AND IS_OBJECT(DATA) {% else %} - {{ ref('bronze__streamline_FR_transactions') }} + {{ ref('bronze__streamline_FR_transactions_testnet') }} WHERE IS_OBJECT(DATA) {% endif %} @@ -75,11 +75,14 @@ base_tx AS ( 1, 10 ) AS origin_function_signature, - TRY_TO_NUMBER( - utils.udf_hex_to_int( + utils.udf_hex_to_int( DATA :mint :: STRING - ) - ) AS mint, + ) AS mint_precise_raw, + utils.udf_decimal_adjust( + mint_precise_raw, + 18 + ) AS mint_precise, + mint_precise :: FLOAT AS mint, TRY_TO_NUMBER( utils.udf_hex_to_int( DATA :maxFeePerGas :: STRING @@ -141,6 +144,8 @@ new_records AS ( t.tx_hash, t.input_data, t.origin_function_signature, + t.mint_precise_raw, + t.mint_precise, t.mint, t.max_fee_per_gas, t.max_priority_fee_per_gas, @@ -222,6 +227,8 @@ missing_data AS ( t.tx_hash, t.input_data, t.origin_function_signature, + t.mint_precise_raw, + t.mint_precise, t.mint, t.max_fee_per_gas, t.max_priority_fee_per_gas, @@ -296,6 +303,8 @@ FINAL AS ( tx_hash, input_data, origin_function_signature, + mint_precise_raw, + mint_precise, mint, max_fee_per_gas, max_priority_fee_per_gas, @@ -344,6 +353,8 @@ SELECT tx_hash, input_data, origin_function_signature, + mint_precise_raw, + mint_precise, mint, max_fee_per_gas, max_priority_fee_per_gas, @@ -391,6 +402,8 @@ SELECT tx_hash, input_data, origin_function_signature, + mint_precise_raw, + mint_precise, mint, max_fee_per_gas, max_priority_fee_per_gas, diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.sql b/models/silver/core/tests/blocks/test_silver__blocks_full.sql new file mode 100644 index 0000000..25dc03b --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__blocks') }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.yml b/models/silver/core/tests/blocks/test_silver__blocks_full.yml new file mode 100644 index 0000000..f90d39c --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.yml @@ -0,0 +1,95 @@ +version: 2 +models: + - name: test_silver__blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EXTRA_DATA + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SHA3_UNCLES + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql new file mode 100644 index 0000000..e122694 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver__blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.yml b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml new file mode 100644 index 0000000..ceb767f --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: test_silver__blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + \ No newline at end of file diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.sql b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.sql new file mode 100644 index 0000000..6d93a21 --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__confirmed_blocks') }} diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.yml b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.yml new file mode 100644 index 0000000..81a9db0 --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_full.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: test_silver__confirmed_blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ \ No newline at end of file diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql new file mode 100644 index 0000000..adbbf1b --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver__confirmed_blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.yml b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.yml new file mode 100644 index 0000000..460ddf1 --- /dev/null +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: test_silver__confirmed_blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - TIMESTAMP_LTZ \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.sql b/models/silver/core/tests/event_logs/test_silver__logs_full.sql new file mode 100644 index 0000000..e117298 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__logs') }} diff --git a/models/silver/core/tests/event_logs/test_silver__logs_full.yml b/models/silver/core/tests/event_logs/test_silver__logs_full.yml new file mode 100644 index 0000000..4b5140c --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_full.yml @@ -0,0 +1,77 @@ +version: 2 +models: + - name: test_silver__logs_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - fsc_utils.tx_block_count: + config: + severity: error + error_if: "!=0" + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ \ No newline at end of file diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.sql b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql new file mode 100644 index 0000000..03400cc --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver__logs') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.yml b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml new file mode 100644 index 0000000..4a2b8a6 --- /dev/null +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: test_silver__logs_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.sql b/models/silver/core/tests/receipts/test_silver__receipts_full.sql new file mode 100644 index 0000000..56f39ae --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__receipts') }} diff --git a/models/silver/core/tests/receipts/test_silver__receipts_full.yml b/models/silver/core/tests/receipts/test_silver__receipts_full.yml new file mode 100644 index 0000000..9a18685 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_full.yml @@ -0,0 +1,82 @@ +version: 2 +models: + - name: test_silver__receipts_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + - name: TYPE + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + + diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.sql b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql new file mode 100644 index 0000000..f83874d --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver__receipts') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.yml b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml new file mode 100644 index 0000000..1e8cfa1 --- /dev/null +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: test_silver__receipts_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + + diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql new file mode 100644 index 0000000..be52e71 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__traces') }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml new file mode 100644 index 0000000..639f77b --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_full.yml @@ -0,0 +1,59 @@ +version: 2 +models: + - name: test_silver__traces_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: IDENTIFIER + tests: + - not_null + - name: ETH_VALUE + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + + diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql new file mode 100644 index 0000000..1dc3e76 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver__traces') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml new file mode 100644 index 0000000..51db5b0 --- /dev/null +++ b/models/silver/core/tests/traces/test_silver__traces_recent.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: test_silver__traces_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - TX_POSITION + - TRACE_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.sql b/models/silver/core/tests/transactions/test_silver__transactions_full.sql new file mode 100644 index 0000000..df58d17 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__transactions') }} diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.yml b/models/silver/core/tests/transactions/test_silver__transactions_full.yml new file mode 100644 index 0000000..fc88627 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.yml @@ -0,0 +1,119 @@ +version: 2 +models: + - name: test_silver__transactions_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: VALUE + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS + tests: + - not_null + - name: INPUT_DATA + tests: + - not_null + - name: TX_STATUS + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + where: NOT IS_PENDING + - name: GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CUMULATIVE_GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_FEE + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + + diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql new file mode 100644 index 0000000..cb859be --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + * +FROM + {{ ref('silver__transactions') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.yml b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml new file mode 100644 index 0000000..3539ac2 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml @@ -0,0 +1,22 @@ +version: 2 +models: + - name: test_silver__transactions_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - fsc_utils.sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index db9480a..880c6cb 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -18,6 +18,11 @@ sources: - name: debug_traceBlockByNumber - name: decoded_logs - name: confirm_blocks + - name: receipts_testnet + - name: blocks_testnet + - name: transactions_testnet + - name: traces_testnet + - name: confirm_blocks_testnet - name: udfs_streamline database: udfs schema: streamline diff --git a/models/streamline/bronze/core/bronze__streamline_FR_blocks.sql b/models/streamline/bronze/core_livequery/bronze__livequery_FR_blocks_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_FR_blocks.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_FR_blocks_testnet.sql diff --git a/models/streamline/bronze/core/bronze__streamline_blocks.sql b/models/streamline/bronze/core_livequery/bronze__livequery_FR_confirm_blocks_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_blocks.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_FR_confirm_blocks_testnet.sql diff --git a/models/streamline/bronze/core/bronze__streamline_FR_receipts.sql b/models/streamline/bronze/core_livequery/bronze__livequery_FR_receipts_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_FR_receipts.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_FR_receipts_testnet.sql diff --git a/models/streamline/bronze/core/bronze__streamline_FR_traces.sql b/models/streamline/bronze/core_livequery/bronze__livequery_FR_traces_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_FR_traces.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_FR_traces_testnet.sql diff --git a/models/streamline/bronze/core/bronze__streamline_FR_transactions.sql b/models/streamline/bronze/core_livequery/bronze__livequery_FR_transactions_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_FR_transactions.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_FR_transactions_testnet.sql diff --git a/models/streamline/bronze/core_livequery/bronze__livequery_blocks_testnet.sql b/models/streamline/bronze/core_livequery/bronze__livequery_blocks_testnet.sql new file mode 100644 index 0000000..acd40d1 --- /dev/null +++ b/models/streamline/bronze/core_livequery/bronze__livequery_blocks_testnet.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +WITH num_seq AS ( + + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id > 1300000 + AND _id <= 1300010 +) +SELECT + block_number AS block_number, + utils.udf_int_to_hex(block_number) AS block_hex, + live.udf_api( + 'POST', + '{blast_testnet_url}',{},{ 'method' :'eth_getBlockByNumber', + 'params' :[ block_hex, False ], + 'id' :1, + 'jsonrpc' :'2.0' }, + 'quicknode_blast_testnet' + ) AS resp, + resp :data :result AS DATA, + SYSDATE() AS _inserted_timestamp +FROM + num_seq diff --git a/models/streamline/bronze/core_livequery/bronze__livequery_confirm_blocks_testnet.sql b/models/streamline/bronze/core_livequery/bronze__livequery_confirm_blocks_testnet.sql new file mode 100644 index 0000000..acd40d1 --- /dev/null +++ b/models/streamline/bronze/core_livequery/bronze__livequery_confirm_blocks_testnet.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +WITH num_seq AS ( + + SELECT + _id AS block_number + FROM + {{ ref('silver__number_sequence') }} + WHERE + _id > 1300000 + AND _id <= 1300010 +) +SELECT + block_number AS block_number, + utils.udf_int_to_hex(block_number) AS block_hex, + live.udf_api( + 'POST', + '{blast_testnet_url}',{},{ 'method' :'eth_getBlockByNumber', + 'params' :[ block_hex, False ], + 'id' :1, + 'jsonrpc' :'2.0' }, + 'quicknode_blast_testnet' + ) AS resp, + resp :data :result AS DATA, + SYSDATE() AS _inserted_timestamp +FROM + num_seq diff --git a/models/streamline/bronze/core/bronze__streamline_receipts.sql b/models/streamline/bronze/core_livequery/bronze__livequery_receipts_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_receipts.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_receipts_testnet.sql diff --git a/models/streamline/bronze/core/bronze__streamline_traces.sql b/models/streamline/bronze/core_livequery/bronze__livequery_traces_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_traces.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_traces_testnet.sql diff --git a/models/streamline/bronze/core/bronze__streamline_transactions.sql b/models/streamline/bronze/core_livequery/bronze__livequery_transactions_testnet.sql similarity index 100% rename from models/streamline/bronze/core/bronze__streamline_transactions.sql rename to models/streamline/bronze/core_livequery/bronze__livequery_transactions_testnet.sql diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_FR_blocks_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_FR_blocks_testnet.sql new file mode 100644 index 0000000..4222601 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_FR_blocks_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_FR_query( + model = "blocks_testnet", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_FR_confirm_blocks_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_FR_confirm_blocks_testnet.sql new file mode 100644 index 0000000..5783947 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_FR_confirm_blocks_testnet.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_FR_query( + model = "confirm_blocks_testnet", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_FR_receipts_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_FR_receipts_testnet.sql new file mode 100644 index 0000000..eccbe92 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_FR_receipts_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_FR_query( + model = 'receipts_testnet', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_FR_traces_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_FR_traces_testnet.sql new file mode 100644 index 0000000..f5246e5 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_FR_traces_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_FR_query( + model = 'traces_testnet', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_FR_transactions_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_FR_transactions_testnet.sql new file mode 100644 index 0000000..78da9ed --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_FR_transactions_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_FR_query( + model = 'transactions_testnet', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_blocks_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_blocks_testnet.sql new file mode 100644 index 0000000..fc25eb9 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_blocks_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_query( + model = "blocks_testnet", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} \ No newline at end of file diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_confirm_blocks_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_confirm_blocks_testnet.sql new file mode 100644 index 0000000..c64aa36 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_confirm_blocks_testnet.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view' +) }} +{{ streamline_external_table_query( + model = "confirm_blocks_testnet", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_receipts_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_receipts_testnet.sql new file mode 100644 index 0000000..adf7fac --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_receipts_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_query( + model = 'receipts_testnet', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_traces_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_traces_testnet.sql new file mode 100644 index 0000000..a188974 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_traces_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_query( + model = 'traces_testnet', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/bronze/core_testnet/bronze__streamline_transactions_testnet.sql b/models/streamline/bronze/core_testnet/bronze__streamline_transactions_testnet.sql new file mode 100644 index 0000000..552b3a9 --- /dev/null +++ b/models/streamline/bronze/core_testnet/bronze__streamline_transactions_testnet.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{# {% set model = this.identifier.split("_") [-1] %} #} +{{ streamline_external_table_query( + model = 'transactions_testnet', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "partition_key" +) }} diff --git a/models/streamline/silver/_block_lookback.sql b/models/streamline/silver/_block_lookback.sql new file mode 100644 index 0000000..41f2108 --- /dev/null +++ b/models/streamline/silver/_block_lookback.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = "ephemeral" +) }} + +SELECT + COALESCE(MIN(block_number), 0) AS block_number +FROM + {{ ref("silver__blocks") }} +WHERE + block_timestamp >= DATEADD('hour', -72, TRUNCATE(SYSDATE(), 'HOUR')) + AND block_timestamp < DATEADD('hour', -71, TRUNCATE(SYSDATE(), 'HOUR')) diff --git a/models/streamline/silver/_max_block_by_date.sql b/models/streamline/silver/_max_block_by_date.sql new file mode 100644 index 0000000..7609b67 --- /dev/null +++ b/models/streamline/silver/_max_block_by_date.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "ephemeral", + unique_key = "block_number", +) }} + +WITH base AS ( + + SELECT + block_timestamp :: DATE AS block_date, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + GROUP BY + block_timestamp :: DATE +) +SELECT + block_date, + block_number +FROM + base +WHERE + block_date <> ( + SELECT + MAX(block_date) + FROM + base + ) diff --git a/models/streamline/silver/_max_block_by_hour.sql b/models/streamline/silver/_max_block_by_hour.sql new file mode 100644 index 0000000..80e3d97 --- /dev/null +++ b/models/streamline/silver/_max_block_by_hour.sql @@ -0,0 +1,37 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH base AS ( + + SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_hour, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + WHERE + block_timestamp > DATEADD( + 'day', + -5, + CURRENT_DATE + ) + GROUP BY + 1 +) +SELECT + block_hour, + block_number +FROM + base +WHERE + block_hour <> ( + SELECT + MAX( + block_hour + ) + FROM + base + ) diff --git a/models/streamline/silver/core/complete/streamline__complete_blocks.sql b/models/streamline/silver/core/complete/streamline__complete_blocks.sql new file mode 100644 index 0000000..3148677 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__complete_blocks.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('bronze__streamline_blocks_testnet') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_core_complete'] +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_blocks_testnet') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_blocks_testnet') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql b/models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql new file mode 100644 index 0000000..79a1a69 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql @@ -0,0 +1,29 @@ +-- depends_on: {{ ref('bronze__streamline_confirm_blocks_testnet') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + tags = ['streamline_core_complete'] +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_confirm_blocks_testnet') }} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1970-01-01' :: TIMESTAMP) _inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze__streamline_FR_confirm_blocks_testnet') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY id + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/complete/streamline__complete_receipts.sql b/models/streamline/silver/core/complete/streamline__complete_receipts.sql new file mode 100644 index 0000000..9aaf5f2 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__complete_receipts.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('bronze__streamline_receipts_testnet') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_core_complete'] +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_receipts_testnet') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_receipts_testnet') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/complete/streamline__complete_traces.sql b/models/streamline/silver/core/complete/streamline__complete_traces.sql new file mode 100644 index 0000000..40f3615 --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__complete_traces.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('bronze__streamline_traces_testnet') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_core_complete'] +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_traces_testnet') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_traces_testnet') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/complete/streamline__complete_transactions.sql b/models/streamline/silver/core/complete/streamline__complete_transactions.sql new file mode 100644 index 0000000..b8d98aa --- /dev/null +++ b/models/streamline/silver/core/complete/streamline__complete_transactions.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('bronze__streamline_transactions_testnet') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_core_complete'] +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_transactions_testnet') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_transactions_testnet') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/silver/core/history/streamline__blocks_history.sql b/models/streamline/silver/core/history/streamline__blocks_history.sql new file mode 100644 index 0000000..2b37658 --- /dev/null +++ b/models/streamline/silver/core/history/streamline__blocks_history.sql @@ -0,0 +1,76 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'blocks_testnet', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +blocks AS ( + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) + EXCEPT + SELECT + id, + block_number + FROM + {{ ref("streamline__complete_blocks") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)) :: STRING + ) AS request + FROM + blocks + ORDER BY + block_number ASC + LIMIT + 1000 --remove for prod diff --git a/models/streamline/silver/core/history/streamline__receipts_history.sql b/models/streamline/silver/core/history/streamline__receipts_history.sql new file mode 100644 index 0000000..35f32c1 --- /dev/null +++ b/models/streamline/silver/core/history/streamline__receipts_history.sql @@ -0,0 +1,72 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'receipts_testnet', 'exploded_key','[\"result\"]', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +blocks AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_receipts") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockReceipts', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number))) :: STRING + ) AS request + FROM + blocks + ORDER BY + block_number ASC + LIMIT + 1000 --remove for prod diff --git a/models/streamline/silver/core/history/streamline__traces_history.sql b/models/streamline/silver/core/history/streamline__traces_history.sql new file mode 100644 index 0000000..28ace86 --- /dev/null +++ b/models/streamline/silver/core/history/streamline__traces_history.sql @@ -0,0 +1,73 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces_testnet', 'exploded_key','[\"result\"]', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +blocks AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_traces") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'debug_traceBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), OBJECT_CONSTRUCT('tracer', 'callTracer', 'timeout', '30s')) + ) :: STRING + ) AS request +FROM + blocks +ORDER BY + block_number ASC +LIMIT + 1000 --remove for prod diff --git a/models/streamline/silver/core/history/streamline__transactions_history.sql b/models/streamline/silver/core/history/streamline__transactions_history.sql new file mode 100644 index 0000000..bcc7762 --- /dev/null +++ b/models/streamline/silver/core/history/streamline__transactions_history.sql @@ -0,0 +1,76 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions_testnet', 'exploded_key','[\"result\", \"transactions\"]', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_history'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +blocks AS ( + SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS id, + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) + EXCEPT + SELECT + id, + block_number + FROM + {{ ref("streamline__complete_transactions") }} + WHERE + block_number <= ( + SELECT + block_number + FROM + last_3_days + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), TRUE)) :: STRING + ) AS request + FROM + blocks + ORDER BY + block_number ASC + LIMIT + 1000 --remove for prod diff --git a/models/streamline/silver/core/realtime/streamline__blocks_realtime.sql b/models/streamline/silver/core/realtime/streamline__blocks_realtime.sql new file mode 100644 index 0000000..ccbcbf2 --- /dev/null +++ b/models/streamline/silver/core/realtime/streamline__blocks_realtime.sql @@ -0,0 +1,86 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'blocks_testnet', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_realtime'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +to_do AS ( + SELECT + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS id, + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + id, + block_number + FROM + {{ ref("streamline__complete_blocks") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)) :: STRING + ) AS request + FROM + to_do + ORDER BY + partition_key ASC + LIMIT + 1200 --remove for prod diff --git a/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql b/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql new file mode 100644 index 0000000..70134f1 --- /dev/null +++ b/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql @@ -0,0 +1,101 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'confirm_blocks_testnet', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_realtime'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +look_back AS ( + SELECT + block_number + FROM + {{ ref("_max_block_by_hour") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 6 +), +tbl AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + block_number IS NOT NULL + AND block_number <= ( + SELECT + block_number + FROM + look_back + ) + AND block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_confirmed_blocks") }} + WHERE + block_number IS NOT NULL + AND block_number <= ( + SELECT + block_number + FROM + look_back + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) + AND block_number >= ( + SELECT + block_number + FROM + last_3_days + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), FALSE)) :: STRING + ) AS request + FROM + tbl + ORDER BY + block_number ASC + LIMIT + 1200 --remove for prod diff --git a/models/streamline/silver/core/realtime/streamline__receipts_realtime.sql b/models/streamline/silver/core/realtime/streamline__receipts_realtime.sql new file mode 100644 index 0000000..c0bc95a --- /dev/null +++ b/models/streamline/silver/core/realtime/streamline__receipts_realtime.sql @@ -0,0 +1,106 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'receipts_testnet', 'exploded_key','[\"result\"]', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_realtime'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +to_do AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_receipts") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do + UNION + SELECT + block_number + FROM + ( + SELECT + block_number + FROM + {{ ref("_missing_receipts") }} + UNION + SELECT + block_number + FROM + {{ ref("_missing_txs") }} + UNION + SELECT + block_number + FROM + {{ ref("_unconfirmed_blocks") }} + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockReceipts', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number))) :: STRING + ) AS request + FROM + ready_blocks + ORDER BY + block_number ASC + LIMIT + 1200 --remove for prod diff --git a/models/streamline/silver/core/realtime/streamline__traces_realtime.sql b/models/streamline/silver/core/realtime/streamline__traces_realtime.sql new file mode 100644 index 0000000..2f4a549 --- /dev/null +++ b/models/streamline/silver/core/realtime/streamline__traces_realtime.sql @@ -0,0 +1,102 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces_testnet', 'exploded_key','[\"result\"]', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_realtime'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +to_do AS ( + SELECT + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number + FROM + {{ ref("streamline__complete_traces") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +), +ready_blocks AS ( + SELECT + block_number + FROM + to_do + UNION + SELECT + block_number + FROM + ( + SELECT + block_number + FROM + {{ ref("_missing_traces") }} + UNION + SELECT + block_number + FROM + {{ ref("_unconfirmed_blocks") }} + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'debug_traceBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), OBJECT_CONSTRUCT('tracer', 'callTracer', 'timeout', '30s')) + ) :: STRING + ) AS request +FROM + ready_blocks +ORDER BY + block_number ASC +LIMIT + 1200 --remove for prod diff --git a/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql b/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql new file mode 100644 index 0000000..2f4f487 --- /dev/null +++ b/models/streamline/silver/core/realtime/streamline__transactions_realtime.sql @@ -0,0 +1,113 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions_testnet', 'exploded_key','[\"result\", \"transactions\"]', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'sm_secret_name','dev/blast/node/mainnet'))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_core_realtime'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +to_do AS ( + SELECT + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS id, + block_number + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + id, + block_number + FROM + {{ ref("streamline__complete_transactions") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + AND _inserted_timestamp >= DATEADD( + 'day', + -4, + SYSDATE() + ) +), +ready_blocks AS ( + SELECT + id, + block_number + FROM + to_do + UNION + SELECT + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS id, + block_number + FROM + ( + SELECT + block_number + FROM + {{ ref("_missing_txs") }} + UNION + SELECT + block_number + FROM + {{ ref("_unconfirmed_blocks") }} + ) +) +SELECT + block_number AS partition_key, + OBJECT_CONSTRUCT( + 'method', + 'POST', + 'url', + '{service}/{Authentication}', + 'headers', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + 'params', + PARSE_JSON('{}'), + 'data', + OBJECT_CONSTRUCT( + 'id', + block_number :: STRING, + 'jsonrpc', + '2.0', + 'method', + 'eth_getBlockByNumber', + 'params', + ARRAY_CONSTRUCT(utils.udf_int_to_hex(block_number), TRUE)) :: STRING + ) AS request + FROM + ready_blocks + ORDER BY + block_number ASC + LIMIT + 1200 --remove for prod diff --git a/models/streamline/silver/core/retry/_missing_receipts.sql b/models/streamline/silver/core/retry/_missing_receipts.sql new file mode 100644 index 0000000..fb055b6 --- /dev/null +++ b/models/streamline/silver/core/retry/_missing_receipts.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + DISTINCT t.block_number AS block_number +FROM + {{ ref("silver__transactions") }} + t + LEFT JOIN {{ ref("silver__receipts") }} + r USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + r.tx_hash IS NULL + AND t.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND t.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + r._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR r._inserted_timestamp IS NULL) diff --git a/models/streamline/silver/core/retry/_missing_traces.sql b/models/streamline/silver/core/retry/_missing_traces.sql new file mode 100644 index 0000000..40033bf --- /dev/null +++ b/models/streamline/silver/core/retry/_missing_traces.sql @@ -0,0 +1,30 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + DISTINCT tx.block_number block_number +FROM + {{ ref("silver__transactions") }} + tx + LEFT JOIN {{ ref("silver__traces") }} + tr + ON tx.block_number = tr.block_number + AND tx.tx_hash = tr.tx_hash + AND tr.block_timestamp >= DATEADD('hour', -84, SYSDATE()) +WHERE + tx.block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND tr.tx_hash IS NULL + AND tx.block_number >= ( + SELECT + block_number + FROM + lookback + ) diff --git a/models/streamline/silver/core/retry/_missing_txs.sql b/models/streamline/silver/core/retry/_missing_txs.sql new file mode 100644 index 0000000..7718e97 --- /dev/null +++ b/models/streamline/silver/core/retry/_missing_txs.sql @@ -0,0 +1,40 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +), +transactions AS ( + SELECT + block_number, + POSITION, + LAG( + POSITION, + 1 + ) over ( + PARTITION BY block_number + ORDER BY + POSITION ASC + ) AS prev_POSITION + FROM + {{ ref("silver__transactions") }} + WHERE + block_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND block_number >= ( + SELECT + block_number + FROM + lookback + ) +) +SELECT + DISTINCT block_number AS block_number +FROM + transactions +WHERE + POSITION - prev_POSITION <> 1 diff --git a/models/streamline/silver/core/retry/_unconfirmed_blocks.sql b/models/streamline/silver/core/retry/_unconfirmed_blocks.sql new file mode 100644 index 0000000..8e2507c --- /dev/null +++ b/models/streamline/silver/core/retry/_unconfirmed_blocks.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "ephemeral" +) }} + +WITH lookback AS ( + + SELECT + block_number + FROM + {{ ref("_block_lookback") }} +) +SELECT + DISTINCT cb.block_number AS block_number +FROM + {{ ref("silver__confirmed_blocks") }} + cb + LEFT JOIN {{ ref("silver__transactions") }} + txs USING ( + block_number, + block_hash, + tx_hash + ) +WHERE + txs.tx_hash IS NULL + AND cb.block_number >= ( + SELECT + block_number + FROM + lookback + ) + AND cb._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + txs._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR txs._inserted_timestamp IS NULL) diff --git a/models/streamline/silver/core/streamline__blocks.sql b/models/streamline/silver/core/streamline__blocks.sql new file mode 100644 index 0000000..ffa8e8e --- /dev/null +++ b/models/streamline/silver/core/streamline__blocks.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = "view", + tags = ['streamline_core_complete'] +) }} + +SELECT + _id AS block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex +FROM + {{ ref("silver__number_sequence") }} +WHERE + _id <= ( + SELECT + COALESCE( + block_number, + 0 + ) + FROM + {{ref("streamline__get_chainhead")}} + ) +ORDER BY + _id ASC diff --git a/models/streamline/silver/core/streamline__get_chainhead.sql b/models/streamline/silver/core/streamline__get_chainhead.sql new file mode 100644 index 0000000..8d706ea --- /dev/null +++ b/models/streamline/silver/core/streamline__get_chainhead.sql @@ -0,0 +1,19 @@ +{{ config ( + materialized = 'table', + tags = ['streamline_core_complete'] +) }} + +SELECT + live.udf_api( + 'POST', + '{blast_testnet_url}', + --update for prod + {},{ 'method' :'eth_blockNumber', + 'params' :[], + 'id' :1, + 'jsonrpc' :'2.0' }, + 'quicknode_blast_testnet' --update for prod + ) AS resp, + utils.udf_hex_to_int( + resp :data :result :: STRING + ) AS block_number diff --git a/tests/blast/test_silver__confirmed_blocks.sql b/tests/blast/test_silver__confirmed_blocks.sql new file mode 100644 index 0000000..8cbac8b --- /dev/null +++ b/tests/blast/test_silver__confirmed_blocks.sql @@ -0,0 +1 @@ +{{ missing_confirmed_txs(ref("test_silver__confirmed_blocks_full"), ref("test_silver__transactions_full")) }} diff --git a/tests/blast/test_silver__missing_receipts.sql b/tests/blast/test_silver__missing_receipts.sql new file mode 100644 index 0000000..8481599 --- /dev/null +++ b/tests/blast/test_silver__missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__receipts_full")) }} diff --git a/tests/blast/test_silver__missing_traces.sql b/tests/blast/test_silver__missing_traces.sql new file mode 100644 index 0000000..c96256f --- /dev/null +++ b/tests/blast/test_silver__missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_full') }} +{{ missing_txs(ref("test_silver__traces_full")) }} diff --git a/tests/blast/test_silver__missing_txs.sql b/tests/blast/test_silver__missing_txs.sql new file mode 100644 index 0000000..6c01a35 --- /dev/null +++ b/tests/blast/test_silver__missing_txs.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__blocks_full') }} +{{ fsc_utils.tx_gaps(ref("test_silver__transactions_full")) }} diff --git a/tests/blast/test_silver__recent_confirmed_blocks.sql b/tests/blast/test_silver__recent_confirmed_blocks.sql new file mode 100644 index 0000000..79cdd66 --- /dev/null +++ b/tests/blast/test_silver__recent_confirmed_blocks.sql @@ -0,0 +1 @@ +{{ missing_confirmed_txs(ref("test_silver__confirmed_blocks_recent"), ref("test_silver__transactions_recent")) }} diff --git a/tests/blast/test_silver__recent_missing_receipts.sql b/tests/blast/test_silver__recent_missing_receipts.sql new file mode 100644 index 0000000..f657fd3 --- /dev/null +++ b/tests/blast/test_silver__recent_missing_receipts.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__receipts_recent")) }} diff --git a/tests/blast/test_silver__recent_missing_traces.sql b/tests/blast/test_silver__recent_missing_traces.sql new file mode 100644 index 0000000..26a69d4 --- /dev/null +++ b/tests/blast/test_silver__recent_missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_silver__traces_recent")) }} diff --git a/tests/blast/test_silver__recent_missing_txs.sql b/tests/blast/test_silver__recent_missing_txs.sql new file mode 100644 index 0000000..985c250 --- /dev/null +++ b/tests/blast/test_silver__recent_missing_txs.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__blocks_recent') }} +{{ fsc_utils.recent_tx_gaps(ref("test_silver__transactions_recent")) }}