flip/blast-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/blast-models.git synced 2026-02-06 14:11:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

bronze and sources

Browse Source
This commit is contained in:
drethereum 2025-03-04 12:45:32 -07:00
parent dad66e6d5a
commit 770585e12b
29 changed files with 77 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dbt_project.yml
View File

@ -80,10 +80,17 @@ vars:
dev:
API_INTEGRATION: AWS_BLAST_API_DEV
EXTERNAL_FUNCTION_URI: y9d0tuavh6.execute-api.us-east-1.amazonaws.com/stg/
ROLES:
- AWS_LAMBDA_BLAST_API
- INTERNAL_DEV
prod:
API_INTEGRATION: AWS_BLAST_API
EXTERNAL_FUNCTION_URI: 42gzudc5si.execute-api.us-east-1.amazonaws.com/prod/
ROLES:
- AWS_LAMBDA_BLAST_API
- INTERNAL_DEV
- DBT_CLOUD_BLAST
#### STREAMLINE 2.0 END ####

13
models/bronze/api_udf/bronze_api__contract_abis.sql
View File

@ -50,7 +50,18 @@ row_nos AS (
),
batched AS ({% for item in range(501) %}
SELECT
rn.contract_address, live.udf_api('GET', CONCAT('https://api.blastscan.io/api?module=contract&action=getabi&address=', rn.contract_address, '&apikey={key}'),{ 'User-Agent': 'FlipsideStreamline' },{}, 'Vault/prod/block_explorers/blast_scan') AS abi_data, SYSDATE() AS _inserted_timestamp
rn.contract_address,
live.udf_api(
'GET',
CONCAT('https://api.blastscan.io/api?module=contract&action=getabi&address=',rn.contract_address,'&apikey={key}'),
OBJECT_CONSTRUCT(
'Content-Type', 'application/json',
'fsc-quantum-state', 'livequery'
),
NULL,
'Vault/prod/block_explorers/blast_scan'
) AS abi_data,
SYSDATE() AS _inserted_timestamp
FROM
row_nos rn
WHERE

46
models/silver/core/silver__blocks.sql
View File

@ -19,48 +19,48 @@ SELECT
) :id :: INT
) AS block_number,
utils.udf_hex_to_int(
DATA :result :baseFeePerGas :: STRING
DATA :baseFeePerGas :: STRING
) :: INT AS base_fee_per_gas,
utils.udf_hex_to_int(
DATA :result :difficulty :: STRING
DATA :difficulty :: STRING
) :: INT AS difficulty,
DATA :result :extraData :: STRING AS extra_data,
DATA :extraData :: STRING AS extra_data,
utils.udf_hex_to_int(
DATA :result :gasLimit :: STRING
DATA :gasLimit :: STRING
) :: INT AS gas_limit,
utils.udf_hex_to_int(
DATA :result :gasUsed :: STRING
DATA :gasUsed :: STRING
) :: INT AS gas_used,
DATA :result :hash :: STRING AS HASH,
DATA :result :logsBloom :: STRING AS logs_bloom,
DATA :result :miner :: STRING AS miner,
DATA :result :mixHash :: STRING AS mixHash,
DATA :hash :: STRING AS HASH,
DATA :logsBloom :: STRING AS logs_bloom,
DATA :miner :: STRING AS miner,
DATA :mixHash :: STRING AS mixHash,
utils.udf_hex_to_int(
DATA :result :nonce :: STRING
DATA :nonce :: STRING
) :: INT AS nonce,
utils.udf_hex_to_int(
DATA :result :number :: STRING
DATA :number :: STRING
) :: INT AS NUMBER,
DATA :result :parentHash :: STRING AS parent_hash,
DATA :result :receiptsRoot :: STRING AS receipts_root,
DATA :result :sha3Uncles :: STRING AS sha3_uncles,
DATA :parentHash :: STRING AS parent_hash,
DATA :receiptsRoot :: STRING AS receipts_root,
DATA :sha3Uncles :: STRING AS sha3_uncles,
utils.udf_hex_to_int(
DATA :result :size :: STRING
DATA :size :: STRING
) :: INT AS SIZE,
DATA :result :stateRoot :: STRING AS state_root,
DATA :stateRoot :: STRING AS state_root,
utils.udf_hex_to_int(
DATA :result :timestamp :: STRING
DATA :timestamp :: STRING
) :: TIMESTAMP AS block_timestamp,
utils.udf_hex_to_int(
DATA :result :totalDifficulty :: STRING
DATA :totalDifficulty :: STRING
) :: INT AS total_difficulty,
ARRAY_SIZE(
DATA :result :transactions
DATA :transactions
) AS tx_count,
DATA :result :transactionsRoot :: STRING AS transactions_root,
DATA :result :uncles AS uncles,
DATA :result :withdrawals AS withdrawals,
DATA :result :withdrawalsRoot :: STRING AS withdrawals_root,
DATA :transactionsRoot :: STRING AS transactions_root,
DATA :uncles AS uncles,
DATA :withdrawals AS withdrawals,
DATA :withdrawalsRoot :: STRING AS withdrawals_root,
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['block_number']

2
models/sources.yml
View File

@ -12,6 +12,8 @@ sources:
- name: traces
- name: decoded_logs
- name: confirm_blocks
- name: blocks_v2
- name: transactions_v2
- name: udfs_streamline
database: udfs
schema: streamline

2
models/streamline/bronze/core/bronze__blocks.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'BLOCKS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) or var('GLOBAL_USES_BLOCKS_TRANSACTIONS_PATH', false) else '' %}
{% set source_version = 'V2' %}
{% set model_type = '' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

19
models/streamline/bronze/core/bronze__blocks_fr.sql
View File

@ -13,29 +13,14 @@ SELECT
_inserted_timestamp
FROM
{{ ref('bronze__blocks_fr_v2') }}
{% if var('GLOBAL_USES_STREAMLINE_V1', false) %}
UNION ALL
SELECT
_partition_by_block_id AS partition_key,
block_number,
VALUE,
DATA,
metadata,
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__blocks_fr_v1') }}
{% endif %}
{% if var('GLOBAL_USES_BLOCKS_TRANSACTIONS_PATH', false) %}
UNION ALL
SELECT
partition_key,
block_number,
VALUE,
DATA,
DATA :result AS DATA,
metadata,
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__blocks_fr_v2_1') }}
{% endif %}
{{ ref('bronze__blocks_fr_v1') }}

10
models/streamline/bronze/core/bronze__blocks_fr_v1.sql
View File

@ -5,10 +5,10 @@
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %}
{% set partition_join_key = '_partition_by_block_id' %}
{% set partition_function = default_vars['partition_function'] %}
{% set partition_join_key = default_vars['partition_join_key'] %}
{% set balances = default_vars['balances'] %}
{% set block_number = false %}
{% set block_number = default_vars['block_number'] %}
{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %}
{# Log configuration details #}
@ -25,7 +25,7 @@
{# Set up dbt configuration #}
{{ config (
materialized = 'view',
tags = ['bronze_core_streamline_v1']
tags = ['bronze_core']
) }}
{# Main query starts here #}
@ -37,4 +37,4 @@
balances = balances,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}
) }}

12
models/streamline/bronze/core/bronze__blocks_fr_v2.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'BLOCKS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = 'V2'%}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
@ -12,14 +12,8 @@
{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %}
{# Log configuration details #}
{{ log_bronze_details(
source_name = source_name,
source_version = source_version,
model_type = model_type,
partition_function = partition_function,
partition_join_key = partition_join_key,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
{{ log_model_details(
vars = default_vars
) }}
{# Set up dbt configuration #}

2
models/streamline/bronze/core/bronze__confirm_blocks.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'CONFIRM_BLOCKS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = '' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

15
models/streamline/bronze/core/bronze__confirm_blocks_fr.sql
View File

@ -12,17 +12,4 @@ SELECT
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__confirm_blocks_fr_v2') }}
{% if var('GLOBAL_USES_STREAMLINE_V1', false) %}
UNION ALL
SELECT
_partition_by_block_id AS partition_key,
block_number,
VALUE,
DATA,
metadata,
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__confirm_blocks_fr_v1') }}
{% endif %}
{{ ref('bronze__confirm_blocks_fr_v2') }}

40
models/streamline/bronze/core/bronze__confirm_blocks_fr_v1.sql
View File

@ -1,40 +0,0 @@
{# Set variables #}
{% set source_name = 'CONFIRM_BLOCKS' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %}
{% set partition_join_key = '_partition_by_block_id' %}
{% set balances = default_vars['balances'] %}
{% set block_number = false %}
{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %}
{# Log configuration details #}
{{ log_bronze_details(
source_name = source_name,
source_version = source_version,
model_type = model_type,
partition_function = partition_function,
partition_join_key = partition_join_key,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}
{# Set up dbt configuration #}
{{ config (
materialized = 'view',
tags = ['bronze_core_streamline_v1']
) }}
{# Main query starts here #}
{{ streamline_external_table_query_fr(
source_name = source_name.lower(),
source_version = source_version.lower(),
partition_function = partition_function,
partition_join_key = partition_join_key,
balances = balances,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}

2
models/streamline/bronze/core/bronze__confirm_blocks_fr_v2.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'CONFIRM_BLOCKS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

2
models/streamline/bronze/core/bronze__receipts.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'RECEIPTS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = '' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

24
models/streamline/bronze/core/bronze__receipts_fr.sql
View File

@ -13,26 +13,4 @@ SELECT
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__receipts_fr_v2') }}
{% if var('GLOBAL_USES_STREAMLINE_V1',false) %}
UNION ALL
SELECT
_partition_by_block_id AS partition_key,
block_number,
COALESCE(
VALUE :"array_index" :: INT,
TRY_TO_NUMBER(
utils.udf_hex_to_int(
VALUE :"data" :"transactionIndex" :: STRING
)
)
) AS array_index,
VALUE,
DATA,
metadata,
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__receipts_fr_v1') }}
{% endif %}
{{ ref('bronze__receipts_fr_v2') }}

40
models/streamline/bronze/core/bronze__receipts_fr_v1.sql
View File

@ -1,40 +0,0 @@
{# Set variables #}
{% set source_name = 'RECEIPTS' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %}
{% set partition_join_key = '_partition_by_block_id' %}
{% set balances = default_vars['balances'] %}
{% set block_number = false %}
{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %}
{# Log configuration details #}
{{ log_bronze_details(
source_name = source_name,
source_version = source_version,
model_type = model_type,
partition_function = partition_function,
partition_join_key = partition_join_key,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}
{# Set up dbt configuration #}
{{ config (
materialized = 'view',
tags = ['bronze_core_streamline_v1','bronze_receipts']
) }}
{# Main query starts here #}
{{ streamline_external_table_query_fr(
source_name = source_name.lower(),
source_version = source_version.lower(),
partition_function = partition_function,
partition_join_key = partition_join_key,
balances = balances,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}

2
models/streamline/bronze/core/bronze__receipts_fr_v2.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'RECEIPTS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

2
models/streamline/bronze/core/bronze__traces.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'TRACES' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = '' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

16
models/streamline/bronze/core/bronze__traces_fr.sql
View File

@ -13,18 +13,4 @@ SELECT
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__traces_fr_v2') }}
{% if var('GLOBAL_USES_STREAMLINE_V1', false) %}
UNION ALL
SELECT
_partition_by_block_id AS partition_key,
block_number,
VALUE :"array_index" :: INT AS array_index,
VALUE,
DATA,
metadata,
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__traces_fr_v1') }}
{% endif %}
{{ ref('bronze__traces_fr_v2') }}

40
models/streamline/bronze/core/bronze__traces_fr_v1.sql
View File

@ -1,40 +0,0 @@
{# Set variables #}
{% set source_name = 'DEBUG_TRACEBLOCKBYNUMBER' if var('GLOBAL_USES_SINGLE_FLIGHT_METHOD',false) else 'TRACES' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %}
{% set partition_join_key = '_partition_by_block_id' %}
{% set balances = default_vars['balances'] %}
{% set block_number = false %}
{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %}
{# Log configuration details #}
{{ log_bronze_details(
source_name = source_name,
source_version = source_version,
model_type = model_type,
partition_function = partition_function,
partition_join_key = partition_join_key,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}
{# Set up dbt configuration #}
{{ config (
materialized = 'view',
tags = ['bronze_core_streamline_v1']
) }}
{# Main query starts here #}
{{ streamline_external_table_query_fr(
source_name = source_name.lower(),
source_version = source_version.lower(),
partition_function = partition_function,
partition_join_key = partition_join_key,
balances = balances,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}

2
models/streamline/bronze/core/bronze__traces_fr_v2.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'TRACES' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

2
models/streamline/bronze/core/bronze__transactions.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'TRANSACTIONS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) or var('GLOBAL_USES_BLOCKS_TRANSACTIONS_PATH', false) else '' %}
{% set source_version = 'V2' %}
{% set model_type = '' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

17
models/streamline/bronze/core/bronze__transactions_fr.sql
View File

@ -13,20 +13,6 @@ SELECT
_inserted_timestamp
FROM
{{ ref('bronze__transactions_fr_v2') }}
{% if var('GLOBAL_USES_STREAMLINE_V1', false) %}
UNION ALL
SELECT
_partition_by_block_id AS partition_key,
block_number,
VALUE,
DATA,
metadata,
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__transactions_fr_v1') }}
{% endif %}
{% if var('GLOBAL_USES_BLOCKS_TRANSACTIONS_PATH', false) %}
UNION ALL
SELECT
partition_key,
@ -37,5 +23,4 @@ SELECT
file_name,
_inserted_timestamp
FROM
{{ ref('bronze__transactions_fr_v2_1') }}
{% endif %}
{{ ref('bronze__transactions_fr_v1') }}

10
models/streamline/bronze/core/bronze__transactions_fr_v1.sql
View File

@ -5,10 +5,10 @@
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
{% set partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)" %}
{% set partition_join_key = '_partition_by_block_id' %}
{% set partition_function = default_vars['partition_function'] %}
{% set partition_join_key = default_vars['partition_join_key'] %}
{% set balances = default_vars['balances'] %}
{% set block_number = false %}
{% set block_number = default_vars['block_number'] %}
{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %}
{# Log configuration details #}
@ -25,7 +25,7 @@
{# Set up dbt configuration #}
{{ config (
materialized = 'view',
tags = ['bronze_core_streamline_v1']
tags = ['bronze_core']
) }}
{# Main query starts here #}
@ -37,4 +37,4 @@
balances = balances,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
) }}
) }}

12
models/streamline/bronze/core/bronze__transactions_fr_v2.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'TRANSACTIONS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = 'V2'%}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
@ -12,14 +12,8 @@
{% set uses_receipts_by_hash = default_vars['uses_receipts_by_hash'] %}
{# Log configuration details #}
{{ log_bronze_details(
source_name = source_name,
source_version = source_version,
model_type = model_type,
partition_function = partition_function,
partition_join_key = partition_join_key,
block_number = block_number,
uses_receipts_by_hash = uses_receipts_by_hash
{{ log_model_details(
vars = default_vars
) }}
{# Set up dbt configuration #}

2
models/streamline/bronze/decoder/bronze__decoded_logs.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'DECODED_LOGS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = '' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

9
models/streamline/bronze/decoder/bronze__decoded_logs_fr.sql
View File

@ -10,11 +10,4 @@
SELECT
*
FROM
{{ ref('bronze__decoded_logs_fr_v2') }}
{% if var('GLOBAL_USES_STREAMLINE_V1', false) %}
UNION ALL
SELECT
*
FROM
{{ ref('bronze__decoded_logs_fr_v1') }}
{% endif %}
{{ ref('bronze__decoded_logs_fr_v2') }}

23
models/streamline/bronze/decoder/bronze__decoded_logs_fr_v1.sql
View File

@ -1,23 +0,0 @@
{# Set variables #}
{% set source_name = 'DECODED_LOGS' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}
{# Log configuration details #}
{{ log_model_details(
vars = default_vars
) }}
{# Set up dbt configuration #}
{{ config (
materialized = 'view',
tags = ['bronze_decoded_logs_streamline_v1']
) }}
{# Main query starts here #}
{{ streamline_external_table_query_decoder_fr(
source_name = source_name.lower(),
source_version = source_version.lower()
) }}

2
models/streamline/bronze/decoder/bronze__decoded_logs_fr_v2.sql
View File

@ -1,6 +1,6 @@
{# Set variables #}
{% set source_name = 'DECODED_LOGS' %}
{% set source_version = 'V2' if var('GLOBAL_USES_STREAMLINE_V1', false) else '' %}
{% set source_version = '' %}
{% set model_type = 'FR' %}
{%- set default_vars = set_default_variables_bronze(source_name, model_type) -%}

18
package-lock.yml
View File

@ -1,18 +0,0 @@
packages:
- package: calogica/dbt_expectations
version: 0.8.2
- package: dbt-labs/dbt_external_tables
version: 0.8.2
- package: dbt-labs/dbt_utils
version: 1.0.0
- git: https://github.com/FlipsideCrypto/fsc-utils.git
revision: eb33ac727af26ebc8a8cc9711d4a6ebc3790a107
- package: get-select/dbt_snowflake_query_tags
version: 2.5.0
- git: https://github.com/FlipsideCrypto/fsc-evm.git
revision: ec6adae14ab4060ad4a553fb7f32d7e57693996d
- package: calogica/dbt_date
version: 0.7.2
- git: https://github.com/FlipsideCrypto/livequery-models.git
revision: b024188be4e9c6bc00ed77797ebdc92d351d620e
sha1_hash: 622a679ecf98e6ebf3c904241902ce5328c77e52