diff --git a/models/evm/silver/core/silver_evm__blocks.sql b/models/evm/silver/core/silver_evm__blocks.sql index 0a8335d..8493f00 100644 --- a/models/evm/silver/core/silver_evm__blocks.sql +++ b/models/evm/silver/core/silver_evm__blocks.sql @@ -74,7 +74,7 @@ WHERE {{ this }} ) {% else %} - {{ ref('bronze_evm__streamline_FR_blocks') }} + {{ ref('bronze_evm__streamline_fr_blocks') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/evm/silver/core/silver_evm__confirmed_blocks.sql b/models/evm/silver/core/silver_evm__confirmed_blocks.sql index e8d67d2..754f332 100644 --- a/models/evm/silver/core/silver_evm__confirmed_blocks.sql +++ b/models/evm/silver/core/silver_evm__confirmed_blocks.sql @@ -31,7 +31,7 @@ WHERE {{ this }} ) {% else %} - {{ ref('bronze_evm__streamline_FR_confirm_blocks') }} + {{ ref('bronze_evm__streamline_fr_confirm_blocks') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/evm/silver/core/silver_evm__fact_traces2.sql b/models/evm/silver/core/silver_evm__fact_traces2.sql new file mode 100644 index 0000000..402a6b4 --- /dev/null +++ b/models/evm/silver/core/silver_evm__fact_traces2.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + incremental_predicates = [fsc_evm.standard_predicate()], + cluster_by = "block_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['traces_reload'] +) }} +{{ fsc_evm.gold_traces_v1( + full_reload_start_block = 110000000, + full_reload_blocks = 5000000, + full_reload_mode = true, + schema_name = 'silver_evm', + sei_traces_mode = true +) }} \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__receipts.sql b/models/evm/silver/core/silver_evm__receipts.sql index fe7ef17..8577619 100644 --- a/models/evm/silver/core/silver_evm__receipts.sql +++ b/models/evm/silver/core/silver_evm__receipts.sql @@ -27,7 +27,7 @@ WHERE ) AND IS_OBJECT(DATA) {% else %} - {{ ref('bronze_evm__streamline_FR_receipts') }} + {{ ref('bronze_evm__streamline_fr_receipts') }} WHERE IS_OBJECT(DATA) {% endif %} diff --git a/models/evm/silver/core/silver_evm__traces.sql b/models/evm/silver/core/silver_evm__traces.sql index 920190b..e788bf1 100644 --- a/models/evm/silver/core/silver_evm__traces.sql +++ b/models/evm/silver/core/silver_evm__traces.sql @@ -33,7 +33,7 @@ WHERE ) AND DATA :result IS NOT NULL {% else %} - {{ ref('bronze_evm__streamline_FR_traces') }} + {{ ref('bronze_evm__streamline_fr_traces') }} WHERE DATA :result IS NOT NULL {% endif %} diff --git a/models/evm/silver/core/silver_evm__traces2.sql b/models/evm/silver/core/silver_evm__traces2.sql index fe19a0b..57ef0aa 100644 --- a/models/evm/silver/core/silver_evm__traces2.sql +++ b/models/evm/silver/core/silver_evm__traces2.sql @@ -3,142 +3,16 @@ materialized = "incremental", incremental_strategy = 'delete+insert', unique_key = "block_number", + incremental_predicates = [fsc_evm.standard_predicate()], cluster_by = ['modified_timestamp::DATE','partition_key'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", full_refresh = false, tags = ['core'] ) }} -{% if execute %} - {% set query_traces %} - CREATE - OR REPLACE temporary TABLE silver_evm.traces2__traces_intermediate_tmp AS - - SELECT - VALUE :BLOCK_NUMBER :: INT AS block_number, - partition_key, - VALUE :array_index :: INT AS tx_position, - DATA :result AS full_traces, - DATA :txHash :: STRING AS tx_hash, - _inserted_timestamp - FROM - -{% if is_incremental() %} -{{ ref('bronze_evm__streamline_traces') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) - AND DATA :result IS NOT NULL -{% else %} - {{ ref('bronze_evm__streamline_FR_traces') }} -WHERE - partition_key <= 80124000 - AND DATA :result IS NOT NULL -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position -ORDER BY - _inserted_timestamp DESC)) = 1 {% endset %} - {% do run_query( - query_traces - ) %} -{% endif %} - -WITH flatten_traces AS ( - SELECT - block_number, - tx_hash, - partition_key, - IFF( - path IN ( - 'result', - 'result.value', - 'result.type', - 'result.to', - 'result.input', - 'result.gasUsed', - 'result.gas', - 'result.from', - 'result.output', - 'result.error', - 'result.revertReason', - 'gasUsed', - 'gas', - 'type', - 'to', - 'from', - 'value', - 'input', - 'error', - 'output', - 'revertReason' - ), - 'ORIGIN', - REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') - ) AS trace_address, - _inserted_timestamp, - OBJECT_AGG( - key, - VALUE - ) AS trace_json, - CASE - WHEN trace_address = 'ORIGIN' THEN NULL - WHEN POSITION( - '_' IN trace_address - ) = 0 THEN 'ORIGIN' - ELSE REGEXP_REPLACE( - trace_address, - '_[0-9]+$', - '', - 1, - 1 - ) - END AS parent_trace_address, - SPLIT( - trace_address, - '_' - ) AS trace_address_array - FROM - silver_evm.traces2__traces_intermediate_tmp txs, - TABLE( - FLATTEN( - input => PARSE_JSON( - txs.full_traces - ), - recursive => TRUE - ) - ) f - WHERE - f.index IS NULL - AND f.key != 'calls' - AND f.path != 'result' - GROUP BY - block_number, - tx_hash, - partition_key, - trace_address, - _inserted_timestamp -) -SELECT - block_number, - tx_hash, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - partition_key, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'tx_hash', 'trace_address'] - ) }} AS traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - flatten_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id -ORDER BY - _inserted_timestamp DESC)) = 1 +{{ fsc_evm.silver_traces_v1( + full_reload_start_block = 80000000, + full_reload_blocks = 5000000, + use_partition_key = true, + schema_name = 'bronze_evm', + sei_traces_mode = true +) }} \ No newline at end of file diff --git a/models/evm/silver/core/silver_evm__transactions.sql b/models/evm/silver/core/silver_evm__transactions.sql index 121a26c..9e9f4fd 100644 --- a/models/evm/silver/core/silver_evm__transactions.sql +++ b/models/evm/silver/core/silver_evm__transactions.sql @@ -27,7 +27,7 @@ WHERE ) AND IS_OBJECT(DATA) {% else %} - {{ ref('bronze_evm__streamline_FR_transactions') }} + {{ ref('bronze_evm__streamline_fr_transactions') }} WHERE IS_OBJECT(DATA) {% endif %} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql index fa5bd5b..d5c9001 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_blocks.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_query_v2( +{{ fsc_evm.streamline_external_table_query( model = "evm_blocks", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql index 3d04f3d..cb81c71 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_confirm_blocks.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_query_v2( +{{ fsc_evm.streamline_external_table_query( model = "evm_confirm_blocks", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql index 71c690c..b338a98 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_receipts.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_query_v2( +{{ fsc_evm.streamline_external_table_query( model = "evm_receipts", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql index e68e330..3db0901 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_traces.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_query_v2( +{{ fsc_evm.streamline_external_table_query( model = "evm_traces", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql b/models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql index 10d3202..82f42bc 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql +++ b/models/evm/streamline/bronze/core/bronze_evm__streamline_transactions.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_query_v2( +{{ fsc_evm.streamline_external_table_query( model = "evm_transactions", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_blocks.sql b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_blocks.sql similarity index 78% rename from models/evm/streamline/bronze/core/bronze_evm__streamline_FR_blocks.sql rename to models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_blocks.sql index 1f83892..75fd420 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_blocks.sql +++ b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_blocks.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_FR_query_v2( +{{ fsc_evm.streamline_external_table_fr_query( model = "evm_blocks", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_confirm_blocks.sql b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_confirm_blocks.sql similarity index 79% rename from models/evm/streamline/bronze/core/bronze_evm__streamline_FR_confirm_blocks.sql rename to models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_confirm_blocks.sql index fe75640..d5b95dc 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_confirm_blocks.sql +++ b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_confirm_blocks.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_FR_query_v2( +{{ fsc_evm.streamline_external_table_fr_query( model = "evm_confirm_blocks", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_receipts.sql b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_receipts.sql similarity index 78% rename from models/evm/streamline/bronze/core/bronze_evm__streamline_FR_receipts.sql rename to models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_receipts.sql index f5c0682..386b163 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_receipts.sql +++ b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_receipts.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_FR_query_v2( +{{ fsc_evm.streamline_external_table_fr_query( model = "evm_receipts", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_traces.sql b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_traces.sql similarity index 78% rename from models/evm/streamline/bronze/core/bronze_evm__streamline_FR_traces.sql rename to models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_traces.sql index c8aa12d..40b34af 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_traces.sql +++ b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_traces.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_FR_query_v2( +{{ fsc_evm.streamline_external_table_fr_query( model = "evm_traces", partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_transactions.sql b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_transactions.sql similarity index 78% rename from models/evm/streamline/bronze/core/bronze_evm__streamline_FR_transactions.sql rename to models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_transactions.sql index ce54050..a992ad5 100644 --- a/models/evm/streamline/bronze/core/bronze_evm__streamline_FR_transactions.sql +++ b/models/evm/streamline/bronze/core/fr/bronze_evm__streamline_fr_transactions.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_FR_query_v2( +{{ fsc_evm.streamline_external_table_fr_query( model = 'evm_transactions', partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )" ) }} diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql index fdab8b2..f341175 100644 --- a/models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_blocks.sql @@ -27,7 +27,7 @@ WHERE FROM {{ this }}) {% else %} - {{ ref('bronze_evm__streamline_FR_blocks') }} + {{ ref('bronze_evm__streamline_fr_blocks') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql index f4abd3c..64f725b 100644 --- a/models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_confirmed_blocks.sql @@ -27,7 +27,7 @@ WHERE FROM {{ this }}) {% else %} - {{ ref('bronze_evm__streamline_FR_confirm_blocks') }} + {{ ref('bronze_evm__streamline_fr_confirm_blocks') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql index f904bcb..ca8870c 100644 --- a/models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_receipts.sql @@ -27,7 +27,7 @@ WHERE FROM {{ this }}) {% else %} - {{ ref('bronze_evm__streamline_FR_receipts') }} + {{ ref('bronze_evm__streamline_fr_receipts') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql index 7160b3a..9ee24cf 100644 --- a/models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_traces.sql @@ -27,7 +27,7 @@ WHERE FROM {{ this }}) {% else %} - {{ ref('bronze_evm__streamline_FR_traces') }} + {{ ref('bronze_evm__streamline_fr_traces') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql b/models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql index 47473b6..24554a7 100644 --- a/models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql +++ b/models/evm/streamline/silver/core/complete/streamline__complete_evm_transactions.sql @@ -27,7 +27,7 @@ WHERE FROM {{ this }}) {% else %} - {{ ref('bronze_evm__streamline_FR_transactions') }} + {{ ref('bronze_evm__streamline_fr_transactions') }} {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number diff --git a/package-lock.yml b/package-lock.yml index faab605..58adbb0 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -7,8 +7,10 @@ packages: revision: eb33ac727af26ebc8a8cc9711d4a6ebc3790a107 - package: get-select/dbt_snowflake_query_tags version: 2.5.0 +- git: https://github.com/FlipsideCrypto/fsc-evm.git + revision: a700111699af1a355d9076dde9aa04e26efb0617 - package: calogica/dbt_date version: 0.7.2 - git: https://github.com/FlipsideCrypto/livequery-models.git revision: b024188be4e9c6bc00ed77797ebdc92d351d620e -sha1_hash: c392d105275def8b2e5ba9fb17cf0bfe2e6a4753 +sha1_hash: 54d3c6488e266ae1e51b95dff6e64016b4e803c9 diff --git a/packages.yml b/packages.yml index 5f8dc65..0117df3 100644 --- a/packages.yml +++ b/packages.yml @@ -6,4 +6,6 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-utils.git revision: v1.29.0 - package: get-select/dbt_snowflake_query_tags - version: [">=2.0.0", "<3.0.0"] \ No newline at end of file + version: [">=2.0.0", "<3.0.0"] + - git: https://github.com/FlipsideCrypto/fsc-evm.git + revision: v1.9.0 \ No newline at end of file