From e2ef3af72d96af3b9dcf994faa197726c130cafc Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 4 Feb 2025 12:31:50 -0700 Subject: [PATCH] blocks silver --- models/bronze/bronze__FR_blocks.sql | 1 + .../silver__actions_events_addkey_s3.sql | 2 + ...ilver__actions_events_function_call_s3.sql | 3 ++ .../actions/silver__actions_events_s3.sql | 2 + .../atlas/silver__atlas_accounts_created.sql | 2 + .../silver__atlas_address_first_action.sql | 2 + models/silver/atlas/silver__atlas_maa.sql | 2 + .../atlas/silver__atlas_nft_30_trailing.sql | 2 + .../atlas/silver__atlas_nft_detailed.sql | 2 + .../silver/atlas/silver__atlas_nft_table.sql | 2 + .../atlas/silver__atlas_nft_transactions.sql | 2 + .../core/{ => lake}/helpers/_retry_range.sql | 0 .../helpers/silver__flatten_receipts.sql | 0 .../silver__receipt_tx_hash_mapping.sql | 0 .../{ => lake}/silver__streamline_blocks.sql | 1 + .../{ => lake}/silver__streamline_blocks.yml | 0 .../silver__streamline_receipts.sql | 0 .../silver__streamline_receipts.yml | 0 .../silver__streamline_receipts_final.sql | 0 .../silver__streamline_receipts_final.yml | 0 .../{ => lake}/silver__streamline_shards.sql | 0 .../{ => lake}/silver__streamline_shards.yml | 0 .../silver__streamline_transactions.sql | 0 .../silver__streamline_transactions.yml | 0 .../silver__streamline_transactions_final.sql | 0 .../silver__streamline_transactions_final.yml | 0 models/silver/core/silver__blocks_v2.sql | 46 +++++++++++++++++++ .../{core => curated}/silver__logs_s3.sql | 0 .../{core => curated}/silver__logs_s3.yml | 0 .../complete/streamline__blocks_complete.sql | 2 +- .../realtime/streamline__blocks_realtime.sql | 13 +++--- 31 files changed, 76 insertions(+), 8 deletions(-) rename models/silver/core/{ => lake}/helpers/_retry_range.sql (100%) rename models/silver/core/{ => lake}/helpers/silver__flatten_receipts.sql (100%) rename models/silver/core/{ => lake}/helpers/silver__receipt_tx_hash_mapping.sql (100%) rename models/silver/core/{ => lake}/silver__streamline_blocks.sql (99%) rename models/silver/core/{ => lake}/silver__streamline_blocks.yml (100%) rename models/silver/core/{ => lake}/silver__streamline_receipts.sql (100%) rename models/silver/core/{ => lake}/silver__streamline_receipts.yml (100%) rename models/silver/core/{ => lake}/silver__streamline_receipts_final.sql (100%) rename models/silver/core/{ => lake}/silver__streamline_receipts_final.yml (100%) rename models/silver/core/{ => lake}/silver__streamline_shards.sql (100%) rename models/silver/core/{ => lake}/silver__streamline_shards.yml (100%) rename models/silver/core/{ => lake}/silver__streamline_transactions.sql (100%) rename models/silver/core/{ => lake}/silver__streamline_transactions.yml (100%) rename models/silver/core/{ => lake}/silver__streamline_transactions_final.sql (100%) rename models/silver/core/{ => lake}/silver__streamline_transactions_final.yml (100%) create mode 100644 models/silver/core/silver__blocks_v2.sql rename models/silver/{core => curated}/silver__logs_s3.sql (100%) rename models/silver/{core => curated}/silver__logs_s3.yml (100%) diff --git a/models/bronze/bronze__FR_blocks.sql b/models/bronze/bronze__FR_blocks.sql index a03bc49..f7f045d 100644 --- a/models/bronze/bronze__FR_blocks.sql +++ b/models/bronze/bronze__FR_blocks.sql @@ -1,3 +1,4 @@ +-- TODO - v2 naming? {{ config ( materialized = 'view', tags = ['streamline_helper'] diff --git a/models/silver/actions/silver__actions_events_addkey_s3.sql b/models/silver/actions/silver__actions_events_addkey_s3.sql index 0f784af..7dd8090 100644 --- a/models/silver/actions/silver__actions_events_addkey_s3.sql +++ b/models/silver/actions/silver__actions_events_addkey_s3.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop +-- Note - a model in Social does use this {{ config( materialized = 'incremental', incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], diff --git a/models/silver/actions/silver__actions_events_function_call_s3.sql b/models/silver/actions/silver__actions_events_function_call_s3.sql index 4cec0cf..abffd80 100644 --- a/models/silver/actions/silver__actions_events_function_call_s3.sql +++ b/models/silver/actions/silver__actions_events_function_call_s3.sql @@ -1,3 +1,6 @@ +-- TODO slated for deprecation and drop +-- Note - must migrate curated to new ez_actions first + {{ config( materialized = 'incremental', incremental_strategy = 'merge', diff --git a/models/silver/actions/silver__actions_events_s3.sql b/models/silver/actions/silver__actions_events_s3.sql index f03c4df..f939946 100644 --- a/models/silver/actions/silver__actions_events_s3.sql +++ b/models/silver/actions/silver__actions_events_s3.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop +-- Note - must migrate curated to new ez_actions first {{ config( materialized = 'incremental', incremental_strategy = 'merge', diff --git a/models/silver/atlas/silver__atlas_accounts_created.sql b/models/silver/atlas/silver__atlas_accounts_created.sql index b8c8639..e9a2969 100644 --- a/models/silver/atlas/silver__atlas_accounts_created.sql +++ b/models/silver/atlas/silver__atlas_accounts_created.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop + {{ config( materialized = 'incremental', unique_key = 'atlas_account_created_id', diff --git a/models/silver/atlas/silver__atlas_address_first_action.sql b/models/silver/atlas/silver__atlas_address_first_action.sql index b92f8f0..2d5653b 100644 --- a/models/silver/atlas/silver__atlas_address_first_action.sql +++ b/models/silver/atlas/silver__atlas_address_first_action.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop + {{ config( materialized = 'incremental', incremental_stratege = 'merge', diff --git a/models/silver/atlas/silver__atlas_maa.sql b/models/silver/atlas/silver__atlas_maa.sql index dccf360..a34b5fc 100644 --- a/models/silver/atlas/silver__atlas_maa.sql +++ b/models/silver/atlas/silver__atlas_maa.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop + {{ config( materialized = 'incremental', incremental_stratege = 'merge', diff --git a/models/silver/atlas/silver__atlas_nft_30_trailing.sql b/models/silver/atlas/silver__atlas_nft_30_trailing.sql index 8491216..8de466e 100644 --- a/models/silver/atlas/silver__atlas_nft_30_trailing.sql +++ b/models/silver/atlas/silver__atlas_nft_30_trailing.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop + {{ config( materialized = 'incremental', unique_key = 'atlas_nft_30_trailing_id', diff --git a/models/silver/atlas/silver__atlas_nft_detailed.sql b/models/silver/atlas/silver__atlas_nft_detailed.sql index 18a3e75..933b029 100644 --- a/models/silver/atlas/silver__atlas_nft_detailed.sql +++ b/models/silver/atlas/silver__atlas_nft_detailed.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop + {{ config( materialized = 'table', unique_key = 'atlas_nft_detailed_id', diff --git a/models/silver/atlas/silver__atlas_nft_table.sql b/models/silver/atlas/silver__atlas_nft_table.sql index 3da02ac..2950dd0 100644 --- a/models/silver/atlas/silver__atlas_nft_table.sql +++ b/models/silver/atlas/silver__atlas_nft_table.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop + {{ config( materialized = 'table', unique_key = 'atlas_nft_table_id', diff --git a/models/silver/atlas/silver__atlas_nft_transactions.sql b/models/silver/atlas/silver__atlas_nft_transactions.sql index 9234881..e69ada6 100644 --- a/models/silver/atlas/silver__atlas_nft_transactions.sql +++ b/models/silver/atlas/silver__atlas_nft_transactions.sql @@ -1,3 +1,5 @@ +-- TODO slated for deprecation and drop + {{ config( materialized = "incremental", cluster_by = ["day"], diff --git a/models/silver/core/helpers/_retry_range.sql b/models/silver/core/lake/helpers/_retry_range.sql similarity index 100% rename from models/silver/core/helpers/_retry_range.sql rename to models/silver/core/lake/helpers/_retry_range.sql diff --git a/models/silver/core/helpers/silver__flatten_receipts.sql b/models/silver/core/lake/helpers/silver__flatten_receipts.sql similarity index 100% rename from models/silver/core/helpers/silver__flatten_receipts.sql rename to models/silver/core/lake/helpers/silver__flatten_receipts.sql diff --git a/models/silver/core/helpers/silver__receipt_tx_hash_mapping.sql b/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql similarity index 100% rename from models/silver/core/helpers/silver__receipt_tx_hash_mapping.sql rename to models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql diff --git a/models/silver/core/silver__streamline_blocks.sql b/models/silver/core/lake/silver__streamline_blocks.sql similarity index 99% rename from models/silver/core/silver__streamline_blocks.sql rename to models/silver/core/lake/silver__streamline_blocks.sql index ca2105d..e76813b 100644 --- a/models/silver/core/silver__streamline_blocks.sql +++ b/models/silver/core/lake/silver__streamline_blocks.sql @@ -1,3 +1,4 @@ +-- Deprecated Process {{ config( materialized = 'incremental', incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], diff --git a/models/silver/core/silver__streamline_blocks.yml b/models/silver/core/lake/silver__streamline_blocks.yml similarity index 100% rename from models/silver/core/silver__streamline_blocks.yml rename to models/silver/core/lake/silver__streamline_blocks.yml diff --git a/models/silver/core/silver__streamline_receipts.sql b/models/silver/core/lake/silver__streamline_receipts.sql similarity index 100% rename from models/silver/core/silver__streamline_receipts.sql rename to models/silver/core/lake/silver__streamline_receipts.sql diff --git a/models/silver/core/silver__streamline_receipts.yml b/models/silver/core/lake/silver__streamline_receipts.yml similarity index 100% rename from models/silver/core/silver__streamline_receipts.yml rename to models/silver/core/lake/silver__streamline_receipts.yml diff --git a/models/silver/core/silver__streamline_receipts_final.sql b/models/silver/core/lake/silver__streamline_receipts_final.sql similarity index 100% rename from models/silver/core/silver__streamline_receipts_final.sql rename to models/silver/core/lake/silver__streamline_receipts_final.sql diff --git a/models/silver/core/silver__streamline_receipts_final.yml b/models/silver/core/lake/silver__streamline_receipts_final.yml similarity index 100% rename from models/silver/core/silver__streamline_receipts_final.yml rename to models/silver/core/lake/silver__streamline_receipts_final.yml diff --git a/models/silver/core/silver__streamline_shards.sql b/models/silver/core/lake/silver__streamline_shards.sql similarity index 100% rename from models/silver/core/silver__streamline_shards.sql rename to models/silver/core/lake/silver__streamline_shards.sql diff --git a/models/silver/core/silver__streamline_shards.yml b/models/silver/core/lake/silver__streamline_shards.yml similarity index 100% rename from models/silver/core/silver__streamline_shards.yml rename to models/silver/core/lake/silver__streamline_shards.yml diff --git a/models/silver/core/silver__streamline_transactions.sql b/models/silver/core/lake/silver__streamline_transactions.sql similarity index 100% rename from models/silver/core/silver__streamline_transactions.sql rename to models/silver/core/lake/silver__streamline_transactions.sql diff --git a/models/silver/core/silver__streamline_transactions.yml b/models/silver/core/lake/silver__streamline_transactions.yml similarity index 100% rename from models/silver/core/silver__streamline_transactions.yml rename to models/silver/core/lake/silver__streamline_transactions.yml diff --git a/models/silver/core/silver__streamline_transactions_final.sql b/models/silver/core/lake/silver__streamline_transactions_final.sql similarity index 100% rename from models/silver/core/silver__streamline_transactions_final.sql rename to models/silver/core/lake/silver__streamline_transactions_final.sql diff --git a/models/silver/core/silver__streamline_transactions_final.yml b/models/silver/core/lake/silver__streamline_transactions_final.yml similarity index 100% rename from models/silver/core/silver__streamline_transactions_final.yml rename to models/silver/core/lake/silver__streamline_transactions_final.yml diff --git a/models/silver/core/silver__blocks_v2.sql b/models/silver/core/silver__blocks_v2.sql new file mode 100644 index 0000000..b3c8ac5 --- /dev/null +++ b/models/silver/core/silver__blocks_v2.sql @@ -0,0 +1,46 @@ +-- depends_on: {{ ref('bronze__blocks') }} +-- depends_on: {{ ref('bronze__FR_blocks') }} + +{{ config ( + materialized = "incremental", + incremental_strategy = 'merge', + unique_key = "block_hash", + cluster_by = ['modified_timestamp::DATE','partition_key'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['scheduled_core'] +) }} + +WITH bronze_blocks AS ( + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + DATA :header :hash :: STRING AS block_hash, + partition_key, + DATA :: VARIANT AS block_json, + _inserted_timestamp + FROM + {% if is_incremental() %} + {{ ref('bronze__blocks') }} + WHERE _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1900-01-01'::TIMESTAMP) AS _inserted_timestamp + FROM {{ this }} + ) AND DATA IS NOT NULL + {% else %} + {{ ref('bronze__FR_blocks') }} + WHERE DATA IS NOT NULL + {% endif %} +) + +SELECT + block_number, + block_hash, + partition_key, + block_json, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['block_hash']) }} AS blocks_v2_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM bronze_blocks + +QUALIFY ROW_NUMBER() OVER (PARTITION BY block_hash ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/core/silver__logs_s3.sql b/models/silver/curated/silver__logs_s3.sql similarity index 100% rename from models/silver/core/silver__logs_s3.sql rename to models/silver/curated/silver__logs_s3.sql diff --git a/models/silver/core/silver__logs_s3.yml b/models/silver/curated/silver__logs_s3.yml similarity index 100% rename from models/silver/core/silver__logs_s3.yml rename to models/silver/curated/silver__logs_s3.yml diff --git a/models/streamline/core/complete/streamline__blocks_complete.sql b/models/streamline/core/complete/streamline__blocks_complete.sql index 46adf6d..c6d2a65 100644 --- a/models/streamline/core/complete/streamline__blocks_complete.sql +++ b/models/streamline/core/complete/streamline__blocks_complete.sql @@ -10,7 +10,7 @@ ) }} SELECT - VALUE :block_number :: INT AS block_number, + VALUE :BLOCK_NUMBER :: INT AS block_number, DATA :header :hash :: STRING AS block_hash, partition_key, _inserted_timestamp, diff --git a/models/streamline/core/realtime/streamline__blocks_realtime.sql b/models/streamline/core/realtime/streamline__blocks_realtime.sql index 2c92ed4..46d96ac 100644 --- a/models/streamline/core/realtime/streamline__blocks_realtime.sql +++ b/models/streamline/core/realtime/streamline__blocks_realtime.sql @@ -5,10 +5,11 @@ target = "{{this.schema}}.{{this.identifier}}", params = { "external_table": "blocks_v2", - "sql_limit": "100", - "producer_batch_size": "100", - "worker_batch_size": "100", - "sql_source": "{{this.identifier}}" + "sql_limit": "3000", + "producer_batch_size": "3000", + "worker_batch_size": "3000", + "sql_source": "{{this.identifier}}", + "order_by_column": "block_number DESC" } ), tags = ['streamline_realtime'] @@ -53,7 +54,7 @@ tbl AS ( -4, SYSDATE() ) - -- AND {} IS NOT NULL -- TODO, determine identifier for bad response + AND block_hash IS NOT NULL ) SELECT block_number, @@ -75,5 +76,3 @@ SELECT 'Vault/prod/near/quicknode/mainnet' ) AS request FROM tbl -ORDER BY - block_number DESC