From 3ea67df7bbb6ca67632b9add6dcbcc0a62b36a40 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 18 Feb 2025 18:06:03 -0700 Subject: [PATCH 01/53] pass chunk/shard id thru to txs rt, txs final in prog, rec final in prog, ... fact_chunks (w header) ? --- models/silver/core/silver__chunks_v2.sql | 4 + models/silver/core/silver__receipts_final.sql | 131 +++++++++++++++ .../core/silver__transactions_final.sql | 155 ++++++++++++++++++ .../silver/core/silver__transactions_v2.sql | 12 +- .../streamline__transactions_complete.sql | 2 + .../streamline__transactions_realtime.sql | 8 +- 6 files changed, 304 insertions(+), 8 deletions(-) create mode 100644 models/silver/core/silver__receipts_final.sql create mode 100644 models/silver/core/silver__transactions_final.sql diff --git a/models/silver/core/silver__chunks_v2.sql b/models/silver/core/silver__chunks_v2.sql index fbcf1cf..c8832a1 100644 --- a/models/silver/core/silver__chunks_v2.sql +++ b/models/silver/core/silver__chunks_v2.sql @@ -9,6 +9,10 @@ post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(chunk_hash)", tags = ['scheduled_core', 'core_v2'] ) }} +-- TODO this can actually be dropped +-- chunks only needed intrapipeline to get tx hashes +-- unless deployed to a fact_chunks table with just the header ? ... +-- chunk author might be helpful to analysts looking at validator performance WITH bronze_chunks AS ( diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql new file mode 100644 index 0000000..e8c601a --- /dev/null +++ b/models/silver/core/silver__receipts_final.sql @@ -0,0 +1,131 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'receipt_id', + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number', ], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", + tags = ['scheduled_core'], + full_refresh = False +) }} +-- TODO if execute block for incremental min blockdate +WITH +{% if var('NEAR_MIGRATE_RECEIPTS', False) %} +lake_receipts_final as ( + select * from {{ ref('silver__streamline_receipts_final') }} + -- TODO incrementally load? + -- TODO rename cols? +), +{% else %} + +txs_with_receipts as ( +select * from {{ ref('silver__transactions_v2') }} +-- TODO incremental logic +), +blocks as ( + select * from {{ ref('silver__blocks_v2') }} + -- TODO incremental logic +), +flatten_receipts as ( + select + tx_hash, + -- chunk_hash, + -- shard_id, + value:predecessor_id::STRING as predecessor_id, + value:priority :: int as priority, + value:receipt :: variant as receipt_json, + value:receipt_id :: STRING as receipt_id, + value:receiver_id :: string as receiver_id + from tx_response, lateral flatten(input => response_json :receipts :: ARRAY) +), +flatten_receipt_outcomes as ( + select + value :block_hash :: STRING as block_hash, + value:id :: STRING as receipt_id, + value:outcome :: variant as outcome_json, + value:proof::array as proof + from tx_response, lateral flatten(input => response_json :receipts_outcome :: ARRAY) +), +receipts_full as ( +select + -- chunk_hash, + -- shard_id, + block_hash, + tx_hash, + r.receipt_id, + predecessor_id, + receiver_id, + priority, + receipt_json, + outcome_json, + proof, + outcome_json :status :Failure IS NULL AS receipt_succeeded, + TRY_PARSE_JSON( + outcome_json :status :Failure + ) AS failure_message, + object_keys( + failure_message + ) [0] :: STRING AS error_type_0, + COALESCE( + object_keys( + TRY_PARSE_JSON( + failure_message [error_type_0] :kind + ) + ) [0] :: STRING, + failure_message [error_type_0] :kind :: STRING + ) AS error_type_1, + COALESCE( + object_keys( + TRY_PARSE_JSON( + failure_message [error_type_0] :kind [error_type_1] + ) + ) [0] :: STRING, + failure_message [error_type_0] :kind [error_type_1] :: STRING + ) AS error_type_2, + failure_message [error_type_0] :kind [error_type_1] [error_type_2] :: STRING AS error_message +from flatten_receipts r left join flatten_receipt_outcomes ro +on r.receipt_id = ro.receipt_id +), +FINAL AS ( + SELECT + -- chunk_hash, + -- shard_id, + block_hash, + tx_hash, + receipt_id, + -- block_id, + -- block_timestamp, + predecessor_id, + receiver_id, + -- signer_id, -- this is more like action_signer_id tbh.. should be renamed + + receipt_json, + outcome_json, + -- outcome_json :receipt_ids :: ARRAY AS receipt_outcome_id, + -- receipt_type, -- this is not a thing + outcome_json :gas_burnt :: NUMBER AS gas_burnt, + outcome_json :status :: variant AS status_value, + outcome_json :logs :: ARRAY AS logs, + proof, + outcome_json :metadata :: variant AS metadata, + receipt_succeeded, + error_type_0, + error_type_1, + error_type_2, + error_message + -- _partition_by_block_number, + -- _inserted_timestamp + FROM + receipts_full +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['receipt_id'] + ) }} AS receipts_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql new file mode 100644 index 0000000..a7eeb65 --- /dev/null +++ b/models/silver/core/silver__transactions_final.sql @@ -0,0 +1,155 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'tx_hash', + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,signer_id,receiver_id);", + tags = ['scheduled_core'] +) }} + +WITH +txs_with_receipts as ( +select * from {{ ref('silver__streamline_transactions_final') }} +-- TODO incrementally load +), +transaction_full as ( + select + -- shard_id, + -- chunk_hash, + origin_block_id AS block_id, + origin_block_timestamp AS block_timestamp, + tx_hash, + response_json :transaction:actions :: ARRAY as actions, + response_json :transaction:nonce :: INT as nonce, + response_json:transaction:priority_fee :: INT as priority_fee, + response_json:transaction:public_key :: STRING as public_key, + response_json:transaction:receiver_id :: STRING as receiver_id, + response_json:transaction:signature :: STRING as signature, + response_json:transaction:signer_id :: STRING as signer_id, + response_json:transaction_outcome:block_hash :: STRING as block_hash, + response_json:transaction_outcome:outcome :: variant as outcome_json, + response_json:status::VARIANT as status_json + from txs_with_receipts +) +-- do we still want to calculate total used gas ? +-- well, no need to pull in any other table anymore as the outcome exists in the underlying record +-- can just flatten array and sum values +actions AS ( + SELECT + tx_hash, + SUM( + VALUE :FunctionCall :gas :: NUMBER + ) AS attached_gas + FROM + base_transactions, + LATERAL FLATTEN( + input => tx :actions + ) + GROUP BY + 1 +), +transactions AS ( + SELECT + block_id, + tx :outcome :block_hash :: STRING AS block_hash, + tx_hash, + block_timestamp, + tx :nonce :: NUMBER AS nonce, + tx :signature :: STRING AS signature, + tx :receiver_id :: STRING AS tx_receiver, + tx :signer_id :: STRING AS tx_signer, + tx, + tx :outcome :outcome :gas_burnt :: NUMBER AS transaction_gas_burnt, + tx :outcome :outcome :tokens_burnt :: NUMBER AS transaction_tokens_burnt, + _partition_by_block_number, + _inserted_timestamp + FROM + base_transactions +), +gas_burnt AS ( + SELECT + tx_hash, + SUM(gas_burnt) AS receipt_gas_burnt, + SUM(execution_outcome :outcome :tokens_burnt :: NUMBER) AS receipt_tokens_burnt + FROM + int_receipts + WHERE + execution_outcome :outcome: tokens_burnt :: NUMBER != 0 + GROUP BY + 1 +), +determine_tx_status AS ( + SELECT + DISTINCT tx_hash, + LAST_VALUE( + receipt_succeeded + ) over ( + PARTITION BY tx_hash + ORDER BY + block_id ASC + ) AS tx_succeeded + FROM + int_receipts +), +FINAL AS ( + SELECT + t.block_id, + t.block_hash, + t.tx_hash, + t.block_timestamp, + t.nonce, + t.signature, + t.tx_receiver, + t.tx_signer, + t.tx, + t.transaction_gas_burnt + g.receipt_gas_burnt AS gas_used, + t.transaction_tokens_burnt + g.receipt_tokens_burnt AS transaction_fee, + COALESCE( + actions.attached_gas, + gas_used + ) AS attached_gas, + s.tx_succeeded, + IFF ( + tx_succeeded, + 'Success', + 'Fail' + ) AS tx_status, -- DEPRECATE TX_STATUS IN GOLD + t._partition_by_block_number, + t._inserted_timestamp + FROM + transactions AS t + INNER JOIN determine_tx_status s + ON t.tx_hash = s.tx_hash + INNER JOIN actions + ON t.tx_hash = actions.tx_hash + INNER JOIN gas_burnt g + ON t.tx_hash = g.tx_hash +) +SELECT + tx_hash, + block_id, + block_hash, + block_timestamp, + nonce, + signature, + tx_receiver, + tx_signer, + tx, + gas_used, + transaction_fee, + attached_gas, + tx_succeeded, + tx_status, + _partition_by_block_number, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS streamline_transactions_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + FINAL + diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index a606990..6435c6f 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -3,7 +3,7 @@ {{ config ( materialized = "incremental", incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], + incremental_predicates = ["dynamic_range_predicate","origin_block_timestamp::date"], unique_key = "tx_hash", cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", @@ -13,8 +13,8 @@ WITH bronze_transactions AS ( SELECT - VALUE :BLOCK_ID :: INT AS block_id, - VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, + VALUE :BLOCK_ID :: INT AS origin_block_id, + VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS origin_block_timestamp_epoch, DATA :transaction :hash :: STRING AS tx_hash, DATA :transaction :signer_id :: STRING AS signer_id, partition_key, @@ -38,9 +38,9 @@ WHERE {% endif %} ) SELECT - block_id, - block_timestamp_epoch, - TO_TIMESTAMP_NTZ(block_timestamp_epoch, 9) AS block_timestamp, + origin_block_id, + origin_block_timestamp_epoch, + TO_TIMESTAMP_NTZ(origin_block_timestamp_epoch, 9) AS origin_block_timestamp, tx_hash, signer_id, partition_key, diff --git a/models/streamline/core/complete/streamline__transactions_complete.sql b/models/streamline/core/complete/streamline__transactions_complete.sql index 6c79df2..1390224 100644 --- a/models/streamline/core/complete/streamline__transactions_complete.sql +++ b/models/streamline/core/complete/streamline__transactions_complete.sql @@ -13,6 +13,8 @@ SELECT VALUE :BLOCK_ID :: INT AS block_id, VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, VALUE :TX_HASH :: STRING AS tx_hash, + VALUE :transaction :shard_id :: STRING AS shard_id, + VALUE :transaction :chunk_hash :: STRING AS chunk_hash, DATA :transaction :signer_id :: STRING AS signer_id, partition_key, _inserted_timestamp, diff --git a/models/streamline/core/realtime/streamline__transactions_realtime.sql b/models/streamline/core/realtime/streamline__transactions_realtime.sql index 9e5c694..5e8582a 100644 --- a/models/streamline/core/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/core/realtime/streamline__transactions_realtime.sql @@ -40,7 +40,9 @@ tbl AS ( block_id, block_timestamp_epoch, tx_hash, - signer_id + signer_id, + shard_id, + chunk_hash FROM {{ ref('streamline__transactions') }} WHERE @@ -59,7 +61,9 @@ tbl AS ( block_id, block_timestamp_epoch, tx_hash, - signer_id + signer_id, + shard_id, + chunk_hash FROM {{ ref('streamline__transactions_complete') }} WHERE From b698c5865d06c8a464427a0222e381f02e487ed8 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 19 Feb 2025 15:43:11 -0700 Subject: [PATCH 02/53] migrate txs logic --- .../lake/silver__streamline_transactions.sql | 2 +- models/silver/core/migration/_migrate_txs.sql | 56 +++++ models/silver/core/silver__receipts_final.sql | 2 +- .../core/silver__transactions_final.sql | 238 +++++++++--------- .../silver/core/silver__transactions_v2.sql | 4 + 5 files changed, 185 insertions(+), 117 deletions(-) create mode 100644 models/silver/core/migration/_migrate_txs.sql diff --git a/models/silver/core/lake/silver__streamline_transactions.sql b/models/silver/core/lake/silver__streamline_transactions.sql index a4502d7..51cac91 100644 --- a/models/silver/core/lake/silver__streamline_transactions.sql +++ b/models/silver/core/lake/silver__streamline_transactions.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], unique_key = 'tx_hash', cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards','scheduled_core'] + tags = ['load', 'load_shards', 'scheduled_core', 'deprecated_lake_archive'] ) }} WITH chunks AS ( diff --git a/models/silver/core/migration/_migrate_txs.sql b/models/silver/core/migration/_migrate_txs.sql new file mode 100644 index 0000000..f203ca5 --- /dev/null +++ b/models/silver/core/migration/_migrate_txs.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'ephemeral' +) }} +-- likely need to add batch logic for the migration +WITH lake_transactions_final AS ( + + SELECT + block_id, + block_timestamp, + tx_hash, + block_hash, + tx_succeeded, + gas_used, + transaction_fee, + attached_gas, + _partition_by_block_number + FROM + {{ ref('silver__streamline_transactions_final') }} +), +lake_transactions_int AS ( + SELECT + tx_hash, + block_id, + shard_number, + chunk_hash, + tx :transaction :: variant AS transaction_json, + tx :outcome :execution_outcome :: variant AS outcome_json, + _partition_by_block_number + FROM + {{ ref('silver__streamline_transactions') }} +), +transaction_archive AS ( + SELECT + i.chunk_hash, + i.shard_number AS shard_id, + f.block_hash, + f.block_id, + f.block_timestamp, + f.tx_hash, + i.transaction_json, + i.outcome_json, + f.tx_succeeded, + f.gas_used, + f.transaction_fee, + f.attached_gas, + f._partition_by_block_number + FROM + lake_transactions_final f + LEFT JOIN lake_transactions_int i + ON f.tx_hash = i.tx_hash + AND f._partition_by_block_number = i._partition_by_block_number +) +SELECT + * +FROM + transaction_archive diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index e8c601a..834ff2e 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -11,7 +11,7 @@ ) }} -- TODO if execute block for incremental min blockdate WITH -{% if var('NEAR_MIGRATE_RECEIPTS', False) %} +{% if var('NEAR_MIGRATE_ARCHIVE', False) %} lake_receipts_final as ( select * from {{ ref('silver__streamline_receipts_final') }} -- TODO incrementally load? diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index a7eeb65..e04da27 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -9,147 +9,155 @@ tags = ['scheduled_core'] ) }} -WITH -txs_with_receipts as ( -select * from {{ ref('silver__streamline_transactions_final') }} --- TODO incrementally load +{% if var('NEAR_MIGRATE_ARCHIVE', False) %} + + SELECT + chunk_hash, + shard_id, + block_id, + block_timestamp, + tx_hash, + transaction_json :actions :: ARRAY AS actions, + transaction_json :nonce :: INT AS nonce, + transaction_json :priority_fee :: INT AS priority_fee, + transaction_json :public_key :: STRING AS public_key, + transaction_json :receiver_id :: STRING AS receiver_id, + transaction_json :signature :: STRING AS signature, + transaction_json :signer_id :: STRING AS signer_id, + outcome_json, + OBJECT_CONSTRUCT() AS status_json, + tx_succeeded, + gas_used, + transaction_fee, + attached_gas, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + {{ ref('_migrate_txs') }} + -- TODO batch logic if needed + + {% else %} + +WITH txs_with_receipts AS ( + SELECT + chunk_hash, + shard_id, + origin_block_id AS block_id, + origin_block_timestamp AS block_timestamp, + tx_hash, + response_json :transaction :: variant AS transaction_json, + response_json :transaction_outcome :outcome :: variant AS outcome_json, + response_json :status :: variant AS status_json, + response_json :receipts_outcome :: ARRAY AS receipts_outcome_json, + response_json :status :Failure IS NULL AS tx_succeeded, + partition_key AS _partition_by_block_number + FROM + {{ ref('silver__transactions_v2') }} + + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + + {% if is_incremental() %} + WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + + {% endif %} ), -transaction_full as ( - select - -- shard_id, - -- chunk_hash, - origin_block_id AS block_id, - origin_block_timestamp AS block_timestamp, - tx_hash, - response_json :transaction:actions :: ARRAY as actions, - response_json :transaction:nonce :: INT as nonce, - response_json:transaction:priority_fee :: INT as priority_fee, - response_json:transaction:public_key :: STRING as public_key, - response_json:transaction:receiver_id :: STRING as receiver_id, - response_json:transaction:signature :: STRING as signature, - response_json:transaction:signer_id :: STRING as signer_id, - response_json:transaction_outcome:block_hash :: STRING as block_hash, - response_json:transaction_outcome:outcome :: variant as outcome_json, - response_json:status::VARIANT as status_json - from txs_with_receipts -) --- do we still want to calculate total used gas ? --- well, no need to pull in any other table anymore as the outcome exists in the underlying record --- can just flatten array and sum values -actions AS ( +determine_receipt_gas_burnt AS ( SELECT tx_hash, SUM( - VALUE :FunctionCall :gas :: NUMBER - ) AS attached_gas + VALUE :outcome :gas_burnt :: INT + ) AS total_gas_burnt_receipts, + SUM( + VALUE :outcome :tokens_burnt :: INT + ) AS total_tokens_burnt_receipts FROM - base_transactions, - LATERAL FLATTEN( - input => tx :actions + txs_with_receipts, + LATERAL FLATTEN ( + input => receipts_outcome_json ) GROUP BY 1 ), -transactions AS ( - SELECT - block_id, - tx :outcome :block_hash :: STRING AS block_hash, - tx_hash, - block_timestamp, - tx :nonce :: NUMBER AS nonce, - tx :signature :: STRING AS signature, - tx :receiver_id :: STRING AS tx_receiver, - tx :signer_id :: STRING AS tx_signer, - tx, - tx :outcome :outcome :gas_burnt :: NUMBER AS transaction_gas_burnt, - tx :outcome :outcome :tokens_burnt :: NUMBER AS transaction_tokens_burnt, - _partition_by_block_number, - _inserted_timestamp - FROM - base_transactions -), -gas_burnt AS ( +determine_attached_gas AS ( SELECT tx_hash, - SUM(gas_burnt) AS receipt_gas_burnt, - SUM(execution_outcome :outcome :tokens_burnt :: NUMBER) AS receipt_tokens_burnt + SUM( + VALUE :FunctionCall :gas :: INT + ) AS total_attached_gas FROM - int_receipts - WHERE - execution_outcome :outcome: tokens_burnt :: NUMBER != 0 - GROUP BY + txs_with_receipts, + LATERAL FLATTEN ( + input => transaction_json :actions :: ARRAY + ) + GROUP BY 1 ), -determine_tx_status AS ( +transactions_final AS ( SELECT - DISTINCT tx_hash, - LAST_VALUE( - receipt_succeeded - ) over ( - PARTITION BY tx_hash - ORDER BY - block_id ASC - ) AS tx_succeeded - FROM - int_receipts -), -FINAL AS ( - SELECT - t.block_id, - t.block_hash, + chunk_hash, + shard_id, + block_hash, + block_id, + block_timestamp, t.tx_hash, - t.block_timestamp, - t.nonce, - t.signature, - t.tx_receiver, - t.tx_signer, - t.tx, - t.transaction_gas_burnt + g.receipt_gas_burnt AS gas_used, - t.transaction_tokens_burnt + g.receipt_tokens_burnt AS transaction_fee, - COALESCE( - actions.attached_gas, - gas_used - ) AS attached_gas, - s.tx_succeeded, - IFF ( - tx_succeeded, - 'Success', - 'Fail' - ) AS tx_status, -- DEPRECATE TX_STATUS IN GOLD - t._partition_by_block_number, - t._inserted_timestamp + transaction_json, + outcome_json, + status_json, + total_gas_burnt_receipts, + total_tokens_burnt_receipts, + total_attached_gas, + tx_succeeded, + _partition_by_block_number FROM - transactions AS t - INNER JOIN determine_tx_status s - ON t.tx_hash = s.tx_hash - INNER JOIN actions - ON t.tx_hash = actions.tx_hash - INNER JOIN gas_burnt g - ON t.tx_hash = g.tx_hash + txs_with_receipts t + LEFT JOIN determine_receipt_gas_burnt d USING (tx_hash) + LEFT JOIN determine_attached_gas A USING (tx_hash) ) SELECT - tx_hash, - block_id, + chunk_hash, + shard_id, block_hash, + block_id, block_timestamp, - nonce, - signature, - tx_receiver, - tx_signer, - tx, - gas_used, - transaction_fee, - attached_gas, + tx_hash, + transaction_json :actions :: ARRAY AS actions, + transaction_json :nonce :: INT AS nonce, + transaction_json :priority_fee :: INT AS priority_fee, + transaction_json :public_key :: STRING AS public_key, + transaction_json :receiver_id :: STRING AS receiver_id, + transaction_json :signature :: STRING AS signature, + transaction_json :signer_id :: STRING AS signer_id, + outcome_json, + status_json, tx_succeeded, - tx_status, + outcome_json :outcome :gas_burnt :: INT + total_gas_burnt_receipts AS gas_used, + outcome_json :outcome :tokens_burnt :: INT + total_tokens_burnt_receipts AS transaction_fee, + COALESCE( + total_attached_gas, + gas_used + ) AS attached_gas, _partition_by_block_number, - _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['tx_hash'] - ) }} AS streamline_transactions_final_id, + ) }} AS transactions_final_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL - + transactions_final +{% endif %} diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index 6435c6f..d1835c6 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -15,6 +15,8 @@ WITH bronze_transactions AS ( SELECT VALUE :BLOCK_ID :: INT AS origin_block_id, VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS origin_block_timestamp_epoch, + VALUE :SHARD_ID :: INT AS shard_id, + VALUE :CHUNK_HASH :: STRING AS chunk_hash, DATA :transaction :hash :: STRING AS tx_hash, DATA :transaction :signer_id :: STRING AS signer_id, partition_key, @@ -41,6 +43,8 @@ SELECT origin_block_id, origin_block_timestamp_epoch, TO_TIMESTAMP_NTZ(origin_block_timestamp_epoch, 9) AS origin_block_timestamp, + shard_id, + chunk_hash, tx_hash, signer_id, partition_key, From 3e59c5031a29224d7d2b1ad3d0e1ce955de86000 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:10:42 -0700 Subject: [PATCH 03/53] receipts migration + final --- .../core/migration/_migrate_receipts.sql | 17 ++ models/silver/core/silver__receipts_final.sql | 178 +++++++++--------- .../core/silver__transactions_final.sql | 19 +- 3 files changed, 108 insertions(+), 106 deletions(-) create mode 100644 models/silver/core/migration/_migrate_receipts.sql diff --git a/models/silver/core/migration/_migrate_receipts.sql b/models/silver/core/migration/_migrate_receipts.sql new file mode 100644 index 0000000..5a88f64 --- /dev/null +++ b/models/silver/core/migration/_migrate_receipts.sql @@ -0,0 +1,17 @@ +{{ + config( + 'materialized' = 'ephemeral' + ) +}} + +SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + COALESCE(receipt_id, receipt_object_id) AS receipt_id, + receipt_actions AS receipt_json, + execution_outcome AS outcome_json, + _partition_by_block_number +FROM + {{ ref('silver__streamline_receipts_final') }} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 834ff2e..7aae758 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -9,113 +9,105 @@ tags = ['scheduled_core'], full_refresh = False ) }} --- TODO if execute block for incremental min blockdate -WITH + {% if var('NEAR_MIGRATE_ARCHIVE', False) %} -lake_receipts_final as ( - select * from {{ ref('silver__streamline_receipts_final') }} - -- TODO incrementally load? - -- TODO rename cols? -), + + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + receipt_id, + receipt_json, + outcome_json, + _partition_by_block_number + FROM + {{ ref('_migrate_receipts') }} + {% if var("BATCH_MIGRATE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} + {% else %} -txs_with_receipts as ( -select * from {{ ref('silver__transactions_v2') }} --- TODO incremental logic -), -blocks as ( - select * from {{ ref('silver__blocks_v2') }} - -- TODO incremental logic -), -flatten_receipts as ( - select +-- TODO if execute block for incremental min blockdate +WITH txs_with_receipts AS ( + SELECT + chunk_hash, + origin_block_id, + origin_block_timestamp, tx_hash, - -- chunk_hash, - -- shard_id, - value:predecessor_id::STRING as predecessor_id, - value:priority :: int as priority, - value:receipt :: variant as receipt_json, - value:receipt_id :: STRING as receipt_id, - value:receiver_id :: string as receiver_id - from tx_response, lateral flatten(input => response_json :receipts :: ARRAY) + response_json :receipts :: ARRAY AS receipts_json, + response_json :receipts_outcome :: ARRAY AS receipts_outcome_json, + response_json :status :Failure IS NULL AS tx_succeeded, + partition_key AS _partition_by_block_number + FROM + {{ ref('silver__transactions_v2') }} + -- TODO incremental logic ), -flatten_receipt_outcomes as ( - select - value :block_hash :: STRING as block_hash, - value:id :: STRING as receipt_id, - value:outcome :: variant as outcome_json, - value:proof::array as proof - from tx_response, lateral flatten(input => response_json :receipts_outcome :: ARRAY) +blocks AS ( + SELECT + block_id, + block_hash, + block_timestamp + FROM + {{ ref('silver__blocks_v2') }} + -- TODO incremental logic ), -receipts_full as ( -select - -- chunk_hash, - -- shard_id, - block_hash, - tx_hash, - r.receipt_id, - predecessor_id, - receiver_id, - priority, - receipt_json, - outcome_json, - proof, - outcome_json :status :Failure IS NULL AS receipt_succeeded, - TRY_PARSE_JSON( - outcome_json :status :Failure - ) AS failure_message, - object_keys( - failure_message - ) [0] :: STRING AS error_type_0, - COALESCE( - object_keys( - TRY_PARSE_JSON( - failure_message [error_type_0] :kind - ) - ) [0] :: STRING, - failure_message [error_type_0] :kind :: STRING - ) AS error_type_1, - COALESCE( - object_keys( - TRY_PARSE_JSON( - failure_message [error_type_0] :kind [error_type_1] - ) - ) [0] :: STRING, - failure_message [error_type_0] :kind [error_type_1] :: STRING - ) AS error_type_2, - failure_message [error_type_0] :kind [error_type_1] [error_type_2] :: STRING AS error_message -from flatten_receipts r left join flatten_receipt_outcomes ro -on r.receipt_id = ro.receipt_id +flatten_receipts AS ( + SELECT + chunk_hash, + tx_hash, + tx_succeeded, + VALUE :: variant AS receipt_json + FROM + txs_with_receipts, + LATERAL FLATTEN( + input => receipts_json + ) +), +flatten_receipt_outcomes AS ( + SELECT + VALUE :block_hash :: STRING AS block_hash, + tx_hash, + VALUE :id :: STRING AS receipt_id, + VALUE :: variant AS outcome_json + FROM + txs_with_receipts, + LATERAL FLATTEN( + input => receipts_outcome_json + ) +), +receipts_full AS ( + SELECT + chunk_hash, + ro.block_hash, + block_id, + block_timestamp, + r.tx_hash, + r.receipt_id, + receipt_json, + outcome_json, + tx_succeeded + FROM + flatten_receipts r + LEFT JOIN flatten_receipt_outcomes ro + ON r.receipt_id = ro.receipt_id + LEFT JOIN blocks b + ON ro.block_hash = b.block_hash ), FINAL AS ( SELECT - -- chunk_hash, - -- shard_id, + chunk_hash, block_hash, + block_id, + block_timestamp, tx_hash, receipt_id, - -- block_id, - -- block_timestamp, - predecessor_id, - receiver_id, - -- signer_id, -- this is more like action_signer_id tbh.. should be renamed - receipt_json, outcome_json, - -- outcome_json :receipt_ids :: ARRAY AS receipt_outcome_id, - -- receipt_type, -- this is not a thing - outcome_json :gas_burnt :: NUMBER AS gas_burnt, - outcome_json :status :: variant AS status_value, - outcome_json :logs :: ARRAY AS logs, - proof, - outcome_json :metadata :: variant AS metadata, - receipt_succeeded, - error_type_0, - error_type_1, - error_type_2, - error_message - -- _partition_by_block_number, - -- _inserted_timestamp + outcome_json :status :Failure IS NULL AS receipt_succeeded, + _partition_by_block_number FROM receipts_full ) diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index e04da27..a592d0a 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -13,7 +13,6 @@ SELECT chunk_hash, - shard_id, block_id, block_timestamp, tx_hash, @@ -39,14 +38,16 @@ '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_txs') }} - -- TODO batch logic if needed + {% if var("BATCH_MIGRATE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} {% else %} WITH txs_with_receipts AS ( SELECT chunk_hash, - shard_id, origin_block_id AS block_id, origin_block_timestamp AS block_timestamp, tx_hash, @@ -59,12 +60,7 @@ WITH txs_with_receipts AS ( FROM {{ ref('silver__transactions_v2') }} - {% if var("MANUAL_FIX") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% else %} - - {% if is_incremental() %} + {% if is_incremental() %} WHERE modified_timestamp >= ( SELECT @@ -72,9 +68,8 @@ WITH txs_with_receipts AS ( FROM {{ this }} ) - {% endif %} - {% endif %} + ), determine_receipt_gas_burnt AS ( SELECT @@ -110,7 +105,6 @@ determine_attached_gas AS ( transactions_final AS ( SELECT chunk_hash, - shard_id, block_hash, block_id, block_timestamp, @@ -130,7 +124,6 @@ transactions_final AS ( ) SELECT chunk_hash, - shard_id, block_hash, block_id, block_timestamp, From 925c66040c1772b28cd2e1c01c72a717b025df97 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:25:13 -0700 Subject: [PATCH 04/53] blocks migration + final --- .../core/lake/silver__streamline_blocks.sql | 2 +- .../core/lake/silver__streamline_receipts.sql | 2 +- .../silver__streamline_receipts_final.sql | 2 +- .../core/lake/silver__streamline_shards.sql | 2 +- .../silver__streamline_transactions_final.sql | 2 +- .../silver/core/migration/_migrate_blocks.sql | 24 +++++++++ .../core/migration/_migrate_receipts.sql | 24 ++++++--- models/silver/core/migration/_migrate_txs.sql | 18 ++++++- models/silver/core/silver__blocks_final.sql | 54 +++++++++++++++++++ models/silver/core/silver__receipts_final.sql | 16 +++--- .../core/silver__transactions_final.sql | 10 +--- 11 files changed, 127 insertions(+), 29 deletions(-) create mode 100644 models/silver/core/migration/_migrate_blocks.sql create mode 100644 models/silver/core/silver__blocks_final.sql diff --git a/models/silver/core/lake/silver__streamline_blocks.sql b/models/silver/core/lake/silver__streamline_blocks.sql index e76813b..123de22 100644 --- a/models/silver/core/lake/silver__streamline_blocks.sql +++ b/models/silver/core/lake/silver__streamline_blocks.sql @@ -6,7 +6,7 @@ merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE', '_partition_by_block_number'], unique_key = 'block_id', - tags = ['load', 'load_blocks','scheduled_core'], + tags = ['load', 'load_blocks','scheduled_core', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_receipts.sql b/models/silver/core/lake/silver__streamline_receipts.sql index 1da7fc2..18963f7 100644 --- a/models/silver/core/lake/silver__streamline_receipts.sql +++ b/models/silver/core/lake/silver__streamline_receipts.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], unique_key = 'receipt_id', cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards','scheduled_core'] + tags = ['load', 'load_shards','scheduled_core', 'deprecated_lake_archive'] ) }} WITH shards AS ( diff --git a/models/silver/core/lake/silver__streamline_receipts_final.sql b/models/silver/core/lake/silver__streamline_receipts_final.sql index 260dbf5..cd1ebea 100644 --- a/models/silver/core/lake/silver__streamline_receipts_final.sql +++ b/models/silver/core/lake/silver__streamline_receipts_final.sql @@ -6,7 +6,7 @@ unique_key = 'receipt_object_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number', ], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['receipt_map','scheduled_core'], + tags = ['receipt_map','scheduled_core', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_shards.sql b/models/silver/core/lake/silver__streamline_shards.sql index e7682bb..bb50f80 100644 --- a/models/silver/core/lake/silver__streamline_shards.sql +++ b/models/silver/core/lake/silver__streamline_shards.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['_inserted_timestamp::DATE', '_partition_by_block_number'], unique_key = 'shard_id', - tags = ['load', 'load_shards','scheduled_core'], + tags = ['load', 'load_shards','scheduled_core', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_transactions_final.sql b/models/silver/core/lake/silver__streamline_transactions_final.sql index b2d772d..7fad2bc 100644 --- a/models/silver/core/lake/silver__streamline_transactions_final.sql +++ b/models/silver/core/lake/silver__streamline_transactions_final.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], - tags = ['receipt_map','scheduled_core'] + tags = ['receipt_map','scheduled_core', 'deprecated_lake_archive'] ) }} WITH int_txs AS ( diff --git a/models/silver/core/migration/_migrate_blocks.sql b/models/silver/core/migration/_migrate_blocks.sql new file mode 100644 index 0000000..1bbe6f5 --- /dev/null +++ b/models/silver/core/migration/_migrate_blocks.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = 'ephemeral' +) }} + +SELECT + block_id, + block_timestamp, + block_hash, + prev_hash, + block_author, + chunks AS chunks_json, + header AS header_json, + _partition_by_block_number, + streamline_blocks_id, + inserted_timestamp, + modified_timestamp, + _invocation_id +FROM + {{ ref('silver__streamline_blocks') }} + + {% if var("BATCH_MIGRATE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} diff --git a/models/silver/core/migration/_migrate_receipts.sql b/models/silver/core/migration/_migrate_receipts.sql index 5a88f64..b32b5e1 100644 --- a/models/silver/core/migration/_migrate_receipts.sql +++ b/models/silver/core/migration/_migrate_receipts.sql @@ -1,17 +1,27 @@ -{{ - config( - 'materialized' = 'ephemeral' - ) -}} +{{ config( + 'materialized' = 'ephemeral' +) }} SELECT chunk_hash, block_id, block_timestamp, tx_hash, - COALESCE(receipt_id, receipt_object_id) AS receipt_id, + COALESCE( + receipt_id, + receipt_object_id + ) AS receipt_id, receipt_actions AS receipt_json, execution_outcome AS outcome_json, - _partition_by_block_number + _partition_by_block_number, + streamline_receipts_final_id, + inserted_timestamp, + modified_timestamp, + _invocation_id FROM {{ ref('silver__streamline_receipts_final') }} + + {% if var("BATCH_MIGRATE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} diff --git a/models/silver/core/migration/_migrate_txs.sql b/models/silver/core/migration/_migrate_txs.sql index f203ca5..42a0405 100644 --- a/models/silver/core/migration/_migrate_txs.sql +++ b/models/silver/core/migration/_migrate_txs.sql @@ -1,7 +1,7 @@ {{ config( materialized = 'ephemeral' ) }} --- likely need to add batch logic for the migration + WITH lake_transactions_final AS ( SELECT @@ -13,9 +13,18 @@ WITH lake_transactions_final AS ( gas_used, transaction_fee, attached_gas, - _partition_by_block_number + _partition_by_block_number, + streamline_transactions_final_id, + inserted_timestamp, + modified_timestamp, + _invocation_id FROM {{ ref('silver__streamline_transactions_final') }} + + {% if var("BATCH_MIGRATE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} ), lake_transactions_int AS ( SELECT @@ -28,6 +37,11 @@ lake_transactions_int AS ( _partition_by_block_number FROM {{ ref('silver__streamline_transactions') }} + + {% if var("BATCH_MIGRATE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} ), transaction_archive AS ( SELECT diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql new file mode 100644 index 0000000..f5f47b0 --- /dev/null +++ b/models/silver/core/silver__blocks_final.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'block_id', + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id);", + tags = ['scheduled_core'] +) }} + +{% if var('NEAR_MIGRATE_ARCHIVE', False) %} + + SELECT + block_id, + block_timestamp, + block_hash, + prev_hash, + block_author, + chunks_json, + header_json, + _partition_by_block_number, + streamline_blocks_id AS blocks_final_id, + inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + {{ ref('silver__streamline_blocks') }} + +{% else %} + +WITH blocks AS ( + SELECT + block_id, + block_timestamp, + block_hash, + block_json :prev_hash :: STRING AS prev_hash, + block_json :author :: STRING AS block_author, + block_json :chunks :: ARRAY AS chunks_json, + block_json :header :: OBJECT AS header_json, + partition_key AS _partition_by_block_number + FROM + {{ ref('silver__blocks_v2') }} +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['block_id'] + ) }} AS blocks_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + blocks diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 7aae758..6837c20 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -20,13 +20,13 @@ receipt_id, receipt_json, outcome_json, - _partition_by_block_number + _partition_by_block_number, + streamline_receipts_final_id AS receipts_final_id, + inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_receipts') }} - {% if var("BATCH_MIGRATE") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% endif %} {% else %} @@ -59,7 +59,8 @@ flatten_receipts AS ( chunk_hash, tx_hash, tx_succeeded, - VALUE :: variant AS receipt_json + VALUE :: variant AS receipt_json, + _partition_by_block_number FROM txs_with_receipts, LATERAL FLATTEN( @@ -88,7 +89,8 @@ receipts_full AS ( r.receipt_id, receipt_json, outcome_json, - tx_succeeded + tx_succeeded, + _partition_by_block_number FROM flatten_receipts r LEFT JOIN flatten_receipt_outcomes ro diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index a592d0a..f6e5c2e 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -30,18 +30,12 @@ transaction_fee, attached_gas, _partition_by_block_number, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash'] - ) }} AS transactions_final_id, - SYSDATE() AS inserted_timestamp, + streamline_transactions_final_id AS transactions_final_id, + inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_txs') }} - {% if var("BATCH_MIGRATE") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% endif %} {% else %} From 41a18602b1844500c501b11823d755e450e1ba7a Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:29:23 -0700 Subject: [PATCH 05/53] dynamic incr pred --- models/silver/core/silver__blocks_final.sql | 6 +++--- models/silver/core/silver__chunks_v2.sql | 2 ++ models/silver/core/silver__receipts_final.sql | 4 ++-- models/silver/core/silver__transactions_final.sql | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index f5f47b0..1249959 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -1,11 +1,11 @@ {{ config( materialized = 'incremental', - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], unique_key = 'block_id', - cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id);", + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id, block_hash);", tags = ['scheduled_core'] ) }} diff --git a/models/silver/core/silver__chunks_v2.sql b/models/silver/core/silver__chunks_v2.sql index c8832a1..02428fc 100644 --- a/models/silver/core/silver__chunks_v2.sql +++ b/models/silver/core/silver__chunks_v2.sql @@ -13,6 +13,8 @@ -- chunks only needed intrapipeline to get tx hashes -- unless deployed to a fact_chunks table with just the header ? ... -- chunk author might be helpful to analysts looking at validator performance +-- the rest of the chunk header is contained in the block json +-- truly just chunk author ... WITH bronze_chunks AS ( diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 6837c20..5b6cdb0 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -1,10 +1,10 @@ {{ config( materialized = 'incremental', - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], unique_key = 'receipt_id', - cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number', ], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", tags = ['scheduled_core'], full_refresh = False diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index f6e5c2e..3677fef 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -1,10 +1,10 @@ {{ config( materialized = 'incremental', - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], unique_key = 'tx_hash', - cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], + cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,signer_id,receiver_id);", tags = ['scheduled_core'] ) }} From ddef538ce6bbdbe4bd5a48c23e2c0cfd849d335b Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:31:13 -0700 Subject: [PATCH 06/53] add chunk / shard to txs rt --- .../core/realtime/streamline__transactions_realtime.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/streamline/core/realtime/streamline__transactions_realtime.sql b/models/streamline/core/realtime/streamline__transactions_realtime.sql index 5e8582a..7340667 100644 --- a/models/streamline/core/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/core/realtime/streamline__transactions_realtime.sql @@ -82,10 +82,12 @@ tbl AS ( AND signer_id IS NOT NULL ) SELECT + shard_id, + chunk_hash, block_id, block_timestamp_epoch, - FLOOR(block_id, -3) AS partition_key, tx_hash, + FLOOR(block_id, -3) AS partition_key, DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS request_timestamp, {{ target.database }}.live.udf_api( 'POST', From ae2c26791897f75d8936051c71457eadec0cca39 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:51:46 -0700 Subject: [PATCH 07/53] fixes, typos, etc --- models/gold/core/core__ez_token_transfers.sql | 2 +- models/silver/core/migration/_migrate_receipts.sql | 2 +- models/silver/core/silver__blocks_final.sql | 5 ++++- models/silver/core/silver__receipts_final.sql | 4 +++- models/silver/core/silver__transactions_final.sql | 4 +++- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index d0f1264..77f0498 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -39,9 +39,9 @@ {% if not min_block_timestamp_day or min_block_timestamp_day == 'None' %} {% set min_block_timestamp_day = '2020-07-01' %} {% endif %} +{{ log("min_block_timestamp_day: " ~ min_block_timestamp_day, info=True) }} {% endif %} -{{ log("min_block_timestamp_day: " ~ min_block_timestamp_day, info=True) }} WITH hourly_prices AS ( SELECT diff --git a/models/silver/core/migration/_migrate_receipts.sql b/models/silver/core/migration/_migrate_receipts.sql index b32b5e1..cce2b4a 100644 --- a/models/silver/core/migration/_migrate_receipts.sql +++ b/models/silver/core/migration/_migrate_receipts.sql @@ -1,5 +1,5 @@ {{ config( - 'materialized' = 'ephemeral' + materialized = 'ephemeral' ) }} SELECT diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 1249959..331c3df 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -6,7 +6,8 @@ unique_key = 'block_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id, block_hash);", - tags = ['scheduled_core'] + tags = ['scheduled_core', 'core_v2'], + full_refresh = False ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} @@ -52,3 +53,5 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM blocks + +{% endif %} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 5b6cdb0..fcc5c9e 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -6,7 +6,7 @@ unique_key = 'receipt_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['scheduled_core'], + tags = ['scheduled_core', 'core_v2'], full_refresh = False ) }} @@ -123,3 +123,5 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM FINAL + +{% endif %} diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 3677fef..530fb43 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -6,7 +6,8 @@ unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,signer_id,receiver_id);", - tags = ['scheduled_core'] + tags = ['scheduled_core', 'core_v2'], + full_refresh = False ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} @@ -147,4 +148,5 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM transactions_final + {% endif %} From 7447745e50be3d2bf1c2f699707e14460ccf47e3 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 19 Feb 2025 17:14:14 -0700 Subject: [PATCH 08/53] coalesces - streamline incremental, migration insts --- models/silver/core/silver__blocks_final.sql | 18 ++++++++++++++++-- models/silver/core/silver__receipts_final.sql | 8 ++++++-- .../silver/core/silver__transactions_final.sql | 8 ++++++-- models/streamline/core/streamline__chunks.sql | 2 +- .../core/streamline__transactions.sql | 2 +- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 331c3df..1f5b2d2 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -22,8 +22,12 @@ header_json, _partition_by_block_number, streamline_blocks_id AS blocks_final_id, - inserted_timestamp, - SYSDATE() AS modified_timestamp, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + SYSDATE() + ) AS inserted_timestamp, + SYSDATE() AS modified_timestamp, -- reset or preserve ? '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__streamline_blocks') }} @@ -42,6 +46,16 @@ WITH blocks AS ( partition_key AS _partition_by_block_number FROM {{ ref('silver__blocks_v2') }} + + {% if is_incremental() %} + WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} ) SELECT *, diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index fcc5c9e..831164c 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -22,8 +22,12 @@ outcome_json, _partition_by_block_number, streamline_receipts_final_id AS receipts_final_id, - inserted_timestamp, - SYSDATE() AS modified_timestamp, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + SYSDATE() + ) AS inserted_timestamp, + SYSDATE() AS modified_timestamp, -- reset or preserve ? '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_receipts') }} diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 530fb43..3b89eed 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -32,8 +32,12 @@ attached_gas, _partition_by_block_number, streamline_transactions_final_id AS transactions_final_id, - inserted_timestamp, - SYSDATE() AS modified_timestamp, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + SYSDATE() + ) AS inserted_timestamp, + SYSDATE() AS modified_timestamp, -- reset or preserve ? '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_txs') }} diff --git a/models/streamline/core/streamline__chunks.sql b/models/streamline/core/streamline__chunks.sql index c45262f..3e2752a 100644 --- a/models/streamline/core/streamline__chunks.sql +++ b/models/streamline/core/streamline__chunks.sql @@ -23,7 +23,7 @@ WITH blocks_complete AS ( WHERE modified_timestamp >= ( SELECT - MAX(modified_timestamp) + COALESCE(MAX(modified_timestamp), '2025-01-01' :: TIMESTAMP_NTZ) FROM {{ this }} ) diff --git a/models/streamline/core/streamline__transactions.sql b/models/streamline/core/streamline__transactions.sql index 9cf82de..b0d3e78 100644 --- a/models/streamline/core/streamline__transactions.sql +++ b/models/streamline/core/streamline__transactions.sql @@ -25,7 +25,7 @@ WITH chunks_complete AS ( WHERE modified_timestamp >= ( SELECT - MAX(modified_timestamp) + COALESCE(MAX(modified_timestamp), '2025-01-01' :: TIMESTAMP_NTZ) FROM {{ this }} ) From 6797503f58d32db60505532bd06ba668d92c4f28 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 20 Feb 2025 09:06:15 -0700 Subject: [PATCH 09/53] upd migrate blocks ref --- models/silver/core/silver__blocks_final.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 1f5b2d2..be09db7 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -30,7 +30,7 @@ SYSDATE() AS modified_timestamp, -- reset or preserve ? '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('_migrate_blocks') }} {% else %} From 5f14b1440a649adf89084d37d807c967fa14ffcf Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 20 Feb 2025 13:58:04 -0700 Subject: [PATCH 10/53] cluster, upd streamline rt to join --- models/silver/core/silver__blocks_v2.sql | 2 +- models/silver/core/silver__chunks_v2.sql | 2 +- .../silver/core/silver__transactions_v2.sql | 2 +- .../realtime/streamline__chunks_realtime.sql | 33 +++----------- .../streamline__transactions_realtime.sql | 45 +++++-------------- 5 files changed, 21 insertions(+), 63 deletions(-) diff --git a/models/silver/core/silver__blocks_v2.sql b/models/silver/core/silver__blocks_v2.sql index 159cd50..505026c 100644 --- a/models/silver/core/silver__blocks_v2.sql +++ b/models/silver/core/silver__blocks_v2.sql @@ -5,7 +5,7 @@ incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "block_hash", - cluster_by = ['modified_timestamp::DATE','partition_key'], + cluster_by = ['modified_timestamp::DATE','block_timestamp::date'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_hash)", tags = ['scheduled_core', 'core_v2'] ) }} diff --git a/models/silver/core/silver__chunks_v2.sql b/models/silver/core/silver__chunks_v2.sql index 02428fc..7756b29 100644 --- a/models/silver/core/silver__chunks_v2.sql +++ b/models/silver/core/silver__chunks_v2.sql @@ -5,7 +5,7 @@ incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "chunk_hash", - cluster_by = ['modified_timestamp::DATE','partition_key'], + cluster_by = ['modified_timestamp::DATE','block_timestamp::date'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(chunk_hash)", tags = ['scheduled_core', 'core_v2'] ) }} diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index d1835c6..38f39c9 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -5,7 +5,7 @@ incremental_strategy = 'merge', incremental_predicates = ["dynamic_range_predicate","origin_block_timestamp::date"], unique_key = "tx_hash", - cluster_by = ['modified_timestamp::DATE','partition_key'], + cluster_by = ['modified_timestamp::DATE','origin_block_timestamp::date'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", tags = ['scheduled_core', 'core_v2'] ) }} diff --git a/models/streamline/core/realtime/streamline__chunks_realtime.sql b/models/streamline/core/realtime/streamline__chunks_realtime.sql index 56ec8ca..35579a0 100644 --- a/models/streamline/core/realtime/streamline__chunks_realtime.sql +++ b/models/streamline/core/realtime/streamline__chunks_realtime.sql @@ -37,41 +37,22 @@ last_3_days AS ( {% endif %} tbl AS ( SELECT - block_id, - block_timestamp_epoch, - chunk_hash + A.block_id, + A.block_timestamp_epoch, + A.chunk_hash FROM - {{ ref('streamline__chunks') }} + {{ ref('streamline__chunks') }} A + LEFT JOIN {{ ref('streamline__chunks_complete') }} B ON A.chunk_hash = B.chunk_hash WHERE ( - block_id >= ( + A.block_id >= ( SELECT block_id FROM last_3_days ) ) - AND chunk_hash IS NOT NULL - EXCEPT - SELECT - block_id, - block_timestamp_epoch, - chunk_hash - FROM - {{ ref('streamline__chunks_complete') }} - WHERE - block_id >= ( - SELECT - block_id - FROM - last_3_days - ) - AND _inserted_timestamp >= DATEADD( - 'day', - -4, - SYSDATE() - ) - AND chunk_hash IS NOT NULL + AND B.chunk_hash IS NULL ) SELECT block_id, diff --git a/models/streamline/core/realtime/streamline__transactions_realtime.sql b/models/streamline/core/realtime/streamline__transactions_realtime.sql index 7340667..4467c65 100644 --- a/models/streamline/core/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/core/realtime/streamline__transactions_realtime.sql @@ -37,49 +37,26 @@ last_3_days AS ( {% endif %} tbl AS ( SELECT - block_id, - block_timestamp_epoch, - tx_hash, - signer_id, - shard_id, - chunk_hash + A.block_id, + A.block_timestamp_epoch, + A.tx_hash, + A.signer_id, + A.shard_id, + A.chunk_hash FROM - {{ ref('streamline__transactions') }} + {{ ref('streamline__transactions') }} A + LEFT JOIN {{ ref('streamline__transactions_complete') }} B ON A.tx_hash = B.tx_hash WHERE ( - block_id >= ( + A.block_id >= ( SELECT block_id FROM last_3_days ) ) - AND tx_hash IS NOT NULL - AND signer_id IS NOT NULL - EXCEPT - SELECT - block_id, - block_timestamp_epoch, - tx_hash, - signer_id, - shard_id, - chunk_hash - FROM - {{ ref('streamline__transactions_complete') }} - WHERE - block_id >= ( - SELECT - block_id - FROM - last_3_days - ) - AND _inserted_timestamp >= DATEADD( - 'day', - -4, - SYSDATE() - ) - AND tx_hash IS NOT NULL - AND signer_id IS NOT NULL + AND A.signer_id IS NOT NULL + AND B.tx_hash IS NULL ) SELECT shard_id, From 0b00ccabaa3ec5f44aa2070570315d735ebcbce7 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 20 Feb 2025 13:59:23 -0700 Subject: [PATCH 11/53] silver_v2 - kill so, kill chunks_ --- models/silver/core/silver__blocks_v2.sql | 1 - models/silver/core/silver__chunks_v2.sql | 66 ------------------- .../silver/core/silver__transactions_v2.sql | 1 - 3 files changed, 68 deletions(-) delete mode 100644 models/silver/core/silver__chunks_v2.sql diff --git a/models/silver/core/silver__blocks_v2.sql b/models/silver/core/silver__blocks_v2.sql index 505026c..7da6dc4 100644 --- a/models/silver/core/silver__blocks_v2.sql +++ b/models/silver/core/silver__blocks_v2.sql @@ -6,7 +6,6 @@ incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], unique_key = "block_hash", cluster_by = ['modified_timestamp::DATE','block_timestamp::date'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_hash)", tags = ['scheduled_core', 'core_v2'] ) }} diff --git a/models/silver/core/silver__chunks_v2.sql b/models/silver/core/silver__chunks_v2.sql deleted file mode 100644 index 7756b29..0000000 --- a/models/silver/core/silver__chunks_v2.sql +++ /dev/null @@ -1,66 +0,0 @@ --- depends_on: {{ ref('bronze__chunks') }} --- depends_on: {{ ref('bronze__FR_chunks') }} -{{ config ( - materialized = "incremental", - incremental_strategy = 'merge', - incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], - unique_key = "chunk_hash", - cluster_by = ['modified_timestamp::DATE','block_timestamp::date'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(chunk_hash)", - tags = ['scheduled_core', 'core_v2'] -) }} --- TODO this can actually be dropped --- chunks only needed intrapipeline to get tx hashes --- unless deployed to a fact_chunks table with just the header ? ... --- chunk author might be helpful to analysts looking at validator performance --- the rest of the chunk header is contained in the block json --- truly just chunk author ... - -WITH bronze_chunks AS ( - - SELECT - VALUE :BLOCK_ID :: INT AS block_id, - VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, - DATA :header :shard_id :: INT AS shard_id, - DATA :header :chunk_hash :: STRING AS chunk_hash, - partition_key, - DATA :: variant AS chunk_json, - _inserted_timestamp - FROM - -{% if is_incremental() %} -{{ ref('bronze__chunks') }} -WHERE - _inserted_timestamp >= ( - SELECT - COALESCE(MAX(_inserted_timestamp), '1900-01-01' :: TIMESTAMP) AS _inserted_timestamp - FROM - {{ this }}) - AND typeof(DATA) != 'NULL_VALUE' - {% else %} - {{ ref('bronze__FR_chunks') }} - WHERE - typeof(DATA) != 'NULL_VALUE' - {% endif %} - ) -SELECT - block_id, - block_timestamp_epoch, - TO_TIMESTAMP_NTZ(block_timestamp_epoch, 9) AS block_timestamp, - shard_id, - chunk_hash, - partition_key, - chunk_json, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key(['chunk_hash']) }} AS chunks_v2_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - bronze_chunks - -qualify ROW_NUMBER() over ( - PARTITION BY chunk_hash - ORDER BY - _inserted_timestamp DESC - ) = 1 diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index 38f39c9..6584360 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -6,7 +6,6 @@ incremental_predicates = ["dynamic_range_predicate","origin_block_timestamp::date"], unique_key = "tx_hash", cluster_by = ['modified_timestamp::DATE','origin_block_timestamp::date'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", tags = ['scheduled_core', 'core_v2'] ) }} From e00d737cdf90e7a45e4e39933458d8d9f52ca2b8 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 20 Feb 2025 14:09:51 -0700 Subject: [PATCH 12/53] upd gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f0981c8..3bd91fd 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ dbt_docs.sh local* .user.yml .cursorignore +.cursor/* From 51deab20a1ddedc350b7e4718ca144b32da4b761 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 27 Feb 2025 12:04:02 -0700 Subject: [PATCH 13/53] upd gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3bd91fd..ad48074 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ dbt_docs.sh local* .user.yml .cursorignore +.cursorrules .cursor/* From 78165c4b48b5269d7aab812b249723a386ff1e6d Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 3 Mar 2025 10:13:44 -0700 Subject: [PATCH 14/53] upd archive select - dont reset modts --- models/silver/core/silver__blocks_final.sql | 2 +- models/silver/core/silver__receipts_final.sql | 2 +- models/silver/core/silver__transactions_final.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index be09db7..e5c8f1b 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -27,7 +27,7 @@ _inserted_timestamp, SYSDATE() ) AS inserted_timestamp, - SYSDATE() AS modified_timestamp, -- reset or preserve ? + modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_blocks') }} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 831164c..7efb405 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -27,7 +27,7 @@ _inserted_timestamp, SYSDATE() ) AS inserted_timestamp, - SYSDATE() AS modified_timestamp, -- reset or preserve ? + modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_receipts') }} diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 3b89eed..ed52ba4 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -37,7 +37,7 @@ _inserted_timestamp, SYSDATE() ) AS inserted_timestamp, - SYSDATE() AS modified_timestamp, -- reset or preserve ? + modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM {{ ref('_migrate_txs') }} From e066993fdfc47f8e5ac01bca3c7d9d50ab6c2283 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 3 Mar 2025 11:41:11 -0700 Subject: [PATCH 15/53] align silver final cols --- .../core/migration/_migrate_receipts.sql | 18 +++++++++-- models/silver/core/migration/_migrate_txs.sql | 22 ++++++++++--- models/silver/core/silver__receipts_final.sql | 16 +++++----- .../core/silver__transactions_final.sql | 31 ++++++------------- 4 files changed, 52 insertions(+), 35 deletions(-) diff --git a/models/silver/core/migration/_migrate_receipts.sql b/models/silver/core/migration/_migrate_receipts.sql index cce2b4a..074caf6 100644 --- a/models/silver/core/migration/_migrate_receipts.sql +++ b/models/silver/core/migration/_migrate_receipts.sql @@ -11,12 +11,24 @@ SELECT receipt_id, receipt_object_id ) AS receipt_id, + COALESCE( + predecessor_id, + receipt_actions :predecessor_id :: STRING + ) AS predecessor_id, + receiver_id, receipt_actions AS receipt_json, execution_outcome AS outcome_json, + receipt_succeeded, _partition_by_block_number, - streamline_receipts_final_id, - inserted_timestamp, - modified_timestamp, + streamline_receipts_final_id AS receipts_final_id, + COALESCE( + inserted_timestamp, + _inserted_timestamp + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + _inserted_timestamp + ) AS modified_timestamp, _invocation_id FROM {{ ref('silver__streamline_receipts_final') }} diff --git a/models/silver/core/migration/_migrate_txs.sql b/models/silver/core/migration/_migrate_txs.sql index 42a0405..e4b6126 100644 --- a/models/silver/core/migration/_migrate_txs.sql +++ b/models/silver/core/migration/_migrate_txs.sql @@ -8,15 +8,23 @@ WITH lake_transactions_final AS ( block_id, block_timestamp, tx_hash, + tx_signer, + tx_receiver, block_hash, tx_succeeded, gas_used, transaction_fee, attached_gas, _partition_by_block_number, - streamline_transactions_final_id, - inserted_timestamp, - modified_timestamp, + streamline_transactions_final_id AS transactions_final_id, + COALESCE( + inserted_timestamp, + _inserted_timestamp + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + _inserted_timestamp + ) AS modified_timestamp, _invocation_id FROM {{ ref('silver__streamline_transactions_final') }} @@ -51,13 +59,19 @@ transaction_archive AS ( f.block_id, f.block_timestamp, f.tx_hash, + f.tx_signer, + f.tx_receiver, i.transaction_json, i.outcome_json, f.tx_succeeded, f.gas_used, f.transaction_fee, f.attached_gas, - f._partition_by_block_number + f._partition_by_block_number, + f.transactions_final_id, + f.inserted_timestamp, + f.modified_timestamp, + f._invocation_id FROM lake_transactions_final f LEFT JOIN lake_transactions_int i diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 7efb405..ac5afc6 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -18,15 +18,15 @@ block_timestamp, tx_hash, receipt_id, + predecessor_id, + receiver_id, receipt_json, outcome_json, + NULL AS tx_succeeded, + receipt_succeeded, _partition_by_block_number, - streamline_receipts_final_id AS receipts_final_id, - COALESCE( - inserted_timestamp, - _inserted_timestamp, - SYSDATE() - ) AS inserted_timestamp, + receipts_final_id, + inserted_timestamp, modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM @@ -105,13 +105,15 @@ receipts_full AS ( FINAL AS ( SELECT chunk_hash, - block_hash, block_id, block_timestamp, tx_hash, receipt_id, + predecessor_id, + receiver_id, receipt_json, outcome_json, + tx_succeeded, outcome_json :status :Failure IS NULL AS receipt_succeeded, _partition_by_block_number FROM diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index ed52ba4..829db1e 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,signer_id,receiver_id);", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,tx_signer,tx_receiver);", tags = ['scheduled_core', 'core_v2'], full_refresh = False ) }} @@ -14,16 +14,13 @@ SELECT chunk_hash, + block_hash, block_id, block_timestamp, tx_hash, - transaction_json :actions :: ARRAY AS actions, - transaction_json :nonce :: INT AS nonce, - transaction_json :priority_fee :: INT AS priority_fee, - transaction_json :public_key :: STRING AS public_key, - transaction_json :receiver_id :: STRING AS receiver_id, - transaction_json :signature :: STRING AS signature, - transaction_json :signer_id :: STRING AS signer_id, + tx_receiver, + tx_signer, + transaction_json, outcome_json, OBJECT_CONSTRUCT() AS status_json, tx_succeeded, @@ -31,12 +28,8 @@ transaction_fee, attached_gas, _partition_by_block_number, - streamline_transactions_final_id AS transactions_final_id, - COALESCE( - inserted_timestamp, - _inserted_timestamp, - SYSDATE() - ) AS inserted_timestamp, + transactions_final_id, + inserted_timestamp, modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM @@ -127,13 +120,9 @@ SELECT block_id, block_timestamp, tx_hash, - transaction_json :actions :: ARRAY AS actions, - transaction_json :nonce :: INT AS nonce, - transaction_json :priority_fee :: INT AS priority_fee, - transaction_json :public_key :: STRING AS public_key, - transaction_json :receiver_id :: STRING AS receiver_id, - transaction_json :signature :: STRING AS signature, - transaction_json :signer_id :: STRING AS signer_id, + transaction_json :receiver_id :: STRING AS tx_receiver, + transaction_json :signer_id :: STRING AS tx_signer, + transaction_json, outcome_json, status_json, tx_succeeded, From bb046412344092fc6ac393bec5b5d9d078a421ca Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 3 Mar 2025 12:03:36 -0700 Subject: [PATCH 16/53] typo, col fixes --- models/silver/core/migration/_migrate_txs.sql | 2 -- models/silver/core/silver__blocks_final.sql | 5 ++--- models/silver/core/silver__receipts_final.sql | 8 ++++---- models/silver/core/silver__transactions_final.sql | 14 +++++--------- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/models/silver/core/migration/_migrate_txs.sql b/models/silver/core/migration/_migrate_txs.sql index e4b6126..758c53b 100644 --- a/models/silver/core/migration/_migrate_txs.sql +++ b/models/silver/core/migration/_migrate_txs.sql @@ -10,7 +10,6 @@ WITH lake_transactions_final AS ( tx_hash, tx_signer, tx_receiver, - block_hash, tx_succeeded, gas_used, transaction_fee, @@ -55,7 +54,6 @@ transaction_archive AS ( SELECT i.chunk_hash, i.shard_number AS shard_id, - f.block_hash, f.block_id, f.block_timestamp, f.tx_hash, diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index e5c8f1b..711a99d 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -6,8 +6,7 @@ unique_key = 'block_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id, block_hash);", - tags = ['scheduled_core', 'core_v2'], - full_refresh = False + tags = ['scheduled_core', 'core_v2'] ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} @@ -39,7 +38,7 @@ WITH blocks AS ( block_id, block_timestamp, block_hash, - block_json :prev_hash :: STRING AS prev_hash, + block_json :header :prev_hash :: STRING AS prev_hash, block_json :author :: STRING AS block_author, block_json :chunks :: ARRAY AS chunks_json, block_json :header :: OBJECT AS header_json, diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index ac5afc6..b91172b 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -6,8 +6,7 @@ unique_key = 'receipt_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['scheduled_core', 'core_v2'], - full_refresh = False + tags = ['scheduled_core', 'core_v2'] ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} @@ -63,6 +62,7 @@ flatten_receipts AS ( chunk_hash, tx_hash, tx_succeeded, + VALUE :receipt_id :: STRING AS receipt_id, VALUE :: variant AS receipt_json, _partition_by_block_number FROM @@ -109,8 +109,8 @@ FINAL AS ( block_timestamp, tx_hash, receipt_id, - predecessor_id, - receiver_id, + receipt_json :predecessor_id :: STRING AS predecessor_id, + receipt_json :receiver_id :: STRING AS receiver_id, receipt_json, outcome_json, tx_succeeded, diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 829db1e..5ef34a4 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -6,15 +6,13 @@ unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,tx_signer,tx_receiver);", - tags = ['scheduled_core', 'core_v2'], - full_refresh = False + tags = ['scheduled_core', 'core_v2'] ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} SELECT chunk_hash, - block_hash, block_id, block_timestamp, tx_hash, @@ -67,10 +65,10 @@ determine_receipt_gas_burnt AS ( SELECT tx_hash, SUM( - VALUE :outcome :gas_burnt :: INT + ZEROIFNULL(VALUE :outcome :gas_burnt :: INT) ) AS total_gas_burnt_receipts, SUM( - VALUE :outcome :tokens_burnt :: INT + ZEROIFNULL(VALUE :outcome :tokens_burnt :: INT) ) AS total_tokens_burnt_receipts FROM txs_with_receipts, @@ -97,7 +95,6 @@ determine_attached_gas AS ( transactions_final AS ( SELECT chunk_hash, - block_hash, block_id, block_timestamp, t.tx_hash, @@ -116,7 +113,6 @@ transactions_final AS ( ) SELECT chunk_hash, - block_hash, block_id, block_timestamp, tx_hash, @@ -126,8 +122,8 @@ SELECT outcome_json, status_json, tx_succeeded, - outcome_json :outcome :gas_burnt :: INT + total_gas_burnt_receipts AS gas_used, - outcome_json :outcome :tokens_burnt :: INT + total_tokens_burnt_receipts AS transaction_fee, + ZEROIFNULL(outcome_json :gas_burnt :: INT) + total_gas_burnt_receipts AS gas_used, + ZEROIFNULL(outcome_json :tokens_burnt :: INT) + total_tokens_burnt_receipts AS transaction_fee, COALESCE( total_attached_gas, gas_used From 9c4d50aa753e09c6a67ed194b81d0d814b4baff3 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 3 Mar 2025 12:28:41 -0700 Subject: [PATCH 17/53] test migration, fixes --- .../silver/core/migration/_migrate_blocks.sql | 18 +++- .../core/migration/_migrate_receipts.sql | 92 +++++++++++++------ models/silver/core/migration/_migrate_txs.sql | 14 ++- models/silver/core/silver__blocks_final.sql | 13 ++- models/silver/core/silver__receipts_final.sql | 6 +- .../core/silver__transactions_final.sql | 4 + 6 files changed, 104 insertions(+), 43 deletions(-) diff --git a/models/silver/core/migration/_migrate_blocks.sql b/models/silver/core/migration/_migrate_blocks.sql index 1bbe6f5..6226e76 100644 --- a/models/silver/core/migration/_migrate_blocks.sql +++ b/models/silver/core/migration/_migrate_blocks.sql @@ -11,14 +11,24 @@ SELECT chunks AS chunks_json, header AS header_json, _partition_by_block_number, - streamline_blocks_id, - inserted_timestamp, - modified_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_id'] + ) }} AS blocks_final_id, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS modified_timestamp, _invocation_id FROM {{ ref('silver__streamline_blocks') }} - {% if var("BATCH_MIGRATE") %} + {% if var("NEAR_MIGRATE_ARCHIVE") %} WHERE {{ partition_load_manual('no_buffer') }} {% endif %} diff --git a/models/silver/core/migration/_migrate_receipts.sql b/models/silver/core/migration/_migrate_receipts.sql index 074caf6..9d0ed8e 100644 --- a/models/silver/core/migration/_migrate_receipts.sql +++ b/models/silver/core/migration/_migrate_receipts.sql @@ -2,38 +2,78 @@ materialized = 'ephemeral' ) }} +WITH lake_receipts_final AS ( + + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + COALESCE( + receipt_id, + receipt_object_id + ) AS receipt_id, + COALESCE( + predecessor_id, + receipt_actions :predecessor_id :: STRING + ) AS predecessor_id, + receiver_id, + receipt_actions AS receipt_json, + execution_outcome AS outcome_json, + receipt_succeeded, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['COALESCE(receipt_id, receipt_object_id)'] + ) }} AS receipts_final_id, + COALESCE( + inserted_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + _inserted_timestamp, + _load_timestamp + ) AS modified_timestamp, + _invocation_id + FROM + {{ ref('silver__streamline_receipts_final') }} + + {% if var("NEAR_MIGRATE_ARCHIVE") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% endif %} +), +lake_transactions_final AS ( + SELECT + tx_hash, + tx_succeeded + FROM + {{ ref('silver__streamline_transactions_final') }} + + {% if var("NEAR_MIGRATE_ARCHIVE") %} + WHERE + {{ partition_load_manual('front') }} + {% endif %} +) SELECT chunk_hash, block_id, block_timestamp, - tx_hash, - COALESCE( - receipt_id, - receipt_object_id - ) AS receipt_id, - COALESCE( - predecessor_id, - receipt_actions :predecessor_id :: STRING - ) AS predecessor_id, + r.tx_hash, + receipt_id, + predecessor_id, receiver_id, - receipt_actions AS receipt_json, - execution_outcome AS outcome_json, + receipt_json, + outcome_json, + tx_succeeded, receipt_succeeded, _partition_by_block_number, - streamline_receipts_final_id AS receipts_final_id, - COALESCE( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp, + receipts_final_id, + inserted_timestamp, + modified_timestamp, _invocation_id FROM - {{ ref('silver__streamline_receipts_final') }} - - {% if var("BATCH_MIGRATE") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% endif %} + lake_receipts_final r + LEFT JOIN lake_transactions_final tx + ON r.tx_hash = tx.tx_hash diff --git a/models/silver/core/migration/_migrate_txs.sql b/models/silver/core/migration/_migrate_txs.sql index 758c53b..c112321 100644 --- a/models/silver/core/migration/_migrate_txs.sql +++ b/models/silver/core/migration/_migrate_txs.sql @@ -15,20 +15,24 @@ WITH lake_transactions_final AS ( transaction_fee, attached_gas, _partition_by_block_number, - streamline_transactions_final_id AS transactions_final_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_final_id, COALESCE( inserted_timestamp, - _inserted_timestamp + _inserted_timestamp, + _load_timestamp ) AS inserted_timestamp, COALESCE( modified_timestamp, - _inserted_timestamp + _inserted_timestamp, + _load_timestamp ) AS modified_timestamp, _invocation_id FROM {{ ref('silver__streamline_transactions_final') }} - {% if var("BATCH_MIGRATE") %} + {% if var("NEAR_MIGRATE_ARCHIVE") %} WHERE {{ partition_load_manual('no_buffer') }} {% endif %} @@ -45,7 +49,7 @@ lake_transactions_int AS ( FROM {{ ref('silver__streamline_transactions') }} - {% if var("BATCH_MIGRATE") %} + {% if var("NEAR_MIGRATE_ARCHIVE") %} WHERE {{ partition_load_manual('no_buffer') }} {% endif %} diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 711a99d..96f717b 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -10,7 +10,10 @@ ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} - + {% if execute %} + {% do log('Migrating blocks ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} + {% do log('Invocation ID: ' ~ invocation_id, info=True) %} + {% endif %} SELECT block_id, block_timestamp, @@ -20,12 +23,8 @@ chunks_json, header_json, _partition_by_block_number, - streamline_blocks_id AS blocks_final_id, - COALESCE( - inserted_timestamp, - _inserted_timestamp, - SYSDATE() - ) AS inserted_timestamp, + blocks_final_id, + inserted_timestamp, modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index b91172b..89cf1d5 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -10,6 +10,10 @@ ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} + {% if execute %} + {% do log('Migrating receipts ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} + {% do log('Invocation ID: ' ~ invocation_id, info=True) %} + {% endif %} SELECT chunk_hash, @@ -21,7 +25,7 @@ receiver_id, receipt_json, outcome_json, - NULL AS tx_succeeded, + tx_succeeded, receipt_succeeded, _partition_by_block_number, receipts_final_id, diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 5ef34a4..129b4bc 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -10,6 +10,10 @@ ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} + {% if execute %} + {% do log('Migrating transactions ' ~ var('RANGE_START') ~ ' to ' ~ var('RANGE_END'), info=True) %} + {% do log('Invocation ID: ' ~ invocation_id, info=True) %} + {% endif %} SELECT chunk_hash, From f7560985e74cdd6ec893fa828bef0225952e1461 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 3 Mar 2025 13:45:25 -0700 Subject: [PATCH 18/53] receipts incr logic --- models/silver/core/silver__blocks_final.sql | 3 +- models/silver/core/silver__receipts_final.sql | 68 +++++++++++++++++-- .../core/silver__transactions_final.sql | 3 +- .../realtime/streamline__blocks_realtime.sql | 19 ++++-- 4 files changed, 78 insertions(+), 15 deletions(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 96f717b..3fc9359 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -6,7 +6,8 @@ unique_key = 'block_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id, block_hash);", - tags = ['scheduled_core', 'core_v2'] + tags = ['scheduled_core', 'core_v2'], + full_refresh = false ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 89cf1d5..214e1a2 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -6,7 +6,8 @@ unique_key = 'receipt_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['scheduled_core', 'core_v2'] + tags = ['scheduled_core', 'core_v2'], + full_refresh = false ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} @@ -36,8 +37,37 @@ {{ ref('_migrate_receipts') }} {% else %} + {% if execute and not var("MANUAL_FIX") %} + {% if is_incremental() %} + {% set max_mod_query %} + SELECT + MAX(modified_timestamp) modified_timestamp + FROM + {{ this }} + {% endset %} + + {% set max_mod = run_query(max_mod_query) [0] [0] %} + + {% set min_block_date_query %} + SELECT + MIN(origin_block_timestamp :: DATE) block_timestamp + FROM + {{ ref('silver__transactions_v2') }} + WHERE + modified_timestamp >= '{{max_mod}}' + {% endset %} + + {% set min_bd = run_query(min_block_date_query) [0] [0] %} + + {% if not min_bd or min_bd == 'None' %} + {% set min_bd = '2099-01-01' %} + {% endif %} + + {% do log('min_block_date: ' ~ min_bd, info=True) %} + + {% endif %} + {% endif %} --- TODO if execute block for incremental min blockdate WITH txs_with_receipts AS ( SELECT chunk_hash, @@ -47,19 +77,35 @@ WITH txs_with_receipts AS ( response_json :receipts :: ARRAY AS receipts_json, response_json :receipts_outcome :: ARRAY AS receipts_outcome_json, response_json :status :Failure IS NULL AS tx_succeeded, - partition_key AS _partition_by_block_number + partition_key AS _partition_by_block_number, + modified_timestamp FROM {{ ref('silver__transactions_v2') }} - -- TODO incremental logic + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + WHERE origin_block_timestamp :: DATE >= '{{min_bd}}' + {% endif %} + {% endif %} ), blocks AS ( SELECT block_id, block_hash, - block_timestamp + block_timestamp, + modified_timestamp FROM {{ ref('silver__blocks_v2') }} - -- TODO incremental logic + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + WHERE block_timestamp :: DATE >= '{{min_bd}}' + {% endif %} + {% endif %} ), flatten_receipts AS ( SELECT @@ -68,7 +114,8 @@ flatten_receipts AS ( tx_succeeded, VALUE :receipt_id :: STRING AS receipt_id, VALUE :: variant AS receipt_json, - _partition_by_block_number + _partition_by_block_number, + modified_timestamp FROM txs_with_receipts, LATERAL FLATTEN( @@ -105,6 +152,13 @@ receipts_full AS ( ON r.receipt_id = ro.receipt_id LEFT JOIN blocks b ON ro.block_hash = b.block_hash + {% if is_incremental() and not var("MANUAL_FIX") %} + WHERE + GREATEST( + COALESCE(r.modified_timestamp, '1970-01-01'), + COALESCE(b.modified_timestamp, '1970-01-01') + ) >= '{{max_mod}}' + {% endif %} ), FINAL AS ( SELECT diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 129b4bc..2cac3e4 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -6,7 +6,8 @@ unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,tx_signer,tx_receiver);", - tags = ['scheduled_core', 'core_v2'] + tags = ['scheduled_core', 'core_v2'], + full_refresh = false ) }} {% if var('NEAR_MIGRATE_ARCHIVE', False) %} diff --git a/models/streamline/core/realtime/streamline__blocks_realtime.sql b/models/streamline/core/realtime/streamline__blocks_realtime.sql index be321d8..043ce51 100644 --- a/models/streamline/core/realtime/streamline__blocks_realtime.sql +++ b/models/streamline/core/realtime/streamline__blocks_realtime.sql @@ -26,13 +26,20 @@ tbl AS ( FROM near.tests_full.final_gaps_tx ) {% else %} -last_3_days AS ( + {% if var('STREAMLINE_BACKFILL', false) %} + last_3_days AS ( + SELECT + 9820210 AS block_id + ), + {% else %} + last_3_days AS ( - SELECT - ZEROIFNULL(block_id) AS block_id - FROM - {{ ref("_block_lookback") }} -), + SELECT + ZEROIFNULL(block_id) AS block_id + FROM + {{ ref("_block_lookback") }} + ), + {% endif %} tbl AS ( SELECT block_id From c6948fbd1f38d90021ca5943746801fb5c71d8f7 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 3 Mar 2025 19:00:40 -0700 Subject: [PATCH 19/53] add no compile flag to docs, var default --- .github/workflows/dbt_docs_update.yml | 2 +- dbt_project.yml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index e509a06..736e7cc 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -47,7 +47,7 @@ jobs: git checkout -B docs origin/main - name: generate dbt docs - run: dbt docs generate -t prod + run: dbt docs generate -t prod --no-compile - name: move files to docs directory run: | diff --git a/dbt_project.yml b/dbt_project.yml index ed1be01..e0ec03c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -67,9 +67,10 @@ vars: MANUAL_FIX: False OBSERV_FULL_TEST: False DBT_FULL_TEST: False - STREAMLINE_LOAD_LOOKBACK_HOURS: 3 - RECEIPT_MAP_LOOKBACK_HOURS: 6 - IS_MIGRATION: False + STREAMLINE_LOAD_LOOKBACK_HOURS: 3 # todo can deprecate + RECEIPT_MAP_LOOKBACK_HOURS: 6 # todo can deprecate + IS_MIGRATION: False # todo can deprecate + NEAR_MIGRATE_ARCHIVE: False HEAL_MODELS: [] core_folders: [ 'silver/streamline', From a22890590599a53195ff9746f7688231859dacd2 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 4 Mar 2025 10:18:51 -0700 Subject: [PATCH 20/53] add coalesce to incremental --- models/silver/core/silver__blocks_final.sql | 2 +- models/silver/core/silver__receipts_final.sql | 7 ++++--- models/silver/core/silver__transactions_final.sql | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 3fc9359..6ae3975 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -50,7 +50,7 @@ WITH blocks AS ( WHERE modified_timestamp >= ( SELECT - MAX(modified_timestamp) + COALESCE(MAX(modified_timestamp), '1970-01-01') FROM {{ this }} ) diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 214e1a2..c383d61 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -41,13 +41,13 @@ {% if is_incremental() %} {% set max_mod_query %} SELECT - MAX(modified_timestamp) modified_timestamp + COALESCE(MAX(modified_timestamp), '1970-01-01') modified_timestamp FROM {{ this }} {% endset %} {% set max_mod = run_query(max_mod_query) [0] [0] %} - + {% do log('max_mod: ' ~ max_mod, info=True) %} {% set min_block_date_query %} SELECT MIN(origin_block_timestamp :: DATE) block_timestamp @@ -58,9 +58,10 @@ {% endset %} {% set min_bd = run_query(min_block_date_query) [0] [0] %} - + {% do log('min_bd: ' ~ min_bd, info=True) %} {% if not min_bd or min_bd == 'None' %} {% set min_bd = '2099-01-01' %} + {% do log('min_bd: ' ~ min_bd, info=True) %} {% endif %} {% do log('min_block_date: ' ~ min_bd, info=True) %} diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 2cac3e4..5cdf435 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -59,7 +59,7 @@ WITH txs_with_receipts AS ( WHERE modified_timestamp >= ( SELECT - MAX(modified_timestamp) + COALESCE(MAX(modified_timestamp), '1970-01-01') FROM {{ this }} ) From bb52d0e74c0e9fc4cea9c0fc6d2e1376023c710c Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 4 Mar 2025 13:24:10 -0700 Subject: [PATCH 21/53] untag v2_final models from scheduled run --- models/silver/core/silver__blocks_final.sql | 2 +- models/silver/core/silver__receipts_final.sql | 2 +- models/silver/core/silver__transactions_v2.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 6ae3975..5043ff8 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -6,7 +6,7 @@ unique_key = 'block_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id, block_hash);", - tags = ['scheduled_core', 'core_v2'], + tags = ['core_v2'], full_refresh = false ) }} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index c383d61..bb291f2 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -6,7 +6,7 @@ unique_key = 'receipt_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['scheduled_core', 'core_v2'], + tags = ['core_v2'], full_refresh = false ) }} diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index 6584360..ba2dead 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -6,7 +6,7 @@ incremental_predicates = ["dynamic_range_predicate","origin_block_timestamp::date"], unique_key = "tx_hash", cluster_by = ['modified_timestamp::DATE','origin_block_timestamp::date'], - tags = ['scheduled_core', 'core_v2'] + tags = ['core_v2'] ) }} WITH bronze_transactions AS ( From acf9931e04142eabfb5577f52f5d985ccc7e867b Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 4 Mar 2025 13:24:59 -0700 Subject: [PATCH 22/53] correct tx model --- models/silver/core/silver__transactions_final.sql | 2 +- models/silver/core/silver__transactions_v2.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 5cdf435..69439c0 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -6,7 +6,7 @@ unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,tx_signer,tx_receiver);", - tags = ['scheduled_core', 'core_v2'], + tags = ['core_v2'], full_refresh = false ) }} diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index ba2dead..6584360 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -6,7 +6,7 @@ incremental_predicates = ["dynamic_range_predicate","origin_block_timestamp::date"], unique_key = "tx_hash", cluster_by = ['modified_timestamp::DATE','origin_block_timestamp::date'], - tags = ['core_v2'] + tags = ['scheduled_core', 'core_v2'] ) }} WITH bronze_transactions AS ( From 0ef0aa8ff97f5937dff089188cb3a8fef263daeb Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 5 Mar 2025 13:09:21 -0700 Subject: [PATCH 23/53] new model yml files and upd blocks gap test --- models/descriptions/chunks_json.md | 5 + models/descriptions/header_json.md | 5 + models/descriptions/outcome_json.md | 3 + models/descriptions/receipt_json.md | 3 + models/descriptions/status_json.md | 3 + models/descriptions/transaction_json.md | 3 + models/silver/core/silver__blocks_final.yml | 75 +++++++++++ models/silver/core/silver__receipts_final.yml | 98 ++++++++++++++ .../core/silver__transactions_final.yml | 120 ++++++++++++++++++ tests/gaps/core/tests__block_gaps.sql | 10 +- 10 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 models/descriptions/chunks_json.md create mode 100644 models/descriptions/header_json.md create mode 100644 models/descriptions/outcome_json.md create mode 100644 models/descriptions/receipt_json.md create mode 100644 models/descriptions/status_json.md create mode 100644 models/descriptions/transaction_json.md create mode 100644 models/silver/core/silver__blocks_final.yml create mode 100644 models/silver/core/silver__receipts_final.yml create mode 100644 models/silver/core/silver__transactions_final.yml diff --git a/models/descriptions/chunks_json.md b/models/descriptions/chunks_json.md new file mode 100644 index 0000000..c35faaa --- /dev/null +++ b/models/descriptions/chunks_json.md @@ -0,0 +1,5 @@ +{% docs chunks_json %} + +A JSON containing an array of chunk headers contained in this block. + +{% enddocs %} diff --git a/models/descriptions/header_json.md b/models/descriptions/header_json.md new file mode 100644 index 0000000..19ce000 --- /dev/null +++ b/models/descriptions/header_json.md @@ -0,0 +1,5 @@ +{% docs header_json %} + +A JSON containing the block header. + +{% enddocs %} diff --git a/models/descriptions/outcome_json.md b/models/descriptions/outcome_json.md new file mode 100644 index 0000000..adc488d --- /dev/null +++ b/models/descriptions/outcome_json.md @@ -0,0 +1,3 @@ +{% docs outcome_json %} +JSON object containing the outcome of the receipt or transaction execution, including gas usage, status, and logs. +{% enddocs %} diff --git a/models/descriptions/receipt_json.md b/models/descriptions/receipt_json.md new file mode 100644 index 0000000..3758801 --- /dev/null +++ b/models/descriptions/receipt_json.md @@ -0,0 +1,3 @@ +{% docs receipt_json %} +JSON object containing the full receipt data including actions, predecessor, receiver, and other metadata. +{% enddocs %} diff --git a/models/descriptions/status_json.md b/models/descriptions/status_json.md new file mode 100644 index 0000000..03fe070 --- /dev/null +++ b/models/descriptions/status_json.md @@ -0,0 +1,3 @@ +{% docs status_json %} +JSON object containing the status of the transaction, including success or failure information. +{% enddocs %} diff --git a/models/descriptions/transaction_json.md b/models/descriptions/transaction_json.md new file mode 100644 index 0000000..9f5e28b --- /dev/null +++ b/models/descriptions/transaction_json.md @@ -0,0 +1,3 @@ +{% docs transaction_json %} +JSON object containing the full transaction data including actions, signer, receiver, and other metadata. +{% enddocs %} diff --git a/models/silver/core/silver__blocks_final.yml b/models/silver/core/silver__blocks_final.yml new file mode 100644 index 0000000..75eeae9 --- /dev/null +++ b/models/silver/core/silver__blocks_final.yml @@ -0,0 +1,75 @@ +version: 2 + +models: + - name: silver__blocks_final + description: |- + Table containing blocks for NEAR. + tests: + - dbt_utils.recency: + datepart: hour + field: inserted_timestamp + interval: 1 + + columns: + - name: block_id + description: "{{ doc('block_id') }}" + tests: + - not_null + - unique + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: block_hash + description: "{{ doc('block_hash') }}" + tests: + - not_null + - unique + + - name: prev_hash + description: "{{ doc('prev_hash') }}" + tests: + - not_null + - unique + + - name: block_author + description: "{{ doc('block_author') }}" + tests: + - not_null + + - name: chunks_json + description: "{{ doc('chunks_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - ARRAY + - OBJECT + + - name: header_json + description: "{{ doc('header_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" + + - name: blocks_final_id + description: "{{ doc('id') }}" + tests: + - not_null + - unique + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + + - name: _invocation_id + diff --git a/models/silver/core/silver__receipts_final.yml b/models/silver/core/silver__receipts_final.yml new file mode 100644 index 0000000..cc62b37 --- /dev/null +++ b/models/silver/core/silver__receipts_final.yml @@ -0,0 +1,98 @@ +version: 2 + +models: + - name: silver__receipts_final + description: |- + Table containing transaction receipts for NEAR blockchain. + tests: + - dbt_utils.recency: + datepart: hour + field: inserted_timestamp + interval: 1 + + columns: + - name: chunk_hash + description: "{{ doc('chunk_hash') }}" + tests: + - not_null + + - name: block_id + description: "{{ doc('block_id') }}" + tests: + - not_null + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: tx_hash + description: "{{ doc('tx_hash') }}" + tests: + - not_null + + - name: receipt_id + description: "{{ doc('receipt_id') }}" + tests: + - not_null + - unique + + - name: predecessor_id + description: "{{ doc('predecessor_id') }}" + tests: + - not_null + + - name: receiver_id + description: "{{ doc('receiver_id') }}" + tests: + - not_null + + - name: receipt_json + description: "{{ doc('receipt_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: outcome_json + description: "{{ doc('outcome_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: tx_succeeded + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: receipt_succeeded + description: "{{ doc('receipt_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" + + - name: receipts_final_id + description: "{{ doc('id') }}" + tests: + - not_null + - unique + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + + - name: _invocation_id + description: "{{ doc('invocation_id') }}" diff --git a/models/silver/core/silver__transactions_final.yml b/models/silver/core/silver__transactions_final.yml new file mode 100644 index 0000000..0a5cbe5 --- /dev/null +++ b/models/silver/core/silver__transactions_final.yml @@ -0,0 +1,120 @@ +version: 2 + +models: + - name: silver__transactions_final + description: |- + Table containing finalized transactions for NEAR blockchain. + tests: + - dbt_utils.recency: + datepart: hour + field: inserted_timestamp + interval: 1 + + columns: + - name: chunk_hash + description: "{{ doc('chunk_hash') }}" + tests: + - not_null + + - name: block_id + description: "{{ doc('block_id') }}" + tests: + - not_null + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: tx_hash + description: "{{ doc('tx_hash') }}" + tests: + - not_null + - unique + + - name: tx_receiver + description: "{{ doc('tx_receiver') }}" + tests: + - not_null + + - name: tx_signer + description: "{{ doc('tx_signer') }}" + tests: + - not_null + + - name: transaction_json + description: "{{ doc('transaction_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: outcome_json + description: "{{ doc('outcome_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: status_json + description: "{{ doc('status_json') }}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: tx_succeeded + description: "{{ doc('tx_succeeded') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: gas_used + description: "{{ doc('gas_used') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: transaction_fee + description: "{{ doc('transaction_fee') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: attached_gas + description: "{{ doc('attached_gas') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: _partition_by_block_number + description: "{{ doc('_partition_by_block_number') }}" + + - name: transactions_final_id + description: "{{ doc('id') }}" + tests: + - not_null + - unique + + - name: inserted_timestamp + description: "{{ doc('inserted_timestamp') }}" + + - name: modified_timestamp + description: "{{ doc('modified_timestamp') }}" + + - name: _invocation_id + description: "{{ doc('invocation_id') }}" diff --git a/tests/gaps/core/tests__block_gaps.sql b/tests/gaps/core/tests__block_gaps.sql index 01d8459..b03e25f 100644 --- a/tests/gaps/core/tests__block_gaps.sql +++ b/tests/gaps/core/tests__block_gaps.sql @@ -22,17 +22,17 @@ WITH silver_blocks AS ( block_id ASC ) AS prior_hash, _partition_by_block_number, - _inserted_timestamp, + inserted_timestamp, SYSDATE() AS _test_timestamp FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} {% if var('DBT_FULL_TEST') %} WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' + inserted_timestamp < SYSDATE() - INTERVAL '1 hour' {% else %} WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' + inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '1 hour' {% endif %} ) @@ -42,5 +42,3 @@ FROM silver_blocks WHERE prior_hash <> prev_hash - {# Filter out false positive from blocks at start of window (whose parent hash was cut off) #} - AND (_inserted_timestamp > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour') From 7dcdfbf604e316dc55de3a4d685e68772407c214 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Sun, 9 Mar 2025 21:07:50 -0600 Subject: [PATCH 24/53] add scheduled_core to v2_final --- models/silver/core/silver__blocks_final.sql | 2 +- models/silver/core/silver__receipts_final.sql | 2 +- models/silver/core/silver__transactions_final.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/silver/core/silver__blocks_final.sql b/models/silver/core/silver__blocks_final.sql index 5043ff8..6ae3975 100644 --- a/models/silver/core/silver__blocks_final.sql +++ b/models/silver/core/silver__blocks_final.sql @@ -6,7 +6,7 @@ unique_key = 'block_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(block_id, block_hash);", - tags = ['core_v2'], + tags = ['scheduled_core', 'core_v2'], full_refresh = false ) }} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index bb291f2..c383d61 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -6,7 +6,7 @@ unique_key = 'receipt_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['core_v2'], + tags = ['scheduled_core', 'core_v2'], full_refresh = false ) }} diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 69439c0..5cdf435 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -6,7 +6,7 @@ unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,tx_signer,tx_receiver);", - tags = ['core_v2'], + tags = ['scheduled_core', 'core_v2'], full_refresh = false ) }} From 1203b6aa52a2f5a307fa83b6b93a735e9cc896a8 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 10 Mar 2025 11:31:29 -0600 Subject: [PATCH 25/53] upd gha scheduled core --- .github/workflows/dbt_run_scheduled_core.yml | 3 +-- .github/workflows/dbt_run_scheduled_non_core.yml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_core.yml b/.github/workflows/dbt_run_scheduled_core.yml index 41c2fcd..13dd2da 100644 --- a/.github/workflows/dbt_run_scheduled_core.yml +++ b/.github/workflows/dbt_run_scheduled_core.yml @@ -42,9 +42,8 @@ jobs: - name: Run DBT Jobs run: | - dbt run-operation dispatch_github_workflow --args "{'repo_name': 'streamline-snowflake', 'workflow_name': 'dbt_run_near_external_table_update', 'gb_id': '${{ secrets.GB_ID}}'}"; dbt seed; - dbt run -s tag:scheduled_core --vars "{ 'RECEIPT_MAP_LOOKBACK_HOURS': 1.25}"; + dbt run -s tag:scheduled_core; dbt run -s tag:scheduled_non_core models/gold; - name: Store logs uses: actions/upload-artifact@v4 diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml index 183a1a8..8367c60 100644 --- a/.github/workflows/dbt_run_scheduled_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -42,7 +42,6 @@ jobs: - name: Run DBT Jobs run: | - dbt run-operation dispatch_github_workflow --args "{'repo_name': 'streamline-snowflake', 'workflow_name': 'dbt_run_near_external_table_update', 'gb_id': '${{ secrets.GB_ID}}'}"; dbt run -s tag:scheduled_non_core models/gold; - name: Store logs From a45f8d0a61e741a216b0f72f69040417f98cd634 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 10 Mar 2025 11:33:01 -0600 Subject: [PATCH 26/53] rm scheduled_core tag from lake models --- models/silver/core/lake/helpers/_retry_range.sql | 2 +- models/silver/core/lake/helpers/silver__flatten_receipts.sql | 2 +- .../core/lake/helpers/silver__receipt_tx_hash_mapping.sql | 2 +- models/silver/core/lake/silver__streamline_blocks.sql | 2 +- models/silver/core/lake/silver__streamline_receipts.sql | 2 +- models/silver/core/lake/silver__streamline_receipts_final.sql | 2 +- models/silver/core/lake/silver__streamline_shards.sql | 2 +- models/silver/core/lake/silver__streamline_transactions.sql | 2 +- .../silver/core/lake/silver__streamline_transactions_final.sql | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/models/silver/core/lake/helpers/_retry_range.sql b/models/silver/core/lake/helpers/_retry_range.sql index ccfce15..3a84e43 100644 --- a/models/silver/core/lake/helpers/_retry_range.sql +++ b/models/silver/core/lake/helpers/_retry_range.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'ephemeral', - tags = ['helper', 'receipt_map','scheduled_core'] + tags = ['helper', 'receipt_map'] ) }} SELECT diff --git a/models/silver/core/lake/helpers/silver__flatten_receipts.sql b/models/silver/core/lake/helpers/silver__flatten_receipts.sql index 64997c8..a24f26d 100644 --- a/models/silver/core/lake/helpers/silver__flatten_receipts.sql +++ b/models/silver/core/lake/helpers/silver__flatten_receipts.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['helper', 'receipt_map','scheduled_core'] + tags = ['helper', 'receipt_map'] ) }} WITH receipts AS ( diff --git a/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql b/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql index 568609f..8801b6c 100644 --- a/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql +++ b/models/silver/core/lake/helpers/silver__receipt_tx_hash_mapping.sql @@ -1,7 +1,7 @@ {{ config( materalized = 'view', unique_key = 'receipt_id', - tags = ['helper', 'receipt_map','scheduled_core'] + tags = ['helper', 'receipt_map'] ) }} WITH diff --git a/models/silver/core/lake/silver__streamline_blocks.sql b/models/silver/core/lake/silver__streamline_blocks.sql index 123de22..27dd868 100644 --- a/models/silver/core/lake/silver__streamline_blocks.sql +++ b/models/silver/core/lake/silver__streamline_blocks.sql @@ -6,7 +6,7 @@ merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE', '_partition_by_block_number'], unique_key = 'block_id', - tags = ['load', 'load_blocks','scheduled_core', 'deprecated_lake_archive'], + tags = ['load', 'load_blocks', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_receipts.sql b/models/silver/core/lake/silver__streamline_receipts.sql index 18963f7..283952c 100644 --- a/models/silver/core/lake/silver__streamline_receipts.sql +++ b/models/silver/core/lake/silver__streamline_receipts.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], unique_key = 'receipt_id', cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards','scheduled_core', 'deprecated_lake_archive'] + tags = ['load', 'load_shards', 'deprecated_lake_archive'] ) }} WITH shards AS ( diff --git a/models/silver/core/lake/silver__streamline_receipts_final.sql b/models/silver/core/lake/silver__streamline_receipts_final.sql index cd1ebea..b48a2a5 100644 --- a/models/silver/core/lake/silver__streamline_receipts_final.sql +++ b/models/silver/core/lake/silver__streamline_receipts_final.sql @@ -6,7 +6,7 @@ unique_key = 'receipt_object_id', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number', ], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receiver_id,predecessor_id);", - tags = ['receipt_map','scheduled_core', 'deprecated_lake_archive'], + tags = ['receipt_map', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_shards.sql b/models/silver/core/lake/silver__streamline_shards.sql index bb50f80..be6e9f6 100644 --- a/models/silver/core/lake/silver__streamline_shards.sql +++ b/models/silver/core/lake/silver__streamline_shards.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['_inserted_timestamp::DATE', '_partition_by_block_number'], unique_key = 'shard_id', - tags = ['load', 'load_shards','scheduled_core', 'deprecated_lake_archive'], + tags = ['load', 'load_shards', 'deprecated_lake_archive'], full_refresh = False ) }} diff --git a/models/silver/core/lake/silver__streamline_transactions.sql b/models/silver/core/lake/silver__streamline_transactions.sql index 51cac91..53a51fe 100644 --- a/models/silver/core/lake/silver__streamline_transactions.sql +++ b/models/silver/core/lake/silver__streamline_transactions.sql @@ -5,7 +5,7 @@ merge_exclude_columns = ['inserted_timestamp'], unique_key = 'tx_hash', cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], - tags = ['load', 'load_shards', 'scheduled_core', 'deprecated_lake_archive'] + tags = ['load', 'load_shards', 'deprecated_lake_archive'] ) }} WITH chunks AS ( diff --git a/models/silver/core/lake/silver__streamline_transactions_final.sql b/models/silver/core/lake/silver__streamline_transactions_final.sql index 7fad2bc..1ff853c 100644 --- a/models/silver/core/lake/silver__streamline_transactions_final.sql +++ b/models/silver/core/lake/silver__streamline_transactions_final.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'tx_hash', cluster_by = ['block_timestamp::DATE','modified_timestamp::DATE', '_partition_by_block_number'], - tags = ['receipt_map','scheduled_core', 'deprecated_lake_archive'] + tags = ['receipt_map', 'deprecated_lake_archive'] ) }} WITH int_txs AS ( From 94f2f6e9317ba141329d2be876631c8c91c96548 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 10 Mar 2025 12:53:54 -0600 Subject: [PATCH 27/53] gold ref updates --- models/gold/core/core__ez_actions.sql | 38 ++--- .../gold/core/core__fact_actions_events.sql | 38 ----- .../gold/core/core__fact_actions_events.yml | 130 ---------------- ...ore__fact_actions_events_function_call.sql | 38 ----- ...ore__fact_actions_events_function_call.yml | 146 ------------------ models/gold/core/core__fact_blocks.sql | 65 ++++---- models/gold/core/core__fact_receipts.sql | 41 ++--- models/gold/core/core__fact_transactions.sql | 27 +--- .../horizon/horizon__fact_decoded_actions.sql | 37 ----- .../horizon/horizon__fact_decoded_actions.yml | 64 -------- models/silver/core/silver__receipts_final.sql | 2 +- 11 files changed, 69 insertions(+), 557 deletions(-) delete mode 100644 models/gold/core/core__fact_actions_events.sql delete mode 100644 models/gold/core/core__fact_actions_events.yml delete mode 100644 models/gold/core/core__fact_actions_events_function_call.sql delete mode 100644 models/gold/core/core__fact_actions_events_function_call.yml delete mode 100644 models/gold/horizon/horizon__fact_decoded_actions.sql delete mode 100644 models/gold/horizon/horizon__fact_decoded_actions.yml diff --git a/models/gold/core/core__ez_actions.sql b/models/gold/core/core__ez_actions.sql index 6f9d96d..5ef321c 100644 --- a/models/gold/core/core__ez_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -8,8 +8,8 @@ post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id);", tags = ['actions', 'curated', 'scheduled_core', 'grail'] ) }} --- depends_on: {{ ref('silver__streamline_transactions_final') }} --- depends_on: {{ ref('silver__streamline_receipts_final') }} +-- depends_on: {{ ref('silver__transactions_final') }} +-- depends_on: {{ ref('silver__receipts_final') }} {% if execute %} @@ -41,14 +41,14 @@ SELECT MIN(block_timestamp) block_timestamp FROM - {{ ref('silver__streamline_transactions_final') }} A + {{ ref('silver__transactions_final') }} A WHERE modified_timestamp >= '{{max_mod}}' UNION ALL SELECT MIN(block_timestamp) block_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} A + {{ ref('silver__receipts_final') }} A WHERE modified_timestamp >= '{{max_mod}}' ) @@ -76,7 +76,7 @@ WITH transactions AS ( transaction_fee AS tx_fee, modified_timestamp FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE @@ -92,19 +92,18 @@ receipts AS ( tx_hash, block_id, block_timestamp, - receipt_object_id AS receipt_id, + receipt_id, receiver_id AS receipt_receiver_id, - signer_id AS receipt_signer_id, - receipt_actions :predecessor_id :: STRING AS receipt_predecessor_id, + receipt_json :receipt :Action :signer_id :: STRING AS receipt_signer_id, + predecessor_id AS receipt_predecessor_id, receipt_succeeded, - gas_burnt AS receipt_gas_burnt, - status_value, - receipt_actions, + outcome_json :outcome :gas_burnt :: NUMBER AS receipt_gas_burnt, + outcome_json :outcome :status :: VARIANT AS status_value, + receipt_json, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} WHERE @@ -132,9 +131,8 @@ join_data AS ( r.receipt_succeeded, r.receipt_gas_burnt, r.status_value, - r.receipt_actions, - r._partition_by_block_number, - r._inserted_timestamp + r.receipt_json, + r._partition_by_block_number FROM receipts r LEFT JOIN transactions t @@ -184,7 +182,7 @@ flatten_actions AS ( ) as receipt_status_value, False AS is_delegated, INDEX AS action_index, - receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, + receipt_json :receipt :Action :gas_price :: NUMBER AS action_gas_price, IFF( VALUE = 'CreateAccount', VALUE, @@ -247,12 +245,11 @@ flatten_actions AS ( ), action_data ) AS action_data_parsed, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM join_data, LATERAL FLATTEN( - receipt_actions :receipt :Action :actions :: ARRAY + receipt_json :receipt :Action :actions :: ARRAY ) ), flatten_delegated_actions AS ( @@ -333,7 +330,6 @@ SELECT action_data_parsed AS action_data, action_gas_price, _partition_by_block_number, - _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['receipt_id', 'action_index'] ) }} AS actions_id, diff --git a/models/gold/core/core__fact_actions_events.sql b/models/gold/core/core__fact_actions_events.sql deleted file mode 100644 index 5dc2df0..0000000 --- a/models/gold/core/core__fact_actions_events.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - tags = ['core'] -) }} - -WITH actions AS ( - - SELECT - * - FROM - {{ ref('silver__actions_events_s3') }} -) -SELECT - action_id, - tx_hash, - receipt_object_id AS receipt_id, - predecessor_id, - receiver_id, - signer_id, - block_id, - block_timestamp, - action_index, - action_name, - action_data, - logs, - receipt_succeeded, - COALESCE( - actions_events_id, - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id', 'action_index'] - ) }} - ) AS fact_actions_events_id, - receipt_object_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp -FROM - actions diff --git a/models/gold/core/core__fact_actions_events.yml b/models/gold/core/core__fact_actions_events.yml deleted file mode 100644 index 6135eee..0000000 --- a/models/gold/core/core__fact_actions_events.yml +++ /dev/null @@ -1,130 +0,0 @@ -version: 2 - -models: - - name: core__fact_actions_events - description: |- - Deprecating soon! Please migrate to using the new `core__ez_actions` table. - This view will be removed in Q1 2025. - tests: - - dbt_utils.recency: - datepart: hours - field: block_timestamp - interval: 2 - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: RECEIPT_ID - description: "{{ doc('receipt_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: ACTION_INDEX - description: "{{ doc('action_index')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: ACTION_DATA - description: "{{ doc('action_data')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - OBJECT - - VARIANT - - - name: LOGS - description: "{{ doc('logs')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: FACT_ACTIONS_EVENTS_ID - description: "{{ doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' AND FACT_ACTIONS_EVENTS_ID != 'cf646ad92e6df243ffabf07c47c0f2c1' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/core/core__fact_actions_events_function_call.sql b/models/gold/core/core__fact_actions_events_function_call.sql deleted file mode 100644 index 637250a..0000000 --- a/models/gold/core/core__fact_actions_events_function_call.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - tags = ['core'] -) }} - -WITH actions_events_function_call AS ( - - SELECT - * - FROM - {{ ref('silver__actions_events_function_call_s3') }} -) -SELECT - action_id, - tx_hash, - receiver_id, - predecessor_id, - signer_id, - block_id, - block_timestamp, - action_name, - method_name, - args, - deposit, - attached_gas, - logs, - receipt_succeeded, - COALESCE( - actions_events_function_call_id, - {{ dbt_utils.generate_surrogate_key( - ['action_id'] - ) }} - ) AS fact_actions_events_function_call_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp -FROM - actions_events_function_call diff --git a/models/gold/core/core__fact_actions_events_function_call.yml b/models/gold/core/core__fact_actions_events_function_call.yml deleted file mode 100644 index c496fa9..0000000 --- a/models/gold/core/core__fact_actions_events_function_call.yml +++ /dev/null @@ -1,146 +0,0 @@ -version: 2 - -models: - - name: core__fact_actions_events_function_call - description: |- - Deprecating soon! Please migrate to using the new `core__ez_actions` table. All FunctionCalls are decoded in the args element where action_name = 'FunctionCall'. - This view will be removed in Q1 2025. - tests: - - dbt_utils.recency: - datepart: hours - field: block_timestamp - interval: 2 - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: ARGS - description: "{{ doc('args')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - OBJECT - - - name: DEPOSIT - description: "{{ doc('deposit')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: ATTACHED_GAS - description: "{{ doc('attached_gas')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: LOGS - description: "{{ doc('logs')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - VARIANT - - OBJECT - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - tests: - - not_null - - - name: FACT_ACTIONS_EVENTS_FUNCTION_CALL_ID - description: "{{doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index fb700e3..4c6857b 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -9,47 +9,42 @@ WITH blocks AS ( SELECT * FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} ) SELECT block_id, block_timestamp, block_hash, block_author, - header, - header :challenges_result :: ARRAY AS block_challenges_result, - header :challenges_root :: STRING AS block_challenges_root, - header :chunk_headers_root :: STRING AS chunk_headers_root, - header :chunk_tx_root :: STRING AS chunk_tx_root, - header :chunk_mask :: ARRAY AS chunk_mask, - header :chunk_receipts_root :: STRING AS chunk_receipts_root, - chunks, - header :chunks_included :: NUMBER AS chunks_included, - epoch_id, - header :epoch_sync_data_hash :: STRING AS epoch_sync_data_hash, - gas_price, - header :last_ds_final_block :: STRING AS last_ds_final_block, - header :last_final_block :: STRING AS last_final_block, - latest_protocol_version, - header: next_bp_hash :: STRING AS next_bp_hash, - next_epoch_id, - header :outcome_root :: STRING AS outcome_root, + header_json AS header, + header_json :challenges_result :: ARRAY AS block_challenges_result, + header_json :challenges_root :: STRING AS block_challenges_root, + header_json :chunk_headers_root :: STRING AS chunk_headers_root, + header_json :chunk_tx_root :: STRING AS chunk_tx_root, + header_json :chunk_mask :: ARRAY AS chunk_mask, + header_json :chunk_receipts_root :: STRING AS chunk_receipts_root, + chunks_json AS chunks, + header_json :chunks_included :: NUMBER AS chunks_included, + header_json :epoch_id :: STRING AS epoch_id, + header_json :epoch_sync_data_hash :: STRING AS epoch_sync_data_hash, + header_json :gas_price :: FLOAT AS gas_price, + header_json :last_ds_final_block :: STRING AS last_ds_final_block, + header_json :last_final_block :: STRING AS last_final_block, + header_json :latest_protocol_version :: STRING AS latest_protocol_version, + header_json :next_bp_hash :: STRING AS next_bp_hash, + header_json :next_epoch_id :: STRING AS next_epoch_id, + header_json :outcome_root :: STRING AS outcome_root, prev_hash, - header :prev_height :: NUMBER AS prev_height, - header :prev_state_root :: STRING AS prev_state_root, - header :random_value :: STRING AS random_value, - header :rent_paid :: FLOAT AS rent_paid, - header :signature :: STRING AS signature, - total_supply, - validator_proposals, - validator_reward, - COALESCE( - streamline_blocks_id, - {{ dbt_utils.generate_surrogate_key( - ['block_id'] - ) }} - ) AS fact_blocks_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + header_json :prev_height :: NUMBER AS prev_height, + header_json :prev_state_root :: STRING AS prev_state_root, + header_json :random_value :: STRING AS random_value, + header_json :rent_paid :: FLOAT AS rent_paid, + header_json :signature :: STRING AS signature, + header_json :total_supply :: FLOAT AS total_supply, + header_json :validator_proposals :: ARRAY AS validator_proposals, + header_json :validator_reward :: FLOAT AS validator_reward, + blocks_final_id AS fact_blocks_id, + inserted_timestamp, + modified_timestamp FROM blocks diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index b403770..51b2efa 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -4,37 +4,24 @@ tags = ['core'] ) }} -WITH receipts AS ( - - SELECT - * - FROM - {{ ref('silver__streamline_receipts_final') }} -) SELECT block_timestamp, block_id, tx_hash, - receipt_object_id AS receipt_id, - receipt_outcome_id, + receipt_id, + outcome_json :outcome :receipt_ids :: ARRAY AS receipt_outcome_id, -- TODO DEPRECATE THIS, it's in outcome_json receiver_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, - receipt_actions AS actions, - execution_outcome AS outcome, - gas_burnt, - status_value, - logs, - proof, - metadata, + predecessor_id, + receipt_json AS actions, -- TODO this should be renamed. It's not just actions, it's the full receipt input + outcome_json AS outcome, + outcome_json :outcome :gas_burnt :: NUMBER AS gas_burnt, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, + outcome_json :proof :: ARRAY AS proof, -- TODO DEPRECATE THIS, it's in outcome_json + outcome_json :outcome :metadata :: VARIANT AS metadata, -- TODO DEPRECATE THIS, it's in outcome_json receipt_succeeded, - COALESCE( - streamline_receipts_final_id, - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id'] - ) }} - ) AS fact_receipts_id, - receipt_object_id, -- to be deprecated - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + receipts_final_id AS fact_receipts_id, + inserted_timestamp, + modified_timestamp FROM - receipts + {{ ref('silver__receipts_final') }} diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 9cf1193..237a200 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -4,34 +4,21 @@ tags = ['core'] ) }} -WITH transactions AS ( - - SELECT - * - FROM - {{ ref('silver__streamline_transactions_final') }} -) SELECT tx_hash, block_id, - block_hash, block_timestamp, - nonce, - signature, + transaction_json :nonce :: INT AS nonce, + transaction_json :signature :: STRING AS signature, tx_receiver, tx_signer, - tx, + transaction_json AS tx, gas_used, transaction_fee, attached_gas, tx_succeeded, - COALESCE( - streamline_transactions_final_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash'] - ) }} - ) AS fact_transactions_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + transactions_final_id AS fact_transactions_id, + inserted_timestamp, + modified_timestamp FROM - transactions + {{ ref('silver__transactions_final') }} diff --git a/models/gold/horizon/horizon__fact_decoded_actions.sql b/models/gold/horizon/horizon__fact_decoded_actions.sql deleted file mode 100644 index 3867d49..0000000 --- a/models/gold/horizon/horizon__fact_decoded_actions.sql +++ /dev/null @@ -1,37 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['core', 'horizon'] -) }} - -WITH horizon AS ( - - SELECT - action_id_horizon, - receipt_object_id, - tx_hash, - block_id, - block_timestamp, - method_name, - args, - deposit, - attached_gas, - receiver_id, - signer_id, - receipt_succeeded, - COALESCE( - horizon_decoded_actions_id, - {{ dbt_utils.generate_surrogate_key( - ['action_id_horizon'] - ) }} - ) AS fact_decoded_actions_id, - COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver_horizon__decoded_actions') }} - WHERE - method_name != 'set' -) -SELECT - * -FROM - horizon diff --git a/models/gold/horizon/horizon__fact_decoded_actions.yml b/models/gold/horizon/horizon__fact_decoded_actions.yml deleted file mode 100644 index b538dee..0000000 --- a/models/gold/horizon/horizon__fact_decoded_actions.yml +++ /dev/null @@ -1,64 +0,0 @@ - -version: 2 - -models: - - name: horizon__fact_decoded_actions - description: |- - Deprecating Soon - please cease using this table by February 1, 2025. - - columns: - - name: ACTION_ID_HORIZON - description: "{{ doc('action_id')}}" - - - name: RECEIPT_OBJECT_ID - description: "{{ doc('receipt_object_id')}}" - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - - - name: ARGS - description: "{{ doc('args')}}" - - - name: DEPOSIT - description: "{{ doc('deposit')}}" - - - name: ATTACHED_GAS - description: "{{ doc('attached_gas')}}" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: FACT_DECODED_ACTIONS_ID - description: "{{doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index c383d61..7abd640 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -173,7 +173,7 @@ FINAL AS ( receipt_json, outcome_json, tx_succeeded, - outcome_json :status :Failure IS NULL AS receipt_succeeded, + outcome_json :outcome :status :Failure IS NULL AS receipt_succeeded, _partition_by_block_number FROM receipts_full From eebd710e18ad72f2057a1a559496b6e1c99736b4 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 10 Mar 2025 13:25:32 -0600 Subject: [PATCH 28/53] test - migration receipt ct --- .../core/tests__receipt_migration_gaps.sql | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/gaps/core/tests__receipt_migration_gaps.sql diff --git a/tests/gaps/core/tests__receipt_migration_gaps.sql b/tests/gaps/core/tests__receipt_migration_gaps.sql new file mode 100644 index 0000000..da24360 --- /dev/null +++ b/tests/gaps/core/tests__receipt_migration_gaps.sql @@ -0,0 +1,35 @@ +{{ config( + severity = 'error', + tags = ['gap_test'] +) }} + +WITH +archive AS ( + select + floor(block_id, -6) as block_group, + count(distinct receipt_id) as receipt_count, + count(distinct block_id) as block_count + from {{ ref('silver__streamline_receipts') }} + where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} + group by 1 +), +destination AS ( + select + floor(block_id, -6) as block_group, + count(distinct receipt_id) as receipt_count, + count(distinct block_id) as block_count + from {{ ref('silver__receipts_final') }} + where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} + group by 1 +) +select + archive.block_group, + archive.receipt_count as receipt_ct_expected, + archive.block_count as block_ct_expected, + destination.receipt_count as receipt_ct_actual, + destination.block_count as block_ct_actual +from archive +left join destination +on archive.block_group = destination.block_group +where receipt_ct_expected <> receipt_ct_actual +order by block_group From 1f83f97472a04b2a49c5700ea6c01bef0ce15d7c Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:55:05 -0600 Subject: [PATCH 29/53] rm where clause on migration gap testing --- tests/gaps/core/tests__receipt_migration_gaps.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/gaps/core/tests__receipt_migration_gaps.sql b/tests/gaps/core/tests__receipt_migration_gaps.sql index da24360..ea40580 100644 --- a/tests/gaps/core/tests__receipt_migration_gaps.sql +++ b/tests/gaps/core/tests__receipt_migration_gaps.sql @@ -10,7 +10,7 @@ archive AS ( count(distinct receipt_id) as receipt_count, count(distinct block_id) as block_count from {{ ref('silver__streamline_receipts') }} - where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} + -- where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} group by 1 ), destination AS ( @@ -19,7 +19,7 @@ destination AS ( count(distinct receipt_id) as receipt_count, count(distinct block_id) as block_count from {{ ref('silver__receipts_final') }} - where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} + -- where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} group by 1 ) select @@ -31,5 +31,5 @@ select from archive left join destination on archive.block_group = destination.block_group -where receipt_ct_expected <> receipt_ct_actual +-- where receipt_ct_expected <> receipt_ct_actual order by block_group From c99b9d41362a63f56250d96eda3c6b18862da5e7 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:08:41 -0600 Subject: [PATCH 30/53] rm comment --- tests/gaps/core/tests__receipt_migration_gaps.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/gaps/core/tests__receipt_migration_gaps.sql b/tests/gaps/core/tests__receipt_migration_gaps.sql index ea40580..cb67573 100644 --- a/tests/gaps/core/tests__receipt_migration_gaps.sql +++ b/tests/gaps/core/tests__receipt_migration_gaps.sql @@ -10,7 +10,6 @@ archive AS ( count(distinct receipt_id) as receipt_count, count(distinct block_id) as block_count from {{ ref('silver__streamline_receipts') }} - -- where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} group by 1 ), destination AS ( @@ -19,7 +18,6 @@ destination AS ( count(distinct receipt_id) as receipt_count, count(distinct block_id) as block_count from {{ ref('silver__receipts_final') }} - -- where floor(block_id, -6) between {{ var('start_block') }} and {{ var('end_block') }} group by 1 ) select From db72d87736cb50d32667c1fae7ef8b0f04560335 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:26:00 -0600 Subject: [PATCH 31/53] stats, observability, drop atlas --- ...ver_observability__blocks_completeness.sql | 8 +- ...ver_observability__chunks_completeness.sql | 16 +-- ...ilver_observability__logs_completeness.sql | 10 +- .../atlas/silver__atlas_accounts_created.sql | 63 ---------- .../atlas/silver__atlas_accounts_created.yml | 35 ------ .../silver__atlas_address_first_action.sql | 66 ----------- .../silver__atlas_address_first_action.yml | 54 --------- models/silver/atlas/silver__atlas_maa.sql | 109 ------------------ models/silver/atlas/silver__atlas_maa.yml | 53 --------- .../atlas/silver__atlas_nft_30_trailing.sql | 51 -------- .../atlas/silver__atlas_nft_30_trailing.yml | 36 ------ .../atlas/silver__atlas_nft_detailed.sql | 50 -------- .../atlas/silver__atlas_nft_detailed.yml | 61 ---------- .../silver/atlas/silver__atlas_nft_table.sql | 57 --------- .../silver/atlas/silver__atlas_nft_table.yml | 66 ----------- .../atlas/silver__atlas_nft_transactions.sql | 107 ----------------- .../atlas/silver__atlas_nft_transactions.yml | 77 ------------- ...ilver_stats__core_metrics_block_hourly.sql | 12 +- .../silver_stats__core_metrics_hourly.sql | 12 +- .../core/tests__receipt_migration_gaps.sql | 7 +- 20 files changed, 33 insertions(+), 917 deletions(-) delete mode 100644 models/silver/atlas/silver__atlas_accounts_created.sql delete mode 100644 models/silver/atlas/silver__atlas_accounts_created.yml delete mode 100644 models/silver/atlas/silver__atlas_address_first_action.sql delete mode 100644 models/silver/atlas/silver__atlas_address_first_action.yml delete mode 100644 models/silver/atlas/silver__atlas_maa.sql delete mode 100644 models/silver/atlas/silver__atlas_maa.yml delete mode 100644 models/silver/atlas/silver__atlas_nft_30_trailing.sql delete mode 100644 models/silver/atlas/silver__atlas_nft_30_trailing.yml delete mode 100644 models/silver/atlas/silver__atlas_nft_detailed.sql delete mode 100644 models/silver/atlas/silver__atlas_nft_detailed.yml delete mode 100644 models/silver/atlas/silver__atlas_nft_table.sql delete mode 100644 models/silver/atlas/silver__atlas_nft_table.yml delete mode 100644 models/silver/atlas/silver__atlas_nft_transactions.sql delete mode 100644 models/silver/atlas/silver__atlas_nft_transactions.yml diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 203fa5d..db693c5 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('silver__streamline_blocks') }} +-- depends_on: {{ ref('silver__blocks_final') }} {{ config( materialized = 'incremental', unique_key = 'test_timestamp', @@ -16,8 +16,8 @@ WITH blocks_joined AS ( b.block_timestamp AS next_block_timestamp, b.prev_hash AS next_prev_hash FROM - {{ ref('silver__streamline_blocks') }} A -- Streamline Migration TODO - change this to fact blocks once table - LEFT JOIN {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} A + LEFT JOIN {{ ref('silver__blocks_final') }} b ON A.block_hash = b.prev_hash WHERE @@ -34,7 +34,7 @@ AND ( SELECT MIN(block_id) AS block_id FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) AND DATEADD('hour', -95, SYSDATE()) diff --git a/models/silver/_observability/silver_observability__chunks_completeness.sql b/models/silver/_observability/silver_observability__chunks_completeness.sql index 5d630c6..b4c8067 100644 --- a/models/silver/_observability/silver_observability__chunks_completeness.sql +++ b/models/silver/_observability/silver_observability__chunks_completeness.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('silver__streamline_blocks') }} +-- depends_on: {{ ref('silver__blocks_final') }} {{ config( materialized = 'incremental', unique_key = 'test_timestamp', @@ -12,11 +12,11 @@ WITH block_chunks_included AS ( SELECT block_id, block_timestamp, - header :chunks_included :: INT AS chunks_included, + header_json :chunks_included :: INT AS chunks_included, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} @@ -29,7 +29,7 @@ AND ( SELECT MIN(block_id) AS block_id FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) AND DATEADD('hour', -95, SYSDATE()) @@ -70,13 +70,13 @@ summary_stats AS ( ), chunks_per_block AS ( SELECT - block_id, + origin_block_id AS block_id, MAX(_inserted_timestamp) AS _inserted_timestamp, COUNT( - DISTINCT chunk :header :chunk_hash :: STRING + chunk_hash ) AS chunk_ct FROM - {{ ref('silver__streamline_shards') }} -- Streamline Migration TODO - change this to fact shards once table + {{ ref('silver__transactions_v2') }} WHERE block_id >= (SELECT min_block FROM summary_stats) AND diff --git a/models/silver/_observability/silver_observability__logs_completeness.sql b/models/silver/_observability/silver_observability__logs_completeness.sql index b37a29f..de4cb5b 100644 --- a/models/silver/_observability/silver_observability__logs_completeness.sql +++ b/models/silver/_observability/silver_observability__logs_completeness.sql @@ -4,7 +4,7 @@ full_refresh = False, tags = ['observability'] ) }} - +-- TODO this can be deprecated. Not a good metric of completeness. WITH summary_stats AS ( SELECT @@ -14,7 +14,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp <= DATEADD('hour', -12, SYSDATE()) @@ -28,7 +28,7 @@ AND ( SELECT MIN(block_id) AS block_id FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) AND DATEADD('hour', -95, SYSDATE()) @@ -82,7 +82,7 @@ broken_blocks AS ( SELECT DISTINCT block_id as block_id FROM - {{ ref('silver__streamline_receipts_final') }} -- Streamline Migration TODO - change this to fact receipts once table + {{ ref('silver__receipts_final') }} r LEFT JOIN {{ ref('silver__logs_s3') }} l USING ( @@ -93,7 +93,7 @@ broken_blocks AS ( WHERE l.tx_hash IS NULL AND ARRAY_SIZE( - r.logs + r.outcome_json :outcome :logs :: ARRAY ) > 0 ), impacted_blocks AS ( diff --git a/models/silver/atlas/silver__atlas_accounts_created.sql b/models/silver/atlas/silver__atlas_accounts_created.sql deleted file mode 100644 index e9a2969..0000000 --- a/models/silver/atlas/silver__atlas_accounts_created.sql +++ /dev/null @@ -1,63 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - unique_key = 'atlas_account_created_id', - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - tags = ['atlas'] -) }} - -WITH accts AS ( - - SELECT - receiver_id, - block_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__streamline_receipts_final') }} - WHERE - receipt_succeeded - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - INTERVAL '2 days' - FROM - {{ this }} - ) - {% endif %} - {% endif %} - - qualify ROW_NUMBER() over ( - PARTITION BY receiver_id - ORDER BY - block_timestamp - ) = 1 -), -FINAL AS ( - SELECT - block_timestamp :: DATE AS "DAY", - COUNT(*) AS wallets_created, - MAX(_modified_timestamp) AS _modified_timestamp - FROM - accts - GROUP BY - 1 -) -SELECT - day, - wallets_created, - _modified_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['DAY'] - ) }} AS atlas_account_created_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL -WHERE - DAY IS NOT NULL diff --git a/models/silver/atlas/silver__atlas_accounts_created.yml b/models/silver/atlas/silver__atlas_accounts_created.yml deleted file mode 100644 index 710c2cf..0000000 --- a/models/silver/atlas/silver__atlas_accounts_created.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_accounts_created - description: |- - Wallet creation on NEAR - columns: - - name: atlas_account_created_id - description: "{{ doc('id')}}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('date')}}" - tests: - - not_null - - - name: wallets_created - description: "{{ doc('wallets_created')}}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_address_first_action.sql b/models/silver/atlas/silver__atlas_address_first_action.sql deleted file mode 100644 index 2d5653b..0000000 --- a/models/silver/atlas/silver__atlas_address_first_action.sql +++ /dev/null @@ -1,66 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - incremental_stratege = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'address', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['atlas'] -) }} - -WITH txs AS ( - - SELECT - tx_signer AS address, - block_id, - block_timestamp, - tx_hash, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__streamline_transactions_final') }} - WHERE - {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} - {% endif %} -), -FINAL AS ( - SELECT - address, - MIN(block_timestamp) AS first_tx_timestamp, - MIN(block_id) AS first_tx_block_id, - MIN(_partition_by_block_number) AS _partition_by_block_number, - MIN(_inserted_timestamp) AS _inserted_timestamp, - MIN(_modified_timestamp) AS _modified_timestamp - FROM - txs - GROUP BY - 1 -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['address'] - ) }} AS atlas_address_first_action_id, - address, - first_tx_timestamp, - first_tx_block_id, - _partition_by_block_number, - _inserted_timestamp, - _modified_timestamp, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY address - ORDER BY - first_tx_timestamp - ) = 1 diff --git a/models/silver/atlas/silver__atlas_address_first_action.yml b/models/silver/atlas/silver__atlas_address_first_action.yml deleted file mode 100644 index 80c2f36..0000000 --- a/models/silver/atlas/silver__atlas_address_first_action.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_address_first_action - description: |- - Parses transactions table for the block number and timestamp of a wallets first signed transaction. - - columns: - - name: atlas_address_first_action_id - description: "{{ doc('id') }}" - tests: - - not_null - - unique - - - name: address - description: "{{ doc('address') }}" - tests: - - not_null - - unique - - - name: first_tx_timestamp - description: "{{ doc('block_timestamp')}}" - tests: - - not_null - - - name: first_tx_block_id - description: "{{ doc('block_id') }}" - tests: - - not_null - - - name: _partition_by_block_number - description: "{{ doc('_partition_by_block_number') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - name: not_null_silver__atlas_address_first_action_INSERTED_TIMESTAMP_ - test_name: not_null - - - name: _modified_timestamp - description: "{{ doc('_modified_timestamp') }}" - - - name: inserted_timestamp - description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - - - name: modified_timestamp - description: "{{ doc('modified_timestamp') }}" - tests: - - not_null - - - name: _invocation_id - description: "{{ doc('invocation_id') }}" diff --git a/models/silver/atlas/silver__atlas_maa.sql b/models/silver/atlas/silver__atlas_maa.sql deleted file mode 100644 index a34b5fc..0000000 --- a/models/silver/atlas/silver__atlas_maa.sql +++ /dev/null @@ -1,109 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - incremental_stratege = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'day', - tags = ['atlas'] -) }} - -WITH dates AS ( - - SELECT - date_day AS day - FROM - {{ source( - 'crosschain', - 'dim_dates' - ) }} - -{% if is_incremental() %} -WHERE - date_day > ( - SELECT - MAX(day) - FROM - {{ this }} - ) - AND date_day < SYSDATE() :: DATE -{% else %} -WHERE - date_day BETWEEN '2020-07-22' - AND SYSDATE() :: DATE -{% endif %} -), -signer_first_date AS ( - SELECT - address, - first_tx_timestamp - FROM - {{ ref('silver__atlas_address_first_action') }} -), -txns AS ( - SELECT - block_timestamp :: DATE AS day, - tx_signer, - first_tx_timestamp, - _inserted_timestamp - FROM - {{ ref('silver__streamline_transactions_final') }} - t - LEFT JOIN signer_first_date s - ON t.tx_signer = s.address - - {% if var("MANUAL_FIX") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% else %} - -{% if is_incremental() %} -WHERE - block_timestamp :: DATE >= ( - SELECT - MAX(day) - FROM - {{ this }} - ) - INTERVAL '30 days' -{% endif %} -{% endif %} -), -FINAL AS ( - SELECT - d.day, - COUNT( - DISTINCT tx_signer - ) AS maa, - COUNT( - DISTINCT IFF( - first_tx_timestamp >= d.day - INTERVAL '30 Days' - AND first_tx_timestamp < d.day, - tx_signer, - NULL - ) - ) AS new_maas, - MAX(_inserted_timestamp) AS _inserted_timestamp - FROM - dates d - LEFT JOIN txns t - ON t.day < d.day - AND t.day >= d.day - INTERVAL '30 days' - WHERE - d.day != SYSDATE() :: DATE - GROUP BY - 1 -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['day'] - ) }} AS atlas_maa_id, - day, - maa, - new_maas, - maa - new_maas AS returning_maas, - _inserted_timestamp, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/atlas/silver__atlas_maa.yml b/models/silver/atlas/silver__atlas_maa.yml deleted file mode 100644 index e8d98ce..0000000 --- a/models/silver/atlas/silver__atlas_maa.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_maa - description: |- - Monthly Active Accounts (wallets) on NEAR, including new and returning wallets, calculated over a rolling 30 day window. An active account, here, is defined as the signing of at least one transaction. - - columns: - - name: atlas_maa_id - description: "{{ doc('id') }}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('active_day') }}" - tests: - - not_null - - unique - - - name: maa - description: "{{ doc('maa')}}" - tests: - - not_null - - - name: new_maas - description: "{{ doc('new_maas') }}" - tests: - - not_null - - - name: returning_maas - description: "{{ doc('returning_maas') }}" - tests: - - not_null - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - name: not_null_silver__atlas_near_maa_INSERTED_TIMESTAMP_ - test_name: not_null - - - name: inserted_timestamp - description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - - - name: modified_timestamp - description: "{{ doc('modified_timestamp') }}" - tests: - - not_null - - - name: _invocation_id - description: "{{ doc('invocation_id') }}" diff --git a/models/silver/atlas/silver__atlas_nft_30_trailing.sql b/models/silver/atlas/silver__atlas_nft_30_trailing.sql deleted file mode 100644 index 8de466e..0000000 --- a/models/silver/atlas/silver__atlas_nft_30_trailing.sql +++ /dev/null @@ -1,51 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'incremental', - unique_key = 'atlas_nft_30_trailing_id', - incremental_strategy = "merge", - merge_exclude_columns = ["inserted_timestamp"], - tags = ['atlas'] -) }} - -WITH date_range AS ( - - SELECT - date_day AS DAY - FROM - {{ ref('silver__dates') }} - WHERE - -{% if is_incremental() %} -date_day >= SYSDATE() - INTERVAL '3 DAY' -{% else %} - date_day >= '2021-01-01' -- first day of data -{% endif %} -AND date_day <= SYSDATE() :: DATE -), -FINAL AS ( - SELECT - d.day AS DAY, - COUNT( - t.tx_hash - ) AS txns - FROM - date_range d - LEFT JOIN {{ ref('silver__atlas_nft_transactions') }} - t - ON t.day BETWEEN d.day - INTERVAL '29 day' - AND d.day - GROUP BY - d.day -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['day'] - ) }} AS atlas_nft_30_trailing_id, - DAY, - txns, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/atlas/silver__atlas_nft_30_trailing.yml b/models/silver/atlas/silver__atlas_nft_30_trailing.yml deleted file mode 100644 index df44dee..0000000 --- a/models/silver/atlas/silver__atlas_nft_30_trailing.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_30_trailing - description: |- - This incremental dbt model generates a summary of NFT transactions from the 'silver__atlas_nft_transactions' table. It provides a daily count of transactions, accounting for a 30-day lookback period for each day within the specified date range. - - columns: - - name: atlas_nft_30_trailing_id - description: "{{ doc('id')}}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('date')}}" - tests: - - not_null - - unique - - name: txns - description: "{{ doc('tx_count')}}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_nft_detailed.sql b/models/silver/atlas/silver__atlas_nft_detailed.sql deleted file mode 100644 index 933b029..0000000 --- a/models/silver/atlas/silver__atlas_nft_detailed.sql +++ /dev/null @@ -1,50 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'table', - unique_key = 'atlas_nft_detailed_id', - tags = ['atlas'] -) }} - -WITH nft_data AS ( - - SELECT - * - FROM - {{ ref('silver__atlas_nft_transactions') }} -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['DAY', 'receiver_id'] - ) }} AS atlas_nft_detailed_id, - DAY, - receiver_id, - COUNT( - DISTINCT token_id - ) AS tokens, - COUNT( - CASE - WHEN method_name = 'nft_transfer' THEN tx_hash - END - ) AS all_transfers, - COUNT( - DISTINCT owner - ) AS owners, - COUNT(*) AS transactions, - COUNT( - CASE - WHEN method_name != 'nft_transfer' THEN tx_hash - END - ) AS mints, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id, - MAX(_inserted_timestamp) AS _inserted_timestamp -FROM - nft_data -GROUP BY - 1, - 2, - 3 -ORDER BY - 4 DESC diff --git a/models/silver/atlas/silver__atlas_nft_detailed.yml b/models/silver/atlas/silver__atlas_nft_detailed.yml deleted file mode 100644 index 7687c0e..0000000 --- a/models/silver/atlas/silver__atlas_nft_detailed.yml +++ /dev/null @@ -1,61 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_detailed - description: |- - This is an incremental dbt model that gives an overview of NFT transactions in NEAR. - -columns: - - name: atlas_nft_detailed_id - description: "{{ doc('id')}}" - tests: - - not_null - - unique - - - name: day - description: "{{ doc('date')}}" - tests: - - not_null - - - name: receiver_id - description: "The identifier of the receiver in the NFT transaction." - tests: - - not_null - - - name: tokens - description: "The count of unique tokens transferred to the receiver on the given day." - tests: - - not_null - - - name: all_transfers - description: "The total number of 'nft_transfer' method transactions that occurred." - tests: - - not_null - - - name: owners - description: "The count of distinct owners who have interacted with the NFT." - tests: - - not_null - - - name: transactions - description: "{{ doc('tx_count')}}" - tests: - - not_null - - - name: mints - description: "The count of transactions where the 'method_name' is not 'nft_transfer', indicating minting actions." - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_nft_table.sql b/models/silver/atlas/silver__atlas_nft_table.sql deleted file mode 100644 index 2950dd0..0000000 --- a/models/silver/atlas/silver__atlas_nft_table.sql +++ /dev/null @@ -1,57 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = 'table', - unique_key = 'atlas_nft_table_id', - tags = ['atlas'] -) }} - -WITH nft_data AS ( - - SELECT - * - FROM - {{ ref('silver__atlas_nft_transactions') }} -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['receiver_id'] - ) }} AS atlas_nft_table_id, - receiver_id, - COUNT( - DISTINCT token_id - ) AS tokens, - COUNT( - CASE - WHEN method_name = 'nft_transfer' - AND DAY >= (SYSDATE() :: DATE - INTERVAL '1 day') THEN tx_hash END - ) AS transfers_24h, - COUNT( - CASE - WHEN method_name = 'nft_transfer' - AND DAY >= (SYSDATE() :: DATE - INTERVAL '3 day') THEN tx_hash END - ) AS transfers_3d, - COUNT( - CASE - WHEN method_name = 'nft_transfer' THEN tx_hash - END - ) AS all_transfers, - COUNT( - DISTINCT owner - ) AS owners, - COUNT(*) AS transactions, - COUNT( - CASE - WHEN method_name != 'nft_transfer' THEN tx_hash - END - ) AS mints, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - nft_data - GROUP BY - 1, - 2 - ORDER BY - 3 DESC diff --git a/models/silver/atlas/silver__atlas_nft_table.yml b/models/silver/atlas/silver__atlas_nft_table.yml deleted file mode 100644 index 68c28f4..0000000 --- a/models/silver/atlas/silver__atlas_nft_table.yml +++ /dev/null @@ -1,66 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_table - description: |- - This view model provides a breakdown of NFT transaction activities by receiver_id. It includes counts of unique tokens, transfers within the last 24 hours and 3 days, all transfers, unique owners, total transactions, and minting events. - - columns: - - name: atlas_nft_table_id - description: "{ { doc('id')}}" - tests: - - unique - - not_null - - - name: receiver_id - description: "{ { doc('receiver_id')}}" - tests: - - not_null - - - name: tokens - description: "The count of unique tokens that have been received by the receiver_id." - tests: - - not_null - - - name: transfers_24h - description: "The count of 'nft_transfer' transactions that occurred in the last 24 hours." - tests: - - not_null - - - name: transfers_3d - description: "The count of 'nft_transfer' transactions that occurred in the last 3 days." - tests: - - not_null - - - name: all_transfers - description: "The total count of 'nft_transfer' transactions." - tests: - - not_null - - - name: owners - description: "The count of distinct owners that have interacted with the receiver's tokens." - tests: - - not_null - - - name: transactions - description: "{{ doc('tx_count')}}" - tests: - - not_null - - - name: mints - description: "The count of transactions where the method_name indicates a minting event rather than a transfer." - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/atlas/silver__atlas_nft_transactions.sql b/models/silver/atlas/silver__atlas_nft_transactions.sql deleted file mode 100644 index e69ada6..0000000 --- a/models/silver/atlas/silver__atlas_nft_transactions.sql +++ /dev/null @@ -1,107 +0,0 @@ --- TODO slated for deprecation and drop - -{{ config( - materialized = "incremental", - cluster_by = ["day"], - unique_key = "atlas_nft_transactions_id", - merge_exclude_columns = ["inserted_timestamp"], - incremental_strategy = "merge", - tags = ['atlas'] -) }} - -WITH nft_mints AS ( - - SELECT - block_timestamp :: DATE AS DAY, - receipt_object_id, - tx_hash, - method_name, - receiver_id, - signer_id, - owner_id AS owner, - token_id, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__standard_nft_mint_s3') }} - WHERE - {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} - {% endif %} -), -nft_transfers AS ( - SELECT - block_timestamp :: DATE AS DAY, - SPLIT( - action_id, - '-' - ) [0] :: STRING AS receipt_object_id, - tx_hash, - method_name, - receiver_id, - signer_id, - args ['receiver_id'] AS owner, - args ['token_id'] AS token_id, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__actions_events_function_call_s3') }} - WHERE - method_name = 'nft_transfer' - AND {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} - {% endif %} -), -unioned_nft_data AS ( - SELECT - * - FROM - nft_mints - UNION ALL - SELECT - * - FROM - nft_transfers -) -SELECT - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id', 'method_name', 'token_id', 'owner'] - ) }} AS atlas_nft_transactions_id, - DAY, - tx_hash, - method_name, - receiver_id, - signer_id, - owner, - token_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id, - _partition_by_block_number, - _inserted_timestamp, - _modified_timestamp -FROM - unioned_nft_data -WHERE - -- failed receipts may have unparsable base64 FunctionCall args - token_id IS NOT NULL - AND owner IS NOT NULL -qualify ROW_NUMBER() over ( - PARTITION BY atlas_nft_transactions_id - ORDER BY - _inserted_timestamp DESC -) = 1 diff --git a/models/silver/atlas/silver__atlas_nft_transactions.yml b/models/silver/atlas/silver__atlas_nft_transactions.yml deleted file mode 100644 index 620c8a6..0000000 --- a/models/silver/atlas/silver__atlas_nft_transactions.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: 2 - -models: - - name: silver__atlas_nft_transactions - description: |- - This incremental dbt model unifies NFT minting and transfer data into a single view, providing a comprehensive look at NFT activities. It captures daily activities by transaction hash, method name, receiver ID, signer ID, owner, and token ID. - tests: - - dbt_utils.recency: - datepart: day - field: _inserted_timestamp - interval: 1 - - columns: - - name: atlas_nft_transactions_id - description: "{{doc('id')}}" - tests: - - unique - - not_null - - - name: day - description: "{{doc('date')}}" - tests: - - not_null - - - name: tx_hash - description: "{{doc('tx_hash')}}" - tests: - - not_null - - - name: method_name - description: "{{doc('method_name')}}" - tests: - - not_null - - - name: receiver_id - description: "{{doc('receiver_id')}}" - tests: - - not_null - - - name: signer_id - description: "{{doc('signer_id')}}" - tests: - - not_null - - - name: owner - description: "{{doc('owner')}}" - tests: - - not_null - - - name: token_id - description: "{{doc('token_id')}}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - tests: - - not_null - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - tests: - - not_null - - - name: _invocation_id - description: "{{doc('invocation_id')}}" - - - name: _partition_by_block_number - description: "{{doc('_partition_by_block_number')}}" - - - name: _inserted_timestamp - description: "{{doc('_inserted_timestamp')}}" - tests: - - not_null - - - name: _modified_timestamp - description: "{{doc('_modified_timestamp')}}" \ No newline at end of file diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql index 68cc9e2..7832e92 100644 --- a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = "block_timestamp_hour", cluster_by = ['block_timestamp_hour::DATE'], - tags = ['curated','scheduled_non_core'] + tags = ['stats','scheduled_non_core'] ) }} /* run incremental timestamp value first then use it as a static value */ {% if execute %} @@ -14,11 +14,11 @@ SELECT MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour FROM - {{ ref('silver__streamline_blocks') }} -- Streamline Migration TODO - change this to fact blocks once table + {{ ref('silver__blocks_final') }} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endset %} @@ -35,7 +35,7 @@ SELECT COUNT( 1 ) AS block_count, - MAX(_inserted_timestamp) AS _inserted_timestamp, + MAX(inserted_timestamp) AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_timestamp_hour'] ) }} AS core_metrics_block_hourly_id, @@ -43,7 +43,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_final') }} WHERE block_timestamp_hour < DATE_TRUNC( 'hour', diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql index 66e1329..e5372be 100644 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = "block_timestamp_hour", cluster_by = ['block_timestamp_hour::DATE'], - tags = ['curated','scheduled_non_core'] + tags = ['stats','scheduled_non_core'] ) }} /* run incremental timestamp value first then use it as a static value */ {% if execute %} @@ -14,11 +14,11 @@ SELECT MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour FROM - {{ ref('silver__streamline_transactions_final') }} -- Streamline Migration TODO - change this to fact transactions once table + {{ ref('silver__transactions_final') }} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endset %} @@ -50,7 +50,7 @@ SELECT DISTINCT tx_receiver ) AS unique_to_count, SUM(transaction_fee / pow(10, 24)) AS total_fees, - MAX(_inserted_timestamp) AS _inserted_timestamp, + MAX(inserted_timestamp) AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['block_timestamp_hour'] ) }} AS core_metrics_hourly_id, @@ -58,7 +58,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__streamline_transactions_final') }} -- Streamline Migration TODO - change this to fact transactions once table + {{ ref('silver__transactions_final') }} WHERE block_timestamp_hour < DATE_TRUNC( 'hour', diff --git a/tests/gaps/core/tests__receipt_migration_gaps.sql b/tests/gaps/core/tests__receipt_migration_gaps.sql index cb67573..abd9d80 100644 --- a/tests/gaps/core/tests__receipt_migration_gaps.sql +++ b/tests/gaps/core/tests__receipt_migration_gaps.sql @@ -7,9 +7,10 @@ WITH archive AS ( select floor(block_id, -6) as block_group, - count(distinct receipt_id) as receipt_count, + count(distinct coalesce(receipt_object_id, receipt_id)) as receipt_count, count(distinct block_id) as block_count - from {{ ref('silver__streamline_receipts') }} + from {{ ref('silver__streamline_receipts_final') }} + where _partition_by_block_number >= 116000000 group by 1 ), destination AS ( @@ -18,6 +19,7 @@ destination AS ( count(distinct receipt_id) as receipt_count, count(distinct block_id) as block_count from {{ ref('silver__receipts_final') }} + where _partition_by_block_number >= 116000000 group by 1 ) select @@ -29,5 +31,4 @@ select from archive left join destination on archive.block_group = destination.block_group --- where receipt_ct_expected <> receipt_ct_actual order by block_group From 6497f42a7db7b1317f9263334c1df5ebf185647a Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:52:02 -0600 Subject: [PATCH 32/53] logs and drop atlas gold --- .../atlas/atlas__ez_nft_contract_metrics.sql | 34 ----- .../atlas/atlas__ez_nft_contract_metrics.yml | 99 -------------- .../atlas__ez_nft_contract_metrics_daily.sql | 33 ----- .../atlas__ez_nft_contract_metrics_daily.yml | 86 ------------ .../atlas/atlas__fact_accounts_created.sql | 21 --- .../atlas/atlas__fact_accounts_created.yml | 44 ------- models/gold/atlas/atlas__fact_maas.sql | 22 ---- models/gold/atlas/atlas__fact_maas.yml | 48 ------- .../atlas/atlas__fact_nft_monthly_txs.sql | 28 ---- .../atlas/atlas__fact_nft_monthly_txs.yml | 41 ------ models/gold/core/core__fact_logs.sql | 3 +- .../silver__actions_events_addkey_s3.sql | 62 --------- .../silver__actions_events_addkey_s3.yml | 55 -------- ...ilver__actions_events_function_call_s3.yml | 73 ----------- .../actions/silver__actions_events_s3.yml | 82 ------------ models/silver/curated/silver__logs_s3.sql | 22 ++-- .../silver_horizon__decoded_actions.sql | 124 ------------------ .../silver_horizon__decoded_actions.yml | 80 ----------- .../horizon/silver_horizon__receipts.sql | 64 --------- .../horizon/silver_horizon__receipts.yml | 81 ------------ 20 files changed, 11 insertions(+), 1091 deletions(-) delete mode 100644 models/gold/atlas/atlas__ez_nft_contract_metrics.sql delete mode 100644 models/gold/atlas/atlas__ez_nft_contract_metrics.yml delete mode 100644 models/gold/atlas/atlas__ez_nft_contract_metrics_daily.sql delete mode 100644 models/gold/atlas/atlas__ez_nft_contract_metrics_daily.yml delete mode 100644 models/gold/atlas/atlas__fact_accounts_created.sql delete mode 100644 models/gold/atlas/atlas__fact_accounts_created.yml delete mode 100644 models/gold/atlas/atlas__fact_maas.sql delete mode 100644 models/gold/atlas/atlas__fact_maas.yml delete mode 100644 models/gold/atlas/atlas__fact_nft_monthly_txs.sql delete mode 100644 models/gold/atlas/atlas__fact_nft_monthly_txs.yml delete mode 100644 models/silver/actions/silver__actions_events_addkey_s3.sql delete mode 100644 models/silver/actions/silver__actions_events_addkey_s3.yml delete mode 100644 models/silver/actions/silver__actions_events_function_call_s3.yml delete mode 100644 models/silver/actions/silver__actions_events_s3.yml delete mode 100644 models/silver/horizon/silver_horizon__decoded_actions.sql delete mode 100644 models/silver/horizon/silver_horizon__decoded_actions.yml delete mode 100644 models/silver/horizon/silver_horizon__receipts.sql delete mode 100644 models/silver/horizon/silver_horizon__receipts.yml diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics.sql b/models/gold/atlas/atlas__ez_nft_contract_metrics.sql deleted file mode 100644 index 8212c37..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -WITH nft_data AS ( - - SELECT - atlas_nft_table_id AS ez_nft_contract_metrics_id, - receiver_id, - tokens, - transfers_24h, - transfers_3d, - all_transfers, - owners, - transactions, - mints, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver__atlas_nft_table') }} -) -SELECT - * -FROM - nft_data diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics.yml b/models/gold/atlas/atlas__ez_nft_contract_metrics.yml deleted file mode 100644 index 33ebde6..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics.yml +++ /dev/null @@ -1,99 +0,0 @@ -version: 2 - -models: - - name: atlas__ez_nft_contract_metrics - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: EZ_NFT_CONTRACT_METRICS_ID - description: "{ { doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: RECEIVER_ID - description: "{ { doc('receiver_id')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TOKENS - description: "{{ doc('tokens_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSFERS_24H - description: "The count of 'nft_transfer' transactions that occurred in the last 24 hours." - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSFERS_3D - description: "The count of 'nft_transfer' transactions that occurred in the last 3 days." - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: ALL_TRANSFERS - description: "{{ doc('all_transfers')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: OWNERS - description: "{{ doc('owner_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSACTIONS - description: "{{ doc('tx_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: MINTS - description: "{{ doc('mint_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.sql b/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.sql deleted file mode 100644 index ca74d83..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.sql +++ /dev/null @@ -1,33 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -WITH nft_detailed AS ( - - SELECT - atlas_nft_detailed_id AS ez_nft_contract_metrics_daily_id, - DAY, - receiver_id, - tokens, - all_transfers, - owners, - transactions, - mints, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver__atlas_nft_detailed') }} -) -SELECT - * -FROM - nft_detailed diff --git a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.yml b/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.yml deleted file mode 100644 index 309344e..0000000 --- a/models/gold/atlas/atlas__ez_nft_contract_metrics_daily.yml +++ /dev/null @@ -1,86 +0,0 @@ -version: 2 - -models: - - name: atlas__ez_nft_contract_metrics_daily - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - -columns: - - name: EZ_NFT_CONTRACT_METRICS_DAILY_ID - description: "{{ doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: DAY - description: "{{ doc('date')}}" - tests: - - not_null - - - name: RECEIVER_ID - description: "{{ doc('tx_receiver')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TOKENS - description: "{{ doc('tokens_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: ALL_TRANSFERS - description: "{{ doc('all_transfers')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: OWNERS - description: "{{ doc('owner_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: TRANSACTIONS - description: "{{ doc('tx_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: MINTS - description: "{{ doc('mint_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/atlas/atlas__fact_accounts_created.sql b/models/gold/atlas/atlas__fact_accounts_created.sql deleted file mode 100644 index d759523..0000000 --- a/models/gold/atlas/atlas__fact_accounts_created.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -SELECT - atlas_account_created_id AS fact_accounts_created_id, - DAY, - wallets_created, - inserted_timestamp, - modified_timestamp -FROM - {{ ref('silver__atlas_accounts_created') }} diff --git a/models/gold/atlas/atlas__fact_accounts_created.yml b/models/gold/atlas/atlas__fact_accounts_created.yml deleted file mode 100644 index 44abe6f..0000000 --- a/models/gold/atlas/atlas__fact_accounts_created.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: 2 - -models: - - name: atlas__fact_accounts_created - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: FACT_ACCOUNTS_CREATED_ID - description: "{ { doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: DAY - description: "{{ doc('date')}}" - tests: - - not_null - - - name: WALLETS_CREATED - description: "{{ doc('wallets_created')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/gold/atlas/atlas__fact_maas.sql b/models/gold/atlas/atlas__fact_maas.sql deleted file mode 100644 index 717dc4a..0000000 --- a/models/gold/atlas/atlas__fact_maas.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - materialized = 'view', - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -SELECT - atlas_maa_id AS fact_maas_id, - day, - maa, - new_maas, - returning_maas, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp -FROM - {{ ref('silver__atlas_maa') }} diff --git a/models/gold/atlas/atlas__fact_maas.yml b/models/gold/atlas/atlas__fact_maas.yml deleted file mode 100644 index fe2f089..0000000 --- a/models/gold/atlas/atlas__fact_maas.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: 2 - -models: - - name: atlas__fact_maas - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: fact_maas_id - description: "{{ doc('id') }}" - tests: - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: day - description: "{{ doc('active_day') }}" - tests: - - not_null - - unique - - - name: maa - description: "{{ doc('maa')}}" - tests: - - not_null - - - name: new_maas - description: "{{ doc('new_maas') }}" - tests: - - not_null - - - name: returning_maas - description: "{{ doc('returning_maas') }}" - tests: - - not_null - - - name: inserted_timestamp - description: "{{ doc('inserted_timestamp') }}" - - - name: modified_timestamp - description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/atlas/atlas__fact_nft_monthly_txs.sql b/models/gold/atlas/atlas__fact_nft_monthly_txs.sql deleted file mode 100644 index 00413e9..0000000 --- a/models/gold/atlas/atlas__fact_nft_monthly_txs.sql +++ /dev/null @@ -1,28 +0,0 @@ -{{ config( - materialized = 'view', - secure = false, - meta={ - 'database_tags':{ - 'table': { - 'PURPOSE': 'ATLAS' - } - } - }, - tags = ['atlas'] -) }} - -WITH TRAILING AS ( - - SELECT - atlas_nft_30_trailing_id AS fact_nft_monthly_txs_id, - DAY, - txns, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp - FROM - {{ ref('silver__atlas_nft_30_trailing') }} -) -SELECT - * -FROM - TRAILING diff --git a/models/gold/atlas/atlas__fact_nft_monthly_txs.yml b/models/gold/atlas/atlas__fact_nft_monthly_txs.yml deleted file mode 100644 index 9b1e83a..0000000 --- a/models/gold/atlas/atlas__fact_nft_monthly_txs.yml +++ /dev/null @@ -1,41 +0,0 @@ -version: 2 - -models: - - name: atlas__fact_nft_monthly_txs - description: |- - Deprecating Soon - these tables previously supported NEAR Atlas which is no longer live. These tables will be removed on Febreary 1, 2025. - Please cease using these tables. - tests: - - dbt_utils.recency: - datepart: days - field: inserted_timestamp - interval: 1 - - columns: - - name: FACT_NFT_MONTHLY_TXS_ID - description: "{{ doc('id')}}" - tests: - - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' - - - name: DAY - description: "{{ doc('date')}}" - tests: - - not_null - - - name: TXNS - description: "{{ doc('tx_count')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index a1fb883..87bc254 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -15,7 +15,7 @@ SELECT block_id, block_timestamp, tx_hash, - receipt_object_id AS receipt_id, + COALESCE(receipt_id, receipt_object_id) AS receipt_id, predecessor_id, receiver_id, signer_id, @@ -30,7 +30,6 @@ SELECT ['log_id'] ) }} ) AS fact_logs_id, - receipt_object_id, -- will drop col eventually COALESCE(inserted_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, COALESCE(modified_timestamp, _inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp FROM diff --git a/models/silver/actions/silver__actions_events_addkey_s3.sql b/models/silver/actions/silver__actions_events_addkey_s3.sql deleted file mode 100644 index 7dd8090..0000000 --- a/models/silver/actions/silver__actions_events_addkey_s3.sql +++ /dev/null @@ -1,62 +0,0 @@ --- TODO slated for deprecation and drop --- Note - a model in Social does use this -{{ config( - materialized = 'incremental', - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'action_id', - cluster_by = ['block_timestamp::DATE', 'modified_timestamp::DATE'], - tags = ['actions', 'curated','scheduled_non_core'] -) }} - -{# NOTE - used downstream in Social models, no longer a gold view on just this #} --- todo deprecate this model - -WITH action_events AS ( - - SELECT - action_id, - tx_hash, - block_id, - block_timestamp, - action_data, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__actions_events_s3') }} - WHERE - action_name = 'AddKey' - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - AND {{ incremental_load_filter('modified_timestamp') }} - {% endif %} -), -addkey_events AS ( - SELECT - action_id, - tx_hash, - block_id, - block_timestamp, - action_data :access_key :nonce :: NUMBER AS nonce, - action_data :public_key :: STRING AS public_key, - action_data :access_key :permission AS permission, - action_data :access_key :permission :FunctionCall :allowance :: FLOAT AS allowance, - action_data :access_key :permission :FunctionCall :method_names :: ARRAY AS method_name, - action_data :access_key :permission :FunctionCall :receiver_id :: STRING AS receiver_id, - _partition_by_block_number, - _inserted_timestamp - FROM - action_events -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['action_id'] - ) }} AS actions_events_addkey_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - addkey_events diff --git a/models/silver/actions/silver__actions_events_addkey_s3.yml b/models/silver/actions/silver__actions_events_addkey_s3.yml deleted file mode 100644 index aadc86c..0000000 --- a/models/silver/actions/silver__actions_events_addkey_s3.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: 2 - -models: - - name: silver__actions_events_addkey_s3 - description: |- - Deprecting soon - no longer updating. - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - - - name: NONCE - description: "{{ doc('nonce')}}" - - - name: PUBLIC_KEY - description: "{{ doc('public_key')}}" - - - name: PERMISSION - description: "{{ doc('permission')}}" - - - name: ALLOWANCE - description: "{{ doc('allowance')}}" - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: actions_events_addkey_id - description: "{{doc('id')}}" - - - name: inserted_timestamp - description: "{{doc('inserted_timestamp')}}" - - - name: modified_timestamp - description: "{{doc('modified_timestamp')}}" - - - name: _invocation_id - description: "{{doc('invocation_id')}}" diff --git a/models/silver/actions/silver__actions_events_function_call_s3.yml b/models/silver/actions/silver__actions_events_function_call_s3.yml deleted file mode 100644 index 382c489..0000000 --- a/models/silver/actions/silver__actions_events_function_call_s3.yml +++ /dev/null @@ -1,73 +0,0 @@ -version: 2 - -models: - - name: silver__actions_events_function_call_s3 - description: |- - This table extracts all FunctionCall events from actions and decodes the arguments for easy use. - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - - - name: METHOD_NAME - description: "{{ doc('method_name')}}" - - - name: ARGS - description: "{{ doc('args')}}" - - - name: DEPOSIT - description: "{{ doc('deposit')}}" - - - name: ATTACHED_GAS - description: "{{ doc('attached_gas')}}" - - - name: LOGS - description: "{{ doc('logs')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: ACTIONS_EVENTS_FUNCTION_CALL_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/actions/silver__actions_events_s3.yml b/models/silver/actions/silver__actions_events_s3.yml deleted file mode 100644 index f03d4a4..0000000 --- a/models/silver/actions/silver__actions_events_s3.yml +++ /dev/null @@ -1,82 +0,0 @@ -version: 2 - -models: - - name: silver__actions_events_s3 - description: |- - This table extracts all action events from a receipt and stores the argument data under action_data. - - columns: - - name: ACTION_ID - description: "{{ doc('action_id')}}" - tests: - - unique: - where: tx_hash != 'J4CZZQrZK6kYPVLkrdbTEpcqhUNZiRxktbMzHviqeGgf' - - not_null - - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: RECEIPT_OBJECT_ID - description: "{{ doc('receipt_object_id')}}" - - - name: CHUNK_HASH - description: "{{ doc('chunk_hash')}}" - tests: - - not_null: - where: "block_id not in (34691244, 34691277)" - - - name: RECEIVER_ID - description: "{{ doc('receiver_id')}}" - - - name: PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - - - name: SIGNER_ID - description: "{{ doc('signer_id')}}" - - - name: ACTION_INDEX - description: "{{ doc('action_index')}}" - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - - - name: ACTION_DATA - description: "{{ doc('action_data')}}" - - - name: LOGS - description: "{{ doc('logs')}}" - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - - - name: ACTIONS_EVENTS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/curated/silver__logs_s3.sql b/models/silver/curated/silver__logs_s3.sql index 3fae89a..ee16a0e 100644 --- a/models/silver/curated/silver__logs_s3.sql +++ b/models/silver/curated/silver__logs_s3.sql @@ -15,17 +15,16 @@ WITH receipts AS ( block_id, block_timestamp, tx_hash, - receipt_object_id, - logs, + receipt_id, + outcome_json :outcome :logs AS logs, receiver_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, -- TODO once exists in receipts final can select directly + predecessor_id, signer_id, - gas_burnt, + outcome_json :outcome :gas_burnt AS gas_burnt, receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} @@ -45,16 +44,15 @@ FINAL AS ( block_id, block_timestamp, tx_hash, - receipt_object_id, + receipt_id, concat_ws( '-', - receipt_object_id, + receipt_id, INDEX ) AS log_id, INDEX AS log_index, receiver_id, predecessor_id, - signer_id, COALESCE( TRY_PARSE_JSON(VALUE), TRY_PARSE_JSON(SPLIT(VALUE, 'EVENT_JSON:') [1]), @@ -63,8 +61,7 @@ FINAL AS ( VALUE ILIKE 'event_json:%' AS is_standard, gas_burnt, receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM receipts, LATERAL FLATTEN( @@ -73,6 +70,7 @@ FINAL AS ( ) SELECT *, + receipt_id AS receipt_object_id, -- maintain for a run but then need to copy values over and then drop {{ dbt_utils.generate_surrogate_key( ['log_id'] ) }} AS logs_id, diff --git a/models/silver/horizon/silver_horizon__decoded_actions.sql b/models/silver/horizon/silver_horizon__decoded_actions.sql deleted file mode 100644 index 13f1d5d..0000000 --- a/models/silver/horizon/silver_horizon__decoded_actions.sql +++ /dev/null @@ -1,124 +0,0 @@ -{{ config( - materialized = 'incremental', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'action_id_horizon', - cluster_by = ['_inserted_timestamp::date', '_partition_by_block_number'], - tags = ['curated', 'horizon','scheduled_non_core'] -) }} -{# Note - multisource model #} --- TODO ez_actions refactor - -WITH all_horizon_receipts AS ( - - SELECT - tx_hash, - receipt_object_id, - receiver_id, - signer_id, - receipt_succeeded, - logs, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver_horizon__receipts') }} - - {% if var("MANUAL_FIX") %} - WHERE {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -decoded_function_calls AS ( - SELECT - SPLIT( - action_id, - '-' - ) [0] :: STRING AS receipt_object_id, - action_id, - tx_hash, - block_id, - block_timestamp, - method_name, - args, - deposit, - attached_gas, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__actions_events_function_call_s3') }} - WHERE - _partition_by_block_number >= 85000000 - AND SPLIT( - action_id, - '-' - ) [0] :: STRING IN ( - SELECT - DISTINCT receipt_object_id - FROM - all_horizon_receipts - ) - - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -FINAL AS ( - SELECT - fc.action_id, - fc.tx_hash, - r.receipt_object_id, - fc.block_id, - fc.block_timestamp, - fc.method_name, - fc.args, - fc.deposit, - fc.attached_gas, - r.receiver_id, - r.signer_id, - r.receipt_succeeded, - r.logs, - fc._partition_by_block_number, - fc._inserted_timestamp - FROM - decoded_function_calls fc - LEFT JOIN all_horizon_receipts r USING (receipt_object_id) -) -SELECT - action_id AS action_id_horizon, - tx_hash, - receipt_object_id, - block_id, - block_timestamp, - method_name, - args, - deposit, - attached_gas, - receiver_id, - signer_id, - receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['action_id_horizon'] - ) }} AS horizon_decoded_actions_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/horizon/silver_horizon__decoded_actions.yml b/models/silver/horizon/silver_horizon__decoded_actions.yml deleted file mode 100644 index 3a56304..0000000 --- a/models/silver/horizon/silver_horizon__decoded_actions.yml +++ /dev/null @@ -1,80 +0,0 @@ - -version: 2 - -models: - - name: silver_horizon__decoded_actions - description: |- - Decoded FunctionCall events for receipts where the contract nearhorizon.near was called. - - columns: - - name: action_id_horizon - description: "{{ doc('action_id')}}" - tests: - - unique - - - name: tx_hash - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: receipt_object_id - description: "{{ doc('receipt_object_id')}}" - - - name: block_id - description: "{{ doc('block_id')}}" - - - name: block_timestamp - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: method_name - description: "{{ doc('method_name')}}" - tests: - - not_null - - - name: args - description: "{{ doc('args')}}" - tests: - - not_null - - - name: deposit - description: "{{ doc('deposit')}}" - - - name: attached_gas - description: "{{ doc('attached_gas')}}" - - - name: receiver_id - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - - name: signer_id - description: "{{ doc('signer_id')}}" - tests: - - not_null - - - name: receipt_succeeded - description: "{{ doc('receipt_succeeded')}}" - - - name: _partition_by_block_number - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - - - name: HORIZON_DECODED_ACTIONS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/horizon/silver_horizon__receipts.sql b/models/silver/horizon/silver_horizon__receipts.sql deleted file mode 100644 index f7af0b5..0000000 --- a/models/silver/horizon/silver_horizon__receipts.sql +++ /dev/null @@ -1,64 +0,0 @@ -{{ config( - materialized = 'incremental', - merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'receipt_object_id', - cluster_by = ['_inserted_timestamp::date', 'block_timestamp::DATE'], - tags = ['curated', 'horizon','scheduled_non_core'] -) }} - -WITH all_horizon_receipts AS ( - - SELECT - tx_hash, - receipt_object_id, - block_id, - block_timestamp, - receipt_index, - chunk_hash, - receipt_actions, - execution_outcome, - receipt_outcome_id, - receiver_id, - signer_id, - receipt_type, - gas_burnt, - status_value, - receipt_succeeded, - logs, - proof, - metadata, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_receipts_final') }} - - WHERE ( - LOWER(signer_id) = 'nearhorizon.near' - OR LOWER(receiver_id) = 'nearhorizon.near' - ) - AND _partition_by_block_number >= 86000000 - - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -) - - SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['receipt_object_id'] - ) }} AS horizon_receipts_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - all_horizon_receipts diff --git a/models/silver/horizon/silver_horizon__receipts.yml b/models/silver/horizon/silver_horizon__receipts.yml deleted file mode 100644 index be74f69..0000000 --- a/models/silver/horizon/silver_horizon__receipts.yml +++ /dev/null @@ -1,81 +0,0 @@ - -version: 2 - -models: - - name: silver_horizon__receipts - description: |- - Filtered receipts where the signer or receiver is the contract nearsocial.near. - - columns: - - name: tx_hash - description: "{{ doc('tx_hash')}}" - - - name: block_id - description: "{{ doc('block_id')}}" - - - name: receipt_index - description: "{{ doc('receipt_index')}}" - - - name: chunk_hash - description: "{{ doc('chunk_hash')}}" - - - name: receipt_actions - description: "{{ doc('receipt')}}" - - - name: execution_outcome - description: "{{ doc('execution_outcome')}}" - - - name: receipt_object_id - description: "{{ doc('receipt_object_id')}}" - tests: - - unique - - - name: receipt_outcome_id - description: "{{ doc('receipt_outcome_id')}}" - - - name: receiver_id - description: "{{ doc('receiver_id')}}" - - - name: signer_id - description: "{{ doc('signer_id')}}" - - - name: receipt_type - description: "{{ doc('receipt_type')}}" - - - name: gas_burnt - description: "{{ doc('gas_burnt')}}" - - - name: status_value - description: "{{ doc('status_value')}}" - - - name: receipt_succeeded - description: "{{ doc('receipt_succeeded')}}" - - - name: logs - description: "{{ doc('logs')}}" - - - name: proof - description: "{{ doc('proof')}}" - - - name: metadata - description: "{{ doc('metadata')}}" - - - name: _partition_by_block_number - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - - - name: HORIZON_RECEIPTS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" From ccec7e7ec56f60b98e1bde326ba91040c388e041 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:13:36 -0600 Subject: [PATCH 33/53] retain legacy actions FOR NOW --- models/silver/actions/README.md | 4 ++ .../silver__actions_events_addkey_s3.sql | 62 +++++++++++++++++++ .../actions/silver__actions_events_s3.sql | 25 ++++---- 3 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 models/silver/actions/README.md create mode 100644 models/silver/actions/silver__actions_events_addkey_s3.sql diff --git a/models/silver/actions/README.md b/models/silver/actions/README.md new file mode 100644 index 0000000..4913a3a --- /dev/null +++ b/models/silver/actions/README.md @@ -0,0 +1,4 @@ +# Deprecating Actions Models + +All 3 of these models will be deprecated and dropped. +There are a number of silver models that need to be migrated to core.ez_actions. Migrate those by EOM (March 2025). diff --git a/models/silver/actions/silver__actions_events_addkey_s3.sql b/models/silver/actions/silver__actions_events_addkey_s3.sql new file mode 100644 index 0000000..7dd8090 --- /dev/null +++ b/models/silver/actions/silver__actions_events_addkey_s3.sql @@ -0,0 +1,62 @@ +-- TODO slated for deprecation and drop +-- Note - a model in Social does use this +{{ config( + materialized = 'incremental', + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'action_id', + cluster_by = ['block_timestamp::DATE', 'modified_timestamp::DATE'], + tags = ['actions', 'curated','scheduled_non_core'] +) }} + +{# NOTE - used downstream in Social models, no longer a gold view on just this #} +-- todo deprecate this model + +WITH action_events AS ( + + SELECT + action_id, + tx_hash, + block_id, + block_timestamp, + action_data, + _partition_by_block_number, + _inserted_timestamp + FROM + {{ ref('silver__actions_events_s3') }} + WHERE + action_name = 'AddKey' + {% if var("MANUAL_FIX") %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + AND {{ incremental_load_filter('modified_timestamp') }} + {% endif %} +), +addkey_events AS ( + SELECT + action_id, + tx_hash, + block_id, + block_timestamp, + action_data :access_key :nonce :: NUMBER AS nonce, + action_data :public_key :: STRING AS public_key, + action_data :access_key :permission AS permission, + action_data :access_key :permission :FunctionCall :allowance :: FLOAT AS allowance, + action_data :access_key :permission :FunctionCall :method_names :: ARRAY AS method_name, + action_data :access_key :permission :FunctionCall :receiver_id :: STRING AS receiver_id, + _partition_by_block_number, + _inserted_timestamp + FROM + action_events +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['action_id'] + ) }} AS actions_events_addkey_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + addkey_events diff --git a/models/silver/actions/silver__actions_events_s3.sql b/models/silver/actions/silver__actions_events_s3.sql index f939946..b77dd26 100644 --- a/models/silver/actions/silver__actions_events_s3.sql +++ b/models/silver/actions/silver__actions_events_s3.sql @@ -15,21 +15,20 @@ WITH receipts AS ( SELECT tx_hash, - receipt_object_id, + receipt_id AS receipt_object_id, receiver_id, - signer_id, + predecessor_id AS signer_id, block_id, block_timestamp, chunk_hash, - logs, - receipt_actions, - execution_outcome, + outcome_json :outcome :logs :: ARRAY AS logs, + receipt_json AS receipt_actions, + outcome_json AS execution_outcome, receipt_succeeded, - gas_burnt, - _partition_by_block_number, - _inserted_timestamp + outcome_json :outcome :gas_burnt :: NUMBER AS gas_burnt, + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} WHERE @@ -64,12 +63,11 @@ flatten_actions AS ( VALUE AS action_object, INDEX AS action_index, receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM receipts, LATERAL FLATTEN( - input => receipt_actions :receipt :Action :actions + input => receipt_actions :receipt :Action :actions :: ARRAY ) ), FINAL AS ( @@ -95,8 +93,7 @@ FINAL AS ( gas_price, gas_burnt, tokens_burnt, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM flatten_actions, LATERAL FLATTEN( From 3ee43bb5251f934b429e33422e83633f85ab0a23 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:31:31 -0600 Subject: [PATCH 34/53] silver curated models refactor --- .../actions/silver__actions_events_s3.sql | 2 +- .../curated/defi/silver__dex_swaps_s3.sql | 258 ------------------ .../curated/defi/silver__dex_swaps_s3.yml | 72 ----- .../curated/defi/silver__dex_swaps_v2.sql | 21 +- .../nft/sales/silver__nft_paras_sales.sql | 12 +- .../nft/silver__standard_nft_mint_s3.sql | 5 +- .../silver/curated/silver__lockup_actions.sql | 4 +- .../curated/staking/silver__pool_events.sql | 17 +- .../staking/silver__staking_pools_s3.sql | 21 +- 9 files changed, 33 insertions(+), 379 deletions(-) delete mode 100644 models/silver/curated/defi/silver__dex_swaps_s3.sql delete mode 100644 models/silver/curated/defi/silver__dex_swaps_s3.yml diff --git a/models/silver/actions/silver__actions_events_s3.sql b/models/silver/actions/silver__actions_events_s3.sql index b77dd26..cf3a903 100644 --- a/models/silver/actions/silver__actions_events_s3.sql +++ b/models/silver/actions/silver__actions_events_s3.sql @@ -17,7 +17,7 @@ WITH receipts AS ( tx_hash, receipt_id AS receipt_object_id, receiver_id, - predecessor_id AS signer_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, block_id, block_timestamp, chunk_hash, diff --git a/models/silver/curated/defi/silver__dex_swaps_s3.sql b/models/silver/curated/defi/silver__dex_swaps_s3.sql deleted file mode 100644 index 640d4ac..0000000 --- a/models/silver/curated/defi/silver__dex_swaps_s3.sql +++ /dev/null @@ -1,258 +0,0 @@ -{{ config( - materialized = "incremental", - unique_key = "swap_id", - incremental_strategy = "merge", - merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ["block_timestamp::DATE"], - tags = ['curated','scheduled_non_core'], - enabled = False -) }} -{# DEPRECATED JANUARY 2024 #} -WITH base_swap_calls AS ( - - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - args, - _inserted_timestamp, - method_name - FROM - {{ ref('silver__actions_events_function_call_s3') }} - WHERE - method_name IN ( - 'swap', - 'ft_transfer_call' - ) {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - AND {{ incremental_load_filter('_inserted_timestamp') }} - {% endif %} -), -base_swaps AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - action_id, - IFF( - method_name = 'ft_transfer_call', - TRY_PARSE_JSON(TRY_PARSE_JSON(args) :msg), - TRY_PARSE_JSON(args) - ) :actions AS actions, - _inserted_timestamp - FROM - base_swap_calls -), -agg_swaps AS ( - SELECT - tx_hash, - ANY_VALUE(block_id) AS block_id, - ANY_VALUE(block_timestamp) AS block_timestamp, - ARRAY_AGG( - action.value - ) within GROUP ( - ORDER BY - action_id, - action.index - ) AS action_list, - ANY_VALUE(_inserted_timestamp) AS _inserted_timestamp - FROM - base_swaps, - LATERAL FLATTEN( - input => actions - ) action - GROUP BY - 1 -), -actions AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - NULLIF( - action.value :pool_id, - NULL - ) AS pool_id, - NULLIF( - action.value :token_in, - NULL - ) :: text AS token_in, - NULLIF( - action.value :token_out, - NULL - ) :: text AS token_out, - action.index AS swap_index, - _inserted_timestamp - FROM - agg_swaps, - LATERAL FLATTEN( - input => action_list - ) action - WHERE - NOT RLIKE( - pool_id, - '.*[a-z].*', - 'i' - ) -), -receipts AS ( - SELECT - block_id, - tx_hash, - -- TODO use the receipt succeeded column here - CASE - WHEN PARSE_JSON( - r.status_value - ) :Failure IS NOT NULL THEN 'Fail' - ELSE 'Success' - END AS success_or_fail, - logs - FROM - {{ ref("silver__streamline_receipts_final") }} - r - WHERE - tx_hash IN ( - SELECT - tx_hash - FROM - actions - ) -), -flat_receipts AS ( - SELECT - tx_hash, - l.value, - l.index, - success_or_fail - FROM - receipts, - LATERAL FLATTEN( - input => logs - ) l -), -swap_logs AS ( - SELECT - tx_hash, - ROW_NUMBER() over ( - PARTITION BY tx_hash - ORDER BY - INDEX ASC - ) - 1 AS swap_index, - VALUE, - success_or_fail - FROM - flat_receipts - WHERE - VALUE LIKE 'Swapped%' -), -transactions AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - tx_signer, - tx_receiver - FROM - {{ ref("silver__streamline_transactions_final") }} - WHERE - tx_hash IN ( - SELECT - tx_hash - FROM - actions - ) -), -token_labels AS ( - SELECT - * - FROM - {{ ref("silver__token_labels") }} -), -final_table AS ( - SELECT - swap_logs.swap_index, - actions._inserted_timestamp, - actions.block_id, - actions.block_timestamp, - swap_logs.tx_hash, - CONCAT( - swap_logs.tx_hash, - '-', - swap_logs.swap_index - ) AS swap_id, - swap_logs.value AS log_data, - transactions.tx_signer AS trader, - transactions.tx_receiver AS platform, - LAST_VALUE( - swap_logs.success_or_fail - ) over ( - PARTITION BY swap_logs.tx_hash - ORDER BY - swap_logs.success_or_fail DESC - ) AS txn_status, - actions.pool_id :: INT AS pool_id, - actions.token_in, - actions.token_out - FROM - actions - INNER JOIN swap_logs - ON ( - swap_logs.tx_hash = actions.tx_hash - AND swap_logs.swap_index = actions.swap_index - ) - JOIN transactions - ON actions.tx_hash = transactions.tx_hash -), -FINAL AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - swap_id, - platform, - trader, - pool_id, - token_in, - token_labels_in.symbol AS token_in_symbol, - REGEXP_SUBSTR( - log_data, - 'Swapped (\\d+)', - 1, - 1, - 'e' - ) :: NUMBER AS amount_in_raw, - amount_in_raw / pow(10, IFNULL(token_labels_in.decimals, 0)) AS amount_in, - token_out, - token_labels_out.symbol AS token_out_symbol, - REGEXP_SUBSTR( - log_data, - 'Swapped \\d+ .+ for (\\d+)', - 1, - 1, - 'e' - ) :: NUMBER AS amount_out_raw, - amount_out_raw / pow(10, IFNULL(token_labels_out.decimals, 0)) AS amount_out, - swap_index, - _inserted_timestamp - FROM - final_table - LEFT JOIN token_labels AS token_labels_in - ON final_table.token_in = token_labels_in.token_contract - LEFT JOIN token_labels AS token_labels_out - ON final_table.token_out = token_labels_out.token_contract - WHERE - txn_status = 'Success' - AND log_data IS NOT NULL -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['swap_id'] - ) }} AS dex_swaps_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/curated/defi/silver__dex_swaps_s3.yml b/models/silver/curated/defi/silver__dex_swaps_s3.yml deleted file mode 100644 index 950b17a..0000000 --- a/models/silver/curated/defi/silver__dex_swaps_s3.yml +++ /dev/null @@ -1,72 +0,0 @@ -version: 2 - -models: - - name: silver__dex_swaps_s3 - description: |- - This table records all the swap transactions occurring in NEAR. This model is being deprecated as of January 2024. It will remain live through February for users to migrate to the new model. - This logic is outdated / inaccurate. - - columns: - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null: - where: _inserted_timestamp <= CURRENT_TIMESTAMP - interval '1 hour' - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null: - where: _inserted_timestamp <= CURRENT_TIMESTAMP - interval '1 hour' - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: SWAP_ID - description: "{{ doc('swap_id')}}" - - - name: PLATFORM - description: "{{ doc('platform')}}" - - - name: TRADER - description: "{{ doc('trader')}}" - - - name: POOL_ID - description: "{{ doc('pool_id')}}" - - - name: TOKEN_IN - description: "{{ doc('token_in')}}" - - - name: AMOUNT_IN - description: "{{ doc('amount_in')}}" - - - name: TOKEN_OUT - description: "{{ doc('token_out')}}" - - - name: AMOUNT_OUT - description: "{{ doc('amount_out')}}" - - - name: SWAP_INDEX - description: "{{ doc('swap_index')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: DEX_SWAPS_ID - description: "{{doc('id')}}" - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" diff --git a/models/silver/curated/defi/silver__dex_swaps_v2.sql b/models/silver/curated/defi/silver__dex_swaps_v2.sql index 2fa7a1c..a46db8e 100644 --- a/models/silver/curated/defi/silver__dex_swaps_v2.sql +++ b/models/silver/curated/defi/silver__dex_swaps_v2.sql @@ -10,7 +10,7 @@ ) }} {# Note - multisource model #} -- depends on {{ ref('silver__logs_s3') }} --- depends on {{ ref('silver__streamline_receipts_final') }} +-- depends on {{ ref('silver__receipts_final') }} {% if execute %} @@ -49,7 +49,7 @@ SELECT MIN(block_timestamp) block_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} A + {{ ref('silver__receipts_final') }} A WHERE modified_timestamp >= '{{max_mod}}' ) @@ -78,7 +78,6 @@ WITH swap_logs AS ( log_index, clean_log, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM {{ ref('silver__logs_s3') }} @@ -97,15 +96,14 @@ WITH swap_logs AS ( ), receipts AS ( SELECT - receipt_object_id, - receipt_actions, + receipt_id AS receipt_object_id, + receipt_json AS receipt_actions, receiver_id, - signer_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE receipt_object_id IN ( SELECT @@ -157,7 +155,6 @@ swap_outcome AS ( '\\1' ) :: STRING AS token_out, _partition_by_block_number, - _inserted_timestamp, modified_timestamp FROM swap_logs @@ -212,8 +209,7 @@ parse_actions AS ( ) AS swap_input_data, r.receiver_id AS receipt_receiver_id, r.signer_id AS receipt_signer_id, - o._partition_by_block_number, - o._inserted_timestamp + o._partition_by_block_number FROM swap_outcome o LEFT JOIN receipts r USING (receipt_object_id) @@ -241,8 +237,7 @@ FINAL AS ( token_in, swap_input_data, LOG, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM parse_actions ) diff --git a/models/silver/curated/nft/sales/silver__nft_paras_sales.sql b/models/silver/curated/nft/sales/silver__nft_paras_sales.sql index 6d7a4d2..c4f7d43 100644 --- a/models/silver/curated/nft/sales/silver__nft_paras_sales.sql +++ b/models/silver/curated/nft/sales/silver__nft_paras_sales.sql @@ -53,15 +53,15 @@ WITH actions_events AS ( status_value AS ( SELECT tx_hash, - status_value, - TRY_PARSE_JSON(REPLACE(LOGS[0] :: STRING, 'EVENT_JSON:', '')) AS event, - PARSE_JSON(BASE64_DECODE_STRING(status_value:SuccessValue)) as SuccessValue, + outcome_json :outcome :status AS status_value, + TRY_PARSE_JSON(REPLACE(outcome_json :outcome :logs[0] :: STRING, 'EVENT_JSON:', '')) AS event, + PARSE_JSON(BASE64_DECODE_STRING(outcome_json :outcome :status :SuccessValue)) as SuccessValue, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE - receipt_actions:predecessor_id = 'marketplace.paras.near' + predecessor_id = 'marketplace.paras.near' AND event:event = 'nft_transfer' diff --git a/models/silver/curated/nft/silver__standard_nft_mint_s3.sql b/models/silver/curated/nft/silver__standard_nft_mint_s3.sql index ca8a8a8..36ba53c 100644 --- a/models/silver/curated/nft/silver__standard_nft_mint_s3.sql +++ b/models/silver/curated/nft/silver__standard_nft_mint_s3.sql @@ -47,10 +47,9 @@ tx AS ( tx_signer, tx_receiver, tx_succeeded, - tx_status, -- TODO deprecate col transaction_fee FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} @@ -214,7 +213,6 @@ mint_tx AS ( tx_signer, tx_receiver, tx_succeeded, - tx_status, transaction_fee FROM tx @@ -239,7 +237,6 @@ FINAL AS ( mint_tx.tx_signer AS tx_signer, mint_tx.tx_receiver AS tx_receiver, mint_tx.tx_succeeded AS tx_succeeded, - mint_tx.tx_status AS tx_status, mint_events.receipt_object_id, mint_events.receiver_id, mint_events.signer_id, diff --git a/models/silver/curated/silver__lockup_actions.sql b/models/silver/curated/silver__lockup_actions.sql index 6fdcd03..ed08599 100644 --- a/models/silver/curated/silver__lockup_actions.sql +++ b/models/silver/curated/silver__lockup_actions.sql @@ -16,9 +16,9 @@ WITH txs AS ( tx_hash, tx_succeeded, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} diff --git a/models/silver/curated/staking/silver__pool_events.sql b/models/silver/curated/staking/silver__pool_events.sql index 822c78d..6dddd95 100644 --- a/models/silver/curated/staking/silver__pool_events.sql +++ b/models/silver/curated/staking/silver__pool_events.sql @@ -12,16 +12,16 @@ WITH receipts AS ( tx_hash, block_id, block_timestamp, - receipt_object_id, + receipt_id AS receipt_object_id, receiver_id, - signer_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, - status_value, - logs, - _inserted_timestamp, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, + predecessor_id, + receipt_json AS receipt_actions, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE receipt_succeeded {% if var("MANUAL_FIX") %} @@ -48,8 +48,7 @@ FINAL AS ( status_value, logs, VALUE AS LOG, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM receipts, LATERAL FLATTEN(logs) diff --git a/models/silver/curated/staking/silver__staking_pools_s3.sql b/models/silver/curated/staking/silver__staking_pools_s3.sql index d228312..9a9baf4 100644 --- a/models/silver/curated/staking/silver__staking_pools_s3.sql +++ b/models/silver/curated/staking/silver__staking_pools_s3.sql @@ -17,12 +17,10 @@ WITH txs AS ( block_id, tx_signer, tx_receiver, - tx, - tx_status, - _partition_by_block_number, - _inserted_timestamp + transaction_json AS tx, + _partition_by_block_number FROM - {{ ref('silver__streamline_transactions_final') }} + {{ ref('silver__transactions_final') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} @@ -48,8 +46,7 @@ function_calls AS ( signer_id, method_name, args, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM {{ ref('silver__actions_events_function_call_s3') }} WHERE @@ -84,9 +81,7 @@ add_addresses_from_tx AS ( signer_id, method_name, args, - tx_status, - txs._partition_by_block_number, - txs._inserted_timestamp + txs._partition_by_block_number FROM function_calls fc LEFT JOIN txs USING (tx_hash) @@ -102,8 +97,7 @@ new_pools AS ( args :reward_fee_fraction ) AS reward_fee_fraction, 'Create' AS tx_type, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM add_addresses_from_tx WHERE @@ -128,8 +122,7 @@ updated_pools AS ( args :reward_fee_fraction ) AS reward_fee_fraction, 'Update' AS tx_type, - _partition_by_block_number, - _inserted_timestamp + _partition_by_block_number FROM add_addresses_from_tx WHERE From 0b15f71566e1510240fb85d6350533b7ff0c6a8b Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:47:59 -0600 Subject: [PATCH 35/53] social and supply --- .../supply/silver__atlas_supply_epochs.sql | 19 ++++++++--- .../silver__atlas_supply_lockup_receipts.sql | 34 +++++++++---------- .../silver/social/silver_social__receipts.sql | 33 +++++++++--------- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/models/silver/atlas/supply/silver__atlas_supply_epochs.sql b/models/silver/atlas/supply/silver__atlas_supply_epochs.sql index 11ad3dd..f6f15f1 100644 --- a/models/silver/atlas/supply/silver__atlas_supply_epochs.sql +++ b/models/silver/atlas/supply/silver__atlas_supply_epochs.sql @@ -8,14 +8,23 @@ WITH blocks AS ( SELECT - * + block_id, + block_timestamp, + block_author, + header_json :total_supply :: NUMBER AS total_supply, + header_json :epoch_id :: STRING AS epoch_id, + _partition_by_block_number FROM - {{ ref('silver__streamline_blocks') }} - WHERE + {{ ref('silver__blocks_final') }} {% if var("MANUAL_FIX") %} - {{ partition_load_manual('no_buffer') }} + WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {{ incremental_load_filter('_inserted_timestamp') }} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) {% endif %} ), epochs AS ( diff --git a/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql b/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql index 5f065ae..18b72da 100644 --- a/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql +++ b/models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql @@ -10,27 +10,27 @@ WITH receipts AS ( SELECT - receipt_object_id, + receipt_id AS receipt_object_id, tx_hash, block_timestamp, - receipt_actions, + receipt_json AS receipt_actions, receiver_id, - status_value, - logs, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp + predecessor_id, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} - WHERE + {{ ref('silver__receipts_final') }} {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {% if var('IS_MIGRATION') %} - {{ incremental_load_filter('_inserted_timestamp') }} - {% else %} - {{ incremental_load_filter('_modified_timestamp') }} - {% endif %} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) {% endif %} ), FINAL AS ( @@ -38,16 +38,14 @@ FINAL AS ( receipt_object_id, tx_hash, block_timestamp, - receipt_actions :predecessor_id :: STRING AS predecessor_id, + predecessor_id, receiver_id, receipt_actions AS actions, object_keys( status_value ) [0] :: STRING AS status, logs, - _partition_by_block_number, - _inserted_timestamp, - _modified_timestamp + _partition_by_block_number FROM receipts WHERE diff --git a/models/silver/social/silver_social__receipts.sql b/models/silver/social/silver_social__receipts.sql index f532c13..5008b87 100644 --- a/models/silver/social/silver_social__receipts.sql +++ b/models/silver/social/silver_social__receipts.sql @@ -2,7 +2,7 @@ materialized = 'incremental', merge_exclude_columns = ["inserted_timestamp"], unique_key = 'receipt_object_id', - cluster_by = ['_inserted_timestamp::date', '_partition_by_block_number'], + cluster_by = ['modified_timestamp::date', '_partition_by_block_number'], tags = ['curated', 'social','scheduled_non_core'] ) }} @@ -10,27 +10,26 @@ WITH all_social_receipts AS ( SELECT tx_hash, - receipt_object_id, + receipt_id AS receipt_object_id, block_id, block_timestamp, - receipt_index, + NULL AS receipt_index, chunk_hash, - receipt_actions, - execution_outcome, - receipt_outcome_id, + receipt_json AS receipt_actions, + outcome_json AS execution_outcome, + outcome_json :outcome :receipt_ids :: ARRAY AS receipt_outcome_id, receiver_id, - receipt_actions :predecessor_id :: STRING AS predecessor_id, - signer_id, - receipt_type, - gas_burnt, - status_value, - logs, - proof, - metadata, - _partition_by_block_number, - _inserted_timestamp + predecessor_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, + NULL AS receipt_type, + outcome_json :outcome :gas_burnt :: NUMBER AS gas_burnt, + outcome_json :outcome :status :: VARIANT AS status_value, + outcome_json :outcome :logs :: ARRAY AS logs, + outcome_json :proof :: ARRAY AS proof, + outcome_json :outcome :metadata :: VARIANT AS metadata, + _partition_by_block_number FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE ( LOWER(signer_id) = 'social.near' From d38112a4d7f06c4c4ebf8f3d40899565e1989f5d Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:51:02 -0600 Subject: [PATCH 36/53] last social --- models/silver/social/silver_social__addkey.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/silver/social/silver_social__addkey.sql b/models/silver/social/silver_social__addkey.sql index cb7ff80..2852fe8 100644 --- a/models/silver/social/silver_social__addkey.sql +++ b/models/silver/social/silver_social__addkey.sql @@ -12,12 +12,12 @@ WITH receipts AS ( SELECT - receipt_object_id, - signer_id, + receipt_id AS receipt_object_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, _partition_by_block_number, - _inserted_timestamp + inserted_timestamp AS _inserted_timestamp FROM - {{ ref('silver__streamline_receipts_final') }} + {{ ref('silver__receipts_final') }} WHERE _partition_by_block_number >= 59670000 From e636f2b37cac5b1bc74b705a0d2397caae2c91a5 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:52:09 -0600 Subject: [PATCH 37/53] upd migration test no filter --- tests/gaps/core/tests__receipt_migration_gaps.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gaps/core/tests__receipt_migration_gaps.sql b/tests/gaps/core/tests__receipt_migration_gaps.sql index abd9d80..251cfcb 100644 --- a/tests/gaps/core/tests__receipt_migration_gaps.sql +++ b/tests/gaps/core/tests__receipt_migration_gaps.sql @@ -10,7 +10,7 @@ archive AS ( count(distinct coalesce(receipt_object_id, receipt_id)) as receipt_count, count(distinct block_id) as block_count from {{ ref('silver__streamline_receipts_final') }} - where _partition_by_block_number >= 116000000 + group by 1 ), destination AS ( @@ -19,7 +19,7 @@ destination AS ( count(distinct receipt_id) as receipt_count, count(distinct block_id) as block_count from {{ ref('silver__receipts_final') }} - where _partition_by_block_number >= 116000000 + group by 1 ) select From 08112b5fe425a09fe943014a310d25789fcca432 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 14:34:04 -0600 Subject: [PATCH 38/53] gap tests & add chunk created height to rt passthrough --- models/silver/core/silver__chunks_v2.sql | 57 ++++++++++ .../silver/core/silver__transactions_v2.sql | 4 + .../complete/streamline__chunks_complete.sql | 2 + .../streamline__transactions_complete.sql | 7 +- .../streamline__transactions_realtime.sql | 6 +- .../core/streamline__transactions.sql | 6 + tests/gaps/core/tests__block_gaps.sql | 43 ++++--- tests/gaps/core/tests__chunk_gaps.sql | 107 +++++++----------- tests/gaps/core/tests__receipt_gaps.sql | 97 +++++++--------- .../core/tests__receipt_migration_gaps.sql | 34 ------ tests/gaps/core/tests__tx_gaps.sql | 97 +++++++--------- .../gaps/downstream/tests__final_gaps_rec.sql | 49 -------- .../gaps/downstream/tests__final_gaps_tx.sql | 55 --------- tests/gaps/downstream/tests__logs_gap.sql | 78 ------------- tests/gaps/downstream/tests__logs_gap.yml | 8 -- 15 files changed, 235 insertions(+), 415 deletions(-) create mode 100644 models/silver/core/silver__chunks_v2.sql delete mode 100644 tests/gaps/core/tests__receipt_migration_gaps.sql delete mode 100644 tests/gaps/downstream/tests__final_gaps_rec.sql delete mode 100644 tests/gaps/downstream/tests__final_gaps_tx.sql delete mode 100644 tests/gaps/downstream/tests__logs_gap.sql delete mode 100644 tests/gaps/downstream/tests__logs_gap.yml diff --git a/models/silver/core/silver__chunks_v2.sql b/models/silver/core/silver__chunks_v2.sql new file mode 100644 index 0000000..229e318 --- /dev/null +++ b/models/silver/core/silver__chunks_v2.sql @@ -0,0 +1,57 @@ +-- depends_on: {{ ref('bronze__chunks') }} +-- depends_on: {{ ref('bronze__FR_chunks') }} +{{ config ( + materialized = "incremental", + incremental_strategy = 'merge', + incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], + unique_key = "chunk_hash", + cluster_by = ['modified_timestamp::DATE','block_timestamp::date'], + tags = ['scheduled_core', 'core_v2'] +) }} + +WITH bronze_chunks AS ( + + SELECT + VALUE :BLOCK_ID :: INT AS block_id, + VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, + VALUE :CHUNK_HASH :: STRING AS chunk_hash, + partition_key, + DATA :: variant AS chunk_json, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__chunks') }} +WHERE + _inserted_timestamp >= ( + SELECT + COALESCE(MAX(_inserted_timestamp), '1900-01-01' :: TIMESTAMP) AS _inserted_timestamp + FROM + {{ this }}) + AND typeof(DATA) != 'NULL_VALUE' + {% else %} + {{ ref('bronze__FR_chunks') }} + WHERE + typeof(DATA) != 'NULL_VALUE' + {% endif %} + ) +SELECT + chunk_hash, + block_id, + block_timestamp_epoch, + TO_TIMESTAMP_NTZ(block_timestamp_epoch, 9) AS block_timestamp, + partition_key, + chunk_json, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['chunk_hash']) }} AS chunks_v2_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + bronze_chunks + +qualify ROW_NUMBER() over ( + PARTITION BY chunk_hash + ORDER BY + _inserted_timestamp DESC + ) = 1 diff --git a/models/silver/core/silver__transactions_v2.sql b/models/silver/core/silver__transactions_v2.sql index 6584360..b17d475 100644 --- a/models/silver/core/silver__transactions_v2.sql +++ b/models/silver/core/silver__transactions_v2.sql @@ -16,6 +16,8 @@ WITH bronze_transactions AS ( VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS origin_block_timestamp_epoch, VALUE :SHARD_ID :: INT AS shard_id, VALUE :CHUNK_HASH :: STRING AS chunk_hash, + VALUE :HEIGHT_CREATED :: INT AS chunk_height_created, + VALUE :HEIGHT_INCLUDED :: INT AS chunk_height_included, DATA :transaction :hash :: STRING AS tx_hash, DATA :transaction :signer_id :: STRING AS signer_id, partition_key, @@ -44,6 +46,8 @@ SELECT TO_TIMESTAMP_NTZ(origin_block_timestamp_epoch, 9) AS origin_block_timestamp, shard_id, chunk_hash, + chunk_height_created, + chunk_height_included, tx_hash, signer_id, partition_key, diff --git a/models/streamline/core/complete/streamline__chunks_complete.sql b/models/streamline/core/complete/streamline__chunks_complete.sql index 6d66ec1..bfc84a3 100644 --- a/models/streamline/core/complete/streamline__chunks_complete.sql +++ b/models/streamline/core/complete/streamline__chunks_complete.sql @@ -13,6 +13,8 @@ SELECT VALUE :BLOCK_ID :: INT AS block_id, VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, VALUE :CHUNK_HASH :: STRING AS chunk_hash, + DATA :header :height_created :: INT AS height_created, + DATA :header :height_included :: INT AS height_included, DATA :header: shard_id :: INT AS shard_id, ARRAY_SIZE( DATA :receipts :: ARRAY diff --git a/models/streamline/core/complete/streamline__transactions_complete.sql b/models/streamline/core/complete/streamline__transactions_complete.sql index 1390224..94c22f9 100644 --- a/models/streamline/core/complete/streamline__transactions_complete.sql +++ b/models/streamline/core/complete/streamline__transactions_complete.sql @@ -10,12 +10,9 @@ ) }} SELECT - VALUE :BLOCK_ID :: INT AS block_id, - VALUE :BLOCK_TIMESTAMP_EPOCH :: INT AS block_timestamp_epoch, VALUE :TX_HASH :: STRING AS tx_hash, - VALUE :transaction :shard_id :: STRING AS shard_id, - VALUE :transaction :chunk_hash :: STRING AS chunk_hash, - DATA :transaction :signer_id :: STRING AS signer_id, + VALUE :chunk_hash :: STRING AS chunk_hash, + VALUE :BLOCK_ID :: INT AS block_id, partition_key, _inserted_timestamp, DATA :transaction :hash :: STRING AS complete_transactions_id, diff --git a/models/streamline/core/realtime/streamline__transactions_realtime.sql b/models/streamline/core/realtime/streamline__transactions_realtime.sql index 4467c65..3144e1f 100644 --- a/models/streamline/core/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/core/realtime/streamline__transactions_realtime.sql @@ -42,7 +42,9 @@ tbl AS ( A.tx_hash, A.signer_id, A.shard_id, - A.chunk_hash + A.chunk_hash, + A.height_created, + A.height_included FROM {{ ref('streamline__transactions') }} A LEFT JOIN {{ ref('streamline__transactions_complete') }} B ON A.tx_hash = B.tx_hash @@ -63,6 +65,8 @@ SELECT chunk_hash, block_id, block_timestamp_epoch, + height_created, + height_included, tx_hash, FLOOR(block_id, -3) AS partition_key, DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS request_timestamp, diff --git a/models/streamline/core/streamline__transactions.sql b/models/streamline/core/streamline__transactions.sql index b0d3e78..30d24a3 100644 --- a/models/streamline/core/streamline__transactions.sql +++ b/models/streamline/core/streamline__transactions.sql @@ -13,6 +13,8 @@ WITH chunks_complete AS ( SELECT block_id, block_timestamp_epoch, + height_created, + height_included, chunk_hash, shard_id, transaction_ids, @@ -35,6 +37,8 @@ flatten_tx_ids AS ( SELECT block_id, block_timestamp_epoch, + height_created, + height_included, chunk_hash, shard_id, VALUE :: STRING AS tx_hash, @@ -50,6 +54,8 @@ flatten_tx_ids AS ( SELECT block_id, block_timestamp_epoch, + height_created, + height_included, chunk_hash, shard_id, tx_hash, diff --git a/tests/gaps/core/tests__block_gaps.sql b/tests/gaps/core/tests__block_gaps.sql index b03e25f..39e8c77 100644 --- a/tests/gaps/core/tests__block_gaps.sql +++ b/tests/gaps/core/tests__block_gaps.sql @@ -2,10 +2,32 @@ severity = 'error', tags = ['gap_test'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} + +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 9820210 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + WITH silver_blocks AS ( SELECT + block_timestamp, + block_hash, block_id, LAG(block_id) over ( ORDER BY @@ -13,32 +35,23 @@ WITH silver_blocks AS ( block_id ASC ) AS prior_block_id, block_id - prior_block_id AS gap_size, - block_timestamp, - block_hash, prev_hash, LAG(block_hash) over ( ORDER BY block_timestamp ASC, block_id ASC - ) AS prior_hash, - _partition_by_block_number, - inserted_timestamp, - SYSDATE() AS _test_timestamp + ) AS prev_hash_actual FROM {{ ref('silver__blocks_final') }} - {% if var('DBT_FULL_TEST') %} - WHERE - inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} + WHERE + block_id >= {{ min_block_id }} + ) SELECT * FROM silver_blocks WHERE - prior_hash <> prev_hash + prev_hash != COALESCE(prev_hash_actual, '') + AND block_id != {{ min_block_id }} diff --git a/tests/gaps/core/tests__chunk_gaps.sql b/tests/gaps/core/tests__chunk_gaps.sql index ee0161d..24f3528 100644 --- a/tests/gaps/core/tests__chunk_gaps.sql +++ b/tests/gaps/core/tests__chunk_gaps.sql @@ -2,80 +2,59 @@ severity = 'error', tags = ['gap_test'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} -WITH blocks AS ( +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 9820210 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + +WITH expected_chunks AS ( SELECT block_id, - header :chunks_included :: INT AS chunk_ct_expected, - _partition_by_block_number, - _inserted_timestamp + _inserted_timestamp, + VALUE :chunk_hash :: STRING AS chunk_hash, + VALUE :height_created :: INT AS height_created, + VALUE :height_included :: INT AS height_included FROM - {{ ref('silver__streamline_blocks') }} + {{ ref('silver__blocks_v2') }}, lateral flatten(input => block_json :chunks :: ARRAY) + WHERE + block_id >= {{ min_block_id }} - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} + qualify(ROW_NUMBER() over (PARTITION BY chunk_hash ORDER BY block_id ASC)) = 1 ), -shards AS ( +actual_chunks AS ( SELECT - block_id, - MAX(_inserted_timestamp) AS _inserted_timestamp, - COUNT( - DISTINCT chunk :header :chunk_hash :: STRING - ) AS chunk_ct_actual + DISTINCT chunk_hash FROM - {{ ref('silver__streamline_shards') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1 -), -comp AS ( - SELECT - b.block_id AS b_block_id, - s.block_id AS s_block_id, - b.chunk_ct_expected, - s.chunk_ct_actual, - _partition_by_block_number, - b._inserted_timestamp AS _inserted_timestamp_blocks, - s._inserted_timestamp AS _inserted_timestamp_shards - FROM - blocks b full - OUTER JOIN shards s USING (block_id) + {{ ref('silver__chunks_v2') }} + WHERE + block_id >= {{ min_block_id }} ) SELECT - COALESCE( - b_block_id, - s_block_id - ) AS block_id, - chunk_ct_expected, - chunk_ct_actual, - _partition_by_block_number, - ( - chunk_ct_actual != chunk_ct_expected - OR b_block_id IS NULL - OR s_block_id IS NULL - ) AS is_missing + block_id, + _inserted_timestamp, + chunk_hash, + height_created, + height_included FROM - comp + expected_chunks e + LEFT JOIN actual_chunks a USING (chunk_hash) WHERE - chunk_ct_expected > 0 - AND is_missing - {# Filter out false positive from blocks at start of window #} - AND _inserted_timestamp_blocks > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour' - AND _inserted_timestamp_shards > SYSDATE() - INTERVAL '7 days' + INTERVAL '1 hour' -ORDER BY - 1 + a.chunk_hash IS NULL + AND _inserted_timestamp <= SYSDATE() - interval '1 hour' + AND height_included >= {{ min_block_id }} diff --git a/tests/gaps/core/tests__receipt_gaps.sql b/tests/gaps/core/tests__receipt_gaps.sql index 781b85b..b3090d6 100644 --- a/tests/gaps/core/tests__receipt_gaps.sql +++ b/tests/gaps/core/tests__receipt_gaps.sql @@ -2,68 +2,59 @@ severity = 'error', tags = ['gap_test'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} -WITH shards AS ( +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 9820210 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + +WITH expected_receipts AS ( SELECT block_id, - _partition_by_block_number, - SUM(ARRAY_SIZE(receipt_execution_outcomes)) AS receipt_ct_expected + chunk_hash, + chunk_json :height_created :: INT as chunk_height_created, + chunk_json :height_included :: INT as chunk_height_included, + _inserted_timestamp, + VALUE :receipt_id :: STRING AS receipt_id FROM - {{ ref('silver__streamline_shards') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 + {{ ref('silver__chunks_v2') }}, lateral flatten(input => chunk_json :receipts :: ARRAY) + WHERE + block_id >= {{ min_block_id }} ), -receipts AS ( +actual_receipts AS ( SELECT - block_id, - _partition_by_block_number, - COUNT(DISTINCT receipt_id) AS receipt_ct_actual_distinct, - COUNT(1) AS receipt_ct_actual_all + DISTINCT receipt_id FROM - {{ ref('silver__streamline_receipts') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 -), -diffs AS ( - SELECT - s.block_id, - s.receipt_ct_expected, - r.receipt_ct_actual_distinct, - r.receipt_ct_actual_all, - s._partition_by_block_number - FROM - shards s - LEFT JOIN receipts r - ON s.block_id = r.block_id + {{ ref('silver__receipts_final') }} + WHERE + block_id >= {{ min_block_id }} ) SELECT - * + block_id, + chunk_hash, + chunk_height_created, + chunk_height_included, + _inserted_timestamp, + receipt_id FROM - diffs + expected_receipts e + LEFT JOIN actual_receipts a USING (receipt_id) WHERE - receipt_ct_expected != receipt_ct_actual_distinct - OR receipt_ct_expected != receipt_ct_actual_all - OR receipt_ct_actual_distinct != receipt_ct_actual_all -ORDER BY - block_id + a.receipt_id IS NULL + AND _inserted_timestamp <= SYSDATE() - interval '2 hours' + AND chunk_height_included >= {{ min_block_id }} diff --git a/tests/gaps/core/tests__receipt_migration_gaps.sql b/tests/gaps/core/tests__receipt_migration_gaps.sql deleted file mode 100644 index 251cfcb..0000000 --- a/tests/gaps/core/tests__receipt_migration_gaps.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{ config( - severity = 'error', - tags = ['gap_test'] -) }} - -WITH -archive AS ( - select - floor(block_id, -6) as block_group, - count(distinct coalesce(receipt_object_id, receipt_id)) as receipt_count, - count(distinct block_id) as block_count - from {{ ref('silver__streamline_receipts_final') }} - - group by 1 -), -destination AS ( - select - floor(block_id, -6) as block_group, - count(distinct receipt_id) as receipt_count, - count(distinct block_id) as block_count - from {{ ref('silver__receipts_final') }} - - group by 1 -) -select - archive.block_group, - archive.receipt_count as receipt_ct_expected, - archive.block_count as block_ct_expected, - destination.receipt_count as receipt_ct_actual, - destination.block_count as block_ct_actual -from archive -left join destination -on archive.block_group = destination.block_group -order by block_group diff --git a/tests/gaps/core/tests__tx_gaps.sql b/tests/gaps/core/tests__tx_gaps.sql index 43812e6..f2eb929 100644 --- a/tests/gaps/core/tests__tx_gaps.sql +++ b/tests/gaps/core/tests__tx_gaps.sql @@ -2,68 +2,59 @@ severity = 'error', tags = ['gap_test'] ) }} +-- depends_on: {{ ref('silver__blocks_v2') }} -WITH shards AS ( +{% if execute %} + + {% if not var('DBT_FULL_TEST') %} + {% set min_block_sql %} + SELECT + MIN(block_id) + FROM + {{ ref('silver__blocks_v2') }} + WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '7 days' + {% endset %} + {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} + {% else %} + {% set min_block_id = 9820210 %} + {% endif %} + {% do log('Min block id: ' ~ min_block_id, info=True) %} +{% endif %} + +WITH expected_txs AS ( SELECT block_id, - _partition_by_block_number, - SUM(ARRAY_SIZE(chunk :transactions :: ARRAY)) AS tx_ct_expected + chunk_hash, + chunk_json :height_created :: INT as chunk_height_created, + chunk_json :height_included :: INT as chunk_height_included, + _inserted_timestamp, + VALUE :hash :: STRING AS tx_hash FROM - {{ ref('silver__streamline_shards') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 + {{ ref('silver__chunks_v2') }}, lateral flatten(input => chunk_json :transactions :: ARRAY) + WHERE + block_id >= {{ min_block_id }} ), -txs AS ( +actual_txs AS ( SELECT - block_id, - _partition_by_block_number, - COUNT(DISTINCT tx_hash) AS tx_ct_actual_distinct, - COUNT(1) AS tx_ct_actual_all + DISTINCT tx_hash FROM - {{ ref('silver__streamline_transactions') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, - 2 -), -diffs AS ( - SELECT - s.block_id, - s.tx_ct_expected, - t.tx_ct_actual_distinct, - t.tx_ct_actual_all, - s._partition_by_block_number - FROM - shards s - LEFT JOIN txs t - ON s.block_id = t.block_id + {{ ref('silver__transactions_v2') }} + WHERE + origin_block_id >= {{ min_block_id }} ) SELECT - * + block_id, + chunk_hash, + chunk_height_created, + chunk_height_included, + _inserted_timestamp, + tx_hash FROM - diffs + expected_txs e + LEFT JOIN actual_txs a USING (tx_hash) WHERE - tx_ct_expected != tx_ct_actual_distinct - OR tx_ct_expected != tx_ct_actual_all - OR tx_ct_actual_distinct != tx_ct_actual_all -ORDER BY - block_id + a.tx_hash IS NULL + AND _inserted_timestamp <= SYSDATE() - interval '2 hours' + AND chunk_height_included >= {{ min_block_id }} diff --git a/tests/gaps/downstream/tests__final_gaps_rec.sql b/tests/gaps/downstream/tests__final_gaps_rec.sql deleted file mode 100644 index 0c16c72..0000000 --- a/tests/gaps/downstream/tests__final_gaps_rec.sql +++ /dev/null @@ -1,49 +0,0 @@ -{{ config( - severity = "error", - tags = ['gap_test'] -) }} - - -WITH r_receipts AS ( - SELECT - DISTINCT receipt_id, - block_id - FROM - {{ ref('silver__streamline_receipts') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -f_receipts AS ( - SELECT - DISTINCT receipt_object_id AS receipt_id, - block_id - FROM - {{ ref('silver__streamline_receipts_final') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -) -SELECT - r_receipts.receipt_id AS receipt_id, - r_receipts.block_id -FROM - r_receipts -LEFT JOIN - f_receipts -ON - r_receipts.receipt_id = f_receipts.receipt_id -WHERE - f_receipts.receipt_id IS NULL diff --git a/tests/gaps/downstream/tests__final_gaps_tx.sql b/tests/gaps/downstream/tests__final_gaps_tx.sql deleted file mode 100644 index 2809058..0000000 --- a/tests/gaps/downstream/tests__final_gaps_tx.sql +++ /dev/null @@ -1,55 +0,0 @@ -{{ config( - severity = "error", - tags = ['gap_test'] -) }} - - -WITH -r_transactions AS ( - SELECT - DISTINCT tx_hash, - block_id, - _signer_id AS signer_id, - shard_id - FROM - {{ ref('silver__streamline_transactions') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -f_transactions AS ( - SELECT - DISTINCT tx_hash, - block_id, - tx_signer AS signer_id - FROM - {{ ref('silver__streamline_transactions_final') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -) -SELECT - r_transactions.tx_hash AS tx_hash, - r_transactions.block_id, - r_transactions.signer_id, - r_transactions.shard_id -FROM - r_transactions -LEFT JOIN - f_transactions -ON - r_transactions.tx_hash = f_transactions.tx_hash -WHERE - f_transactions.tx_hash IS NULL diff --git a/tests/gaps/downstream/tests__logs_gap.sql b/tests/gaps/downstream/tests__logs_gap.sql deleted file mode 100644 index 658d1f7..0000000 --- a/tests/gaps/downstream/tests__logs_gap.sql +++ /dev/null @@ -1,78 +0,0 @@ -{{ config( - severity = "error", - tags = ['gap_test'] -) }} - -WITH r_logs AS ( - - SELECT - receipt_id, - block_id, - ARRAY_SIZE(execution_outcome :outcome :logs :: ARRAY) AS log_ct - FROM - {{ ref('silver__streamline_receipts') }} - WHERE - ARRAY_SIZE(execution_outcome :outcome :logs :: ARRAY) > 0 - {% if var('DBT_FULL_TEST') %} - AND _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - AND _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -r_final_logs AS ( - SELECT - receipt_object_id AS receipt_id, - block_timestamp, - block_id, - ARRAY_SIZE(logs) AS log_ct - FROM - {{ ref('silver__streamline_receipts_final') }} - WHERE - ARRAY_SIZE(logs) > 0 - - {% if var('DBT_FULL_TEST') %} - AND _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - AND _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} -), -logs AS ( - SELECT - receipt_object_id AS receipt_id, - block_id, - count(1) AS log_ct - FROM - {{ ref('silver__logs_s3') }} - - {% if var('DBT_FULL_TEST') %} - WHERE - _inserted_timestamp < SYSDATE() - INTERVAL '1 hour' - {% else %} - WHERE - _inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' - AND SYSDATE() - INTERVAL '1 hour' - {% endif %} - GROUP BY - 1, 2 -) -SELECT - r.receipt_id, - r.block_id, - rf.block_timestamp, - r.log_ct AS log_ct_expected, - rf.log_ct AS log_ct_final_expected, - l.log_ct AS log_ct_actual, - FLOOR(r.block_id, -3) AS _partition_by_block_number -FROM - r_logs r - LEFT JOIN r_final_logs rf - ON r.receipt_id = rf.receipt_id - AND r.block_id = rf.block_id - LEFT JOIN logs l - ON r.receipt_id = l.receipt_id - AND r.block_id = l.block_id -WHERE - r.log_ct != l.log_ct - OR rf.log_ct IS NULL diff --git a/tests/gaps/downstream/tests__logs_gap.yml b/tests/gaps/downstream/tests__logs_gap.yml deleted file mode 100644 index 2c47219..0000000 --- a/tests/gaps/downstream/tests__logs_gap.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: 2 - -tests: - name: tests__logs_gap - description: |- - This test checks for gaps in logs by checking against the initial receipts model and receipts_final. - If there are gaps between receipts and receipts_final, that will likely lead to log gaps. - The user must look at the data to determine where the gap may be. From 7ae79302e6aecd3e1f38ae1dd04b989867139c35 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 15:00:05 -0600 Subject: [PATCH 39/53] add tag gap_test_core --- tests/gaps/core/tests__block_gaps.sql | 2 +- tests/gaps/core/tests__chunk_gaps.sql | 2 +- tests/gaps/core/tests__receipt_gaps.sql | 2 +- tests/gaps/core/tests__tx_gaps.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gaps/core/tests__block_gaps.sql b/tests/gaps/core/tests__block_gaps.sql index 39e8c77..aea1dad 100644 --- a/tests/gaps/core/tests__block_gaps.sql +++ b/tests/gaps/core/tests__block_gaps.sql @@ -1,6 +1,6 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} -- depends_on: {{ ref('silver__blocks_v2') }} diff --git a/tests/gaps/core/tests__chunk_gaps.sql b/tests/gaps/core/tests__chunk_gaps.sql index 24f3528..fcd2f09 100644 --- a/tests/gaps/core/tests__chunk_gaps.sql +++ b/tests/gaps/core/tests__chunk_gaps.sql @@ -1,6 +1,6 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} -- depends_on: {{ ref('silver__blocks_v2') }} diff --git a/tests/gaps/core/tests__receipt_gaps.sql b/tests/gaps/core/tests__receipt_gaps.sql index b3090d6..62a9a8b 100644 --- a/tests/gaps/core/tests__receipt_gaps.sql +++ b/tests/gaps/core/tests__receipt_gaps.sql @@ -1,6 +1,6 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} -- depends_on: {{ ref('silver__blocks_v2') }} diff --git a/tests/gaps/core/tests__tx_gaps.sql b/tests/gaps/core/tests__tx_gaps.sql index f2eb929..fb0d57f 100644 --- a/tests/gaps/core/tests__tx_gaps.sql +++ b/tests/gaps/core/tests__tx_gaps.sql @@ -1,6 +1,6 @@ {{ config( severity = 'error', - tags = ['gap_test'] + tags = ['gap_test', 'gap_test_core'] ) }} -- depends_on: {{ ref('silver__blocks_v2') }} From 71cda390f06db50652093b76b3bf8c05c6c5f137 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 15:29:35 -0600 Subject: [PATCH 40/53] fix signer_id in logs --- models/silver/curated/silver__logs_s3.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/curated/silver__logs_s3.sql b/models/silver/curated/silver__logs_s3.sql index ee16a0e..31588e5 100644 --- a/models/silver/curated/silver__logs_s3.sql +++ b/models/silver/curated/silver__logs_s3.sql @@ -19,7 +19,7 @@ WITH receipts AS ( outcome_json :outcome :logs AS logs, receiver_id, predecessor_id, - signer_id, + receipt_json :receipt :Action :signer_id :: STRING AS signer_id, outcome_json :outcome :gas_burnt AS gas_burnt, receipt_succeeded, _partition_by_block_number From 7d12c908be4d53b15d237a416efaaf90225c1c5d Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 16:58:03 -0600 Subject: [PATCH 41/53] receipts_final - add qualify and extra day on blocks scan --- models/gold/core/core__fact_blocks.sql | 2 +- models/gold/core/core__fact_blocks.yml | 2 +- models/silver/core/silver__receipts_final.sql | 4 +++- models/silver/curated/silver__logs_s3.sql | 1 + tests/gaps/core/tests__block_gaps.sql | 2 +- tests/gaps/core/tests__chunk_gaps.sql | 2 +- tests/gaps/core/tests__receipt_gaps.sql | 2 +- tests/gaps/core/tests__tx_gaps.sql | 2 +- 8 files changed, 10 insertions(+), 7 deletions(-) diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index 4c6857b..b2cde7d 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -30,7 +30,7 @@ SELECT header_json :gas_price :: FLOAT AS gas_price, header_json :last_ds_final_block :: STRING AS last_ds_final_block, header_json :last_final_block :: STRING AS last_final_block, - header_json :latest_protocol_version :: STRING AS latest_protocol_version, + header_json :latest_protocol_version :: INT AS latest_protocol_version, header_json :next_bp_hash :: STRING AS next_bp_hash, header_json :next_epoch_id :: STRING AS next_epoch_id, header_json :outcome_root :: STRING AS outcome_root, diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index cde8c8e..b499f0a 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -185,7 +185,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - FLOAT + - INT - name: NEXT_BP_HASH description: "{{ doc('next_bp_hash')}}" diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 7abd640..a93f210 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -104,7 +104,7 @@ blocks AS ( {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - WHERE block_timestamp :: DATE >= '{{min_bd}}' + WHERE block_timestamp :: DATE >= '{{min_bd}}' - interval '1 day' {% endif %} {% endif %} ), @@ -189,4 +189,6 @@ SELECT FROM FINAL +qualify(row_number() over (partition by receipt_id order by block_id is not null desc, modified_timestamp desc) = 1) + {% endif %} diff --git a/models/silver/curated/silver__logs_s3.sql b/models/silver/curated/silver__logs_s3.sql index 31588e5..69be8f1 100644 --- a/models/silver/curated/silver__logs_s3.sql +++ b/models/silver/curated/silver__logs_s3.sql @@ -53,6 +53,7 @@ FINAL AS ( INDEX AS log_index, receiver_id, predecessor_id, + signer_id, COALESCE( TRY_PARSE_JSON(VALUE), TRY_PARSE_JSON(SPLIT(VALUE, 'EVENT_JSON:') [1]), diff --git a/tests/gaps/core/tests__block_gaps.sql b/tests/gaps/core/tests__block_gaps.sql index aea1dad..0308d35 100644 --- a/tests/gaps/core/tests__block_gaps.sql +++ b/tests/gaps/core/tests__block_gaps.sql @@ -17,7 +17,7 @@ {% endset %} {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} {% else %} - {% set min_block_id = 9820210 %} + {% set min_block_id = 140868759 %} {% endif %} {% do log('Min block id: ' ~ min_block_id, info=True) %} {% endif %} diff --git a/tests/gaps/core/tests__chunk_gaps.sql b/tests/gaps/core/tests__chunk_gaps.sql index fcd2f09..a1af7f4 100644 --- a/tests/gaps/core/tests__chunk_gaps.sql +++ b/tests/gaps/core/tests__chunk_gaps.sql @@ -17,7 +17,7 @@ {% endset %} {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} {% else %} - {% set min_block_id = 9820210 %} + {% set min_block_id = 140868759 %} {% endif %} {% do log('Min block id: ' ~ min_block_id, info=True) %} {% endif %} diff --git a/tests/gaps/core/tests__receipt_gaps.sql b/tests/gaps/core/tests__receipt_gaps.sql index 62a9a8b..390501e 100644 --- a/tests/gaps/core/tests__receipt_gaps.sql +++ b/tests/gaps/core/tests__receipt_gaps.sql @@ -17,7 +17,7 @@ {% endset %} {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} {% else %} - {% set min_block_id = 9820210 %} + {% set min_block_id = 140868759 %} {% endif %} {% do log('Min block id: ' ~ min_block_id, info=True) %} {% endif %} diff --git a/tests/gaps/core/tests__tx_gaps.sql b/tests/gaps/core/tests__tx_gaps.sql index fb0d57f..5d3047c 100644 --- a/tests/gaps/core/tests__tx_gaps.sql +++ b/tests/gaps/core/tests__tx_gaps.sql @@ -17,7 +17,7 @@ {% endset %} {% set min_block_id = run_query(min_block_sql).columns[0].values()[0] %} {% else %} - {% set min_block_id = 9820210 %} + {% set min_block_id = 140868759 %} {% endif %} {% do log('Min block id: ' ~ min_block_id, info=True) %} {% endif %} From bc38c309d2d02587812e772ec84614dfd9c45c0e Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 17:03:15 -0600 Subject: [PATCH 42/53] upd test configs --- models/gold/core/core__fact_blocks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index b499f0a..2ec07de 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -33,7 +33,7 @@ models: tests: - not_null - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -297,9 +297,9 @@ models: description: "{{doc('id')}}" tests: - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: INSERTED_TIMESTAMP description: "{{doc('inserted_timestamp')}}" From 5381e0dc0cedacacd30e1fca054352c3c4e3432e Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 11 Mar 2025 18:23:21 -0600 Subject: [PATCH 43/53] slight test tweaks --- models/gold/core/core__fact_blocks.yml | 6 ++++-- models/gold/defi/defi__fact_intents.yml | 3 ++- models/silver/core/silver__receipts_final.yml | 1 - models/silver/core/silver__transactions_final.yml | 1 - 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index 2ec07de..5b2b750 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -154,7 +154,8 @@ models: - name: GAS_PRICE description: "{{ doc('gas_price')}}" tests: - - not_null + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER @@ -181,7 +182,8 @@ models: - name: LATEST_PROTOCOL_VERSION description: "{{ doc('latest_protocol_version')}}" tests: - - not_null + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER diff --git a/models/gold/defi/defi__fact_intents.yml b/models/gold/defi/defi__fact_intents.yml index c9ed012..d6b2150 100644 --- a/models/gold/defi/defi__fact_intents.yml +++ b/models/gold/defi/defi__fact_intents.yml @@ -65,7 +65,8 @@ models: - name: DIP4_VERSION description: "{{ doc('dip4_version') }}" tests: - - not_null + - not_null: + where: memo != 'refund' - name: GAS_BURNT description: "{{ doc('gas_burnt') }}" diff --git a/models/silver/core/silver__receipts_final.yml b/models/silver/core/silver__receipts_final.yml index cc62b37..0d8f2f7 100644 --- a/models/silver/core/silver__receipts_final.yml +++ b/models/silver/core/silver__receipts_final.yml @@ -86,7 +86,6 @@ models: description: "{{ doc('id') }}" tests: - not_null - - unique - name: inserted_timestamp description: "{{ doc('inserted_timestamp') }}" diff --git a/models/silver/core/silver__transactions_final.yml b/models/silver/core/silver__transactions_final.yml index 0a5cbe5..c728306 100644 --- a/models/silver/core/silver__transactions_final.yml +++ b/models/silver/core/silver__transactions_final.yml @@ -108,7 +108,6 @@ models: description: "{{ doc('id') }}" tests: - not_null - - unique - name: inserted_timestamp description: "{{ doc('inserted_timestamp') }}" From d4625f5a219bd9feb36a1a404a194340d2b20f2a Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 11:31:03 -0600 Subject: [PATCH 44/53] add datecast --- models/silver/core/silver__receipts_final.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index a93f210..c5335b4 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -104,7 +104,7 @@ blocks AS ( {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - WHERE block_timestamp :: DATE >= '{{min_bd}}' - interval '1 day' + WHERE block_timestamp :: DATE >= '{{min_bd}}' :: DATE - interval '1 day' {% endif %} {% endif %} ), From bacba9041937380f05609b44fe1721b1abc731e7 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 11:32:50 -0600 Subject: [PATCH 45/53] rm 1 day interval on blocks --- models/silver/core/silver__receipts_final.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index c5335b4..fbb29b4 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -104,7 +104,7 @@ blocks AS ( {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - WHERE block_timestamp :: DATE >= '{{min_bd}}' :: DATE - interval '1 day' + WHERE block_timestamp :: DATE >= '{{min_bd}}' :: DATE {% endif %} {% endif %} ), From a9c46bf4d459b406ae9c81ba5eeb3780e785044d Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 12:18:03 -0600 Subject: [PATCH 46/53] coalesce origin block timestamp --- models/silver/core/silver__receipts_final.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index fbb29b4..8a52b52 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -110,6 +110,7 @@ blocks AS ( ), flatten_receipts AS ( SELECT + origin_block_timestamp, chunk_hash, tx_hash, tx_succeeded, @@ -140,7 +141,7 @@ receipts_full AS ( chunk_hash, ro.block_hash, block_id, - block_timestamp, + COALESCE(block_timestamp, origin_block_timestamp) AS block_timestamp, r.tx_hash, r.receipt_id, receipt_json, From 1e31a46b637186b2d0eeb5030968b3dab7520c31 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:59:25 -0600 Subject: [PATCH 47/53] try dynamic_range_predicate_custom --- macros/dbt/get_merge.sql | 7 +- macros/dynamic_range_predicate_custom.sql | 117 ++++++++++++++++++ models/silver/core/silver__receipts_final.sql | 4 +- 3 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 macros/dynamic_range_predicate_custom.sql diff --git a/macros/dbt/get_merge.sql b/macros/dbt/get_merge.sql index e3c75ef..7961ef2 100644 --- a/macros/dbt/get_merge.sql +++ b/macros/dbt/get_merge.sql @@ -1,4 +1,9 @@ {% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} - {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {% if incremental_predicates[0] == "dynamic_range_predicate_custom" %} + {% set predicates = [dynamic_range_predicate_custom(source, incremental_predicates[1], "DBT_INTERNAL_DEST")] %} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, predicates) %} + {% else %} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {% endif %} {{ return(merge_sql) }} {% endmacro %} diff --git a/macros/dynamic_range_predicate_custom.sql b/macros/dynamic_range_predicate_custom.sql new file mode 100644 index 0000000..f11a8b7 --- /dev/null +++ b/macros/dynamic_range_predicate_custom.sql @@ -0,0 +1,117 @@ +{% macro dynamic_range_predicate_custom(source, predicate_column, output_alias="") -%} + {% set supported_data_types = ["INTEGER","DATE"] %} + {% set predicate_column_data_type_query %} + SELECT typeof({{predicate_column}}::variant) + FROM {{ source }} + WHERE {{predicate_column}} IS NOT NULL + LIMIT 1; + {% endset %} + {% set predicate_column_data_type_result = run_query(predicate_column_data_type_query) %} + {% if predicate_column_data_type_result.rows|length == 0 %} + {{ return('1=1') }} + {% endif %} + {% set predicate_column_data_type = predicate_column_data_type_result.columns[0].values()[0] %} + + + {% if predicate_column_data_type not in supported_data_types %} + {{ exceptions.raise_compiler_error("Data type of "~ predicate_column_data_type ~" is not supported, use one of "~ supported_data_types ~" column instead") }} + {% endif %} + + {% set get_start_end_query %} + SELECT + MIN( + {{ predicate_column }} + ) AS full_range_start, + MAX( + {{ predicate_column }} + ) AS full_range_end + FROM + {{ source }} + {% endset %} + {% set start_end_results = run_query(get_start_end_query).columns %} + {% set start_preciate_value = start_end_results[0].values()[0] %} + {% set end_predicate_value = start_end_results[1].values()[0] %} + + {% set get_limits_query %} + WITH block_range AS ( + {% if predicate_column_data_type == "INTEGER" %} + SELECT + SEQ4() + {{ start_preciate_value }} as predicate_value + FROM + TABLE(GENERATOR(rowcount => {{ end_predicate_value - start_preciate_value }}+1)) + {% else %} + SELECT + date_day as predicate_value + FROM + crosschain.core.dim_dates + WHERE + date_day BETWEEN '{{ start_preciate_value }}' AND '{{ end_predicate_value }}' + {% endif %} + ), + partition_block_counts AS ( + SELECT + b.predicate_value, + COUNT(r.{{ predicate_column }}) AS count_in_window + FROM + block_range b + LEFT OUTER JOIN {{ source }} + r + ON r.{{ predicate_column }} = b.predicate_value + GROUP BY + 1 + ), + range_groupings AS ( + SELECT + predicate_value, + count_in_window, + conditional_change_event( + count_in_window > 0 + ) over ( + ORDER BY + predicate_value + ) AS group_val + FROM + partition_block_counts + ), + contiguous_ranges AS ( + SELECT + MIN(predicate_value) AS start_value, + MAX(predicate_value) AS end_value + FROM + range_groupings + WHERE + count_in_window > 0 + GROUP BY + group_val + ), + between_stmts AS ( + SELECT + CONCAT( + '{{ output_alias~"." if output_alias else "" }}', + '{{ predicate_column }} between \'', + start_value, + '\' and \'', + end_value, + '\'' + ) AS b + FROM + contiguous_ranges + ) + SELECT + CONCAT('(', LISTAGG(b, ' OR '), ')', ' OR ', '(', '{{ output_alias~"." if output_alias else "" }}{{ predicate_column }} is null', ')') + FROM + between_stmts + {% endset %} + + {% set between_stmts = run_query(get_limits_query).columns[0].values()[0] %} + + {% if between_stmts != '()' %} + /* in case empty update array */ + {% set predicate_override = between_stmts %} + {% else %} + {% set predicate_override = '1=1' %} + /* need to have something or it will error since it expects at least 1 predicate */ + {% endif %} + + {{ return(predicate_override) }} +{% endmacro %} diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 8a52b52..807e15b 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - incremental_predicates = ["dynamic_range_predicate","block_timestamp::date"], + incremental_predicates = ["dynamic_range_predicate_custom","block_timestamp::date"], incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], unique_key = 'receipt_id', @@ -141,7 +141,7 @@ receipts_full AS ( chunk_hash, ro.block_hash, block_id, - COALESCE(block_timestamp, origin_block_timestamp) AS block_timestamp, + block_timestamp, r.tx_hash, r.receipt_id, receipt_json, From e2fd17f3f18e68b8c6f9f77ec32ea705ec8b0b2a Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 15:17:23 -0600 Subject: [PATCH 48/53] caps --- .../core/complete/streamline__transactions_complete.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/streamline/core/complete/streamline__transactions_complete.sql b/models/streamline/core/complete/streamline__transactions_complete.sql index 94c22f9..262615f 100644 --- a/models/streamline/core/complete/streamline__transactions_complete.sql +++ b/models/streamline/core/complete/streamline__transactions_complete.sql @@ -11,7 +11,7 @@ SELECT VALUE :TX_HASH :: STRING AS tx_hash, - VALUE :chunk_hash :: STRING AS chunk_hash, + VALUE :CHUNK_HASH :: STRING AS chunk_hash, VALUE :BLOCK_ID :: INT AS block_id, partition_key, _inserted_timestamp, From 4f654ab523d406fdeb9a774b48864ffa2e170c1f Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 15:25:20 -0600 Subject: [PATCH 49/53] staking epochs incr logic --- .../silver/curated/staking/silver__staking_epochs.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/models/silver/curated/staking/silver__staking_epochs.sql b/models/silver/curated/staking/silver__staking_epochs.sql index d1b48e2..5203e4d 100644 --- a/models/silver/curated/staking/silver__staking_epochs.sql +++ b/models/silver/curated/staking/silver__staking_epochs.sql @@ -13,11 +13,18 @@ WITH pool_events AS ( * FROM {{ ref('silver__pool_events') }} - WHERE + {% if var("MANUAL_FIX") %} + WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {{ incremental_load_filter('_inserted_timestamp') }} + WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) {% endif %} AND LOG LIKE 'Epoch%' ), From e5ac376aa761d0f5c5db7d6a7ce3123105408894 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 16:04:35 -0600 Subject: [PATCH 50/53] upd project version --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index e0ec03c..8196a51 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: "near_models" -version: "1.3.0" +version: "2.0.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. From 7402abf3528eef55b2507001afc1d751872473d3 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 16:18:40 -0600 Subject: [PATCH 51/53] mv dbt runs to 1 line --- .github/workflows/dbt_run_scheduled_core.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_core.yml b/.github/workflows/dbt_run_scheduled_core.yml index 13dd2da..cc1b922 100644 --- a/.github/workflows/dbt_run_scheduled_core.yml +++ b/.github/workflows/dbt_run_scheduled_core.yml @@ -43,8 +43,7 @@ jobs: - name: Run DBT Jobs run: | dbt seed; - dbt run -s tag:scheduled_core; - dbt run -s tag:scheduled_non_core models/gold; + dbt run -s tag:scheduled_core tag:scheduled_non_core models/gold; - name: Store logs uses: actions/upload-artifact@v4 with: From 49cd3e60de8cf290d9bf3cb577b49ce2861143df Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 14 Mar 2025 16:24:32 -0600 Subject: [PATCH 52/53] upd partition macro --- macros/manual_batch_refresh.sql | 9 +++++---- models/silver/core/silver__receipts_final.sql | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/macros/manual_batch_refresh.sql b/macros/manual_batch_refresh.sql index 722f59e..ee06713 100644 --- a/macros/manual_batch_refresh.sql +++ b/macros/manual_batch_refresh.sql @@ -1,5 +1,6 @@ {% macro partition_load_manual( - scope = 'no_buffer' + scope = 'no_buffer', + partition_field = '_partition_by_block_number' ) %} {# if range_start and range_end not set in cli, default to earliest rpc data #} {% set range_start = var( @@ -19,17 +20,17 @@ 1 ) %} {% if scope == 'front' %} - _partition_by_block_number BETWEEN {{ range_start }} - ( + {{ partition_field }} BETWEEN {{ range_start }} - ( 10000 * {{ front_buffer }} ) AND {{ range_end }} {% elif scope == 'end' %} - _partition_by_block_number BETWEEN {{ range_start }} + {{ partition_field }} BETWEEN {{ range_start }} AND {{ range_end }} + ( 10000 * {{ end_buffer }} ) {% elif scope == 'no_buffer' %} - _partition_by_block_number BETWEEN {{ range_start }} + {{ partition_field }} BETWEEN {{ range_start }} AND {{ range_end }} {% else %} TRUE diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 807e15b..2fbe960 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -84,7 +84,7 @@ WITH txs_with_receipts AS ( {{ ref('silver__transactions_v2') }} {% if var("MANUAL_FIX") %} WHERE - {{ partition_load_manual('no_buffer') }} + {{ partition_load_manual('no_buffer', 'partition_key') }} {% else %} {% if is_incremental() %} WHERE origin_block_timestamp :: DATE >= '{{min_bd}}' @@ -101,7 +101,7 @@ blocks AS ( {{ ref('silver__blocks_v2') }} {% if var("MANUAL_FIX") %} WHERE - {{ partition_load_manual('no_buffer') }} + {{ partition_load_manual('no_buffer', 'partition_key') }} {% else %} {% if is_incremental() %} WHERE block_timestamp :: DATE >= '{{min_bd}}' :: DATE From d81412781a97b7bcb5a696d7d9ed4e2ae9bc4d74 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:43:34 -0600 Subject: [PATCH 53/53] construct initial receipt record --- models/silver/core/silver__receipts_final.sql | 61 +++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/models/silver/core/silver__receipts_final.sql b/models/silver/core/silver__receipts_final.sql index 2fbe960..cdc26d7 100644 --- a/models/silver/core/silver__receipts_final.sql +++ b/models/silver/core/silver__receipts_final.sql @@ -75,8 +75,8 @@ WITH txs_with_receipts AS ( origin_block_id, origin_block_timestamp, tx_hash, - response_json :receipts :: ARRAY AS receipts_json, - response_json :receipts_outcome :: ARRAY AS receipts_outcome_json, + response_json, + response_json :transaction_outcome :outcome :receipt_ids [0] :: STRING AS initial_receipt_id, response_json :status :Failure IS NULL AS tx_succeeded, partition_key AS _partition_by_block_number, modified_timestamp @@ -121,7 +121,7 @@ flatten_receipts AS ( FROM txs_with_receipts, LATERAL FLATTEN( - input => receipts_json + input => response_json :receipts :: ARRAY ) ), flatten_receipt_outcomes AS ( @@ -133,7 +133,7 @@ flatten_receipt_outcomes AS ( FROM txs_with_receipts, LATERAL FLATTEN( - input => receipts_outcome_json + input => response_json :receipts_outcome :: ARRAY ) ), receipts_full AS ( @@ -162,6 +162,43 @@ receipts_full AS ( ) >= '{{max_mod}}' {% endif %} ), +initial_receipt_full AS ( + SELECT + chunk_hash, + origin_block_id AS block_id, + origin_block_timestamp AS block_timestamp, + txr.tx_hash, + initial_receipt_id AS receipt_id, + OBJECT_CONSTRUCT( + 'predecessor_id', response_json :transaction :signer_id :: STRING, + 'priority', response_json :transaction :priority_fee :: INTEGER, + 'receipt', OBJECT_CONSTRUCT( + 'Action', OBJECT_CONSTRUCT( + 'actions', response_json :transaction :actions :: ARRAY, + 'gas_price', Null, + 'input_data_ids', Null, + 'is_promise_yield', Null, + 'output_data_receivers', Null, + 'signer_id', response_json :transaction :signer_id :: STRING, + 'signer_public_key', response_json :transaction :public_key :: STRING + ) + ), + 'receipt_id', initial_receipt_id :: STRING, + 'receiver_id', response_json :transaction :receiver_id :: STRING + ) AS receipt_json, + outcome_json, + tx_succeeded, + _partition_by_block_number + FROM + txs_with_receipts txr + LEFT JOIN flatten_receipt_outcomes ro + ON txr.initial_receipt_id = ro.receipt_id + AND txr.tx_hash = ro.tx_hash + {% if is_incremental() and not var("MANUAL_FIX") %} + WHERE + modified_timestamp >= '{{max_mod}}' + {% endif %} +), FINAL AS ( SELECT chunk_hash, @@ -178,6 +215,22 @@ FINAL AS ( _partition_by_block_number FROM receipts_full + UNION ALL + SELECT + chunk_hash, + block_id, + block_timestamp, + tx_hash, + receipt_id, + receipt_json :predecessor_id :: STRING AS predecessor_id, + receipt_json :receiver_id :: STRING AS receiver_id, + receipt_json, + outcome_json, + tx_succeeded, + tx_succeeded AS receipt_succeeded, + _partition_by_block_number + FROM + initial_receipt_full ) SELECT *,