diff --git a/models/bronze/core/bronze__messages.sql b/models/bronze/core/bronze__messages.sql index 1b586ed..55fbf67 100644 --- a/models/bronze/core/bronze__messages.sql +++ b/models/bronze/core/bronze__messages.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'view' + materialized = 'view', + enabled = false ) }} {{ streamline_external_table_FR_query_v2( model = 'messages_tdl', diff --git a/models/gold/core/core__fact_account_states.sql b/models/gold/core/core__fact_account_states.sql index 2bb2f3e..90e909e 100644 --- a/models/gold/core/core__fact_account_states.sql +++ b/models/gold/core/core__fact_account_states.sql @@ -9,8 +9,8 @@ ) }} SELECT - account, TO_TIMESTAMP(TIMESTAMP) AS block_timestamp, + account, TIMESTAMP, last_trans_lt, last_trans_hash AS last_tx_hash, diff --git a/models/gold/core/core__fact_balances.sql b/models/gold/core/core__fact_balances.sql index c028167..71ec467 100644 --- a/models/gold/core/core__fact_balances.sql +++ b/models/gold/core/core__fact_balances.sql @@ -9,8 +9,8 @@ ) }} SELECT - address, TO_TIMESTAMP(TIMESTAMP) AS block_timestamp, + address, asset, amount, mintless_claimed, diff --git a/models/gold/core/core__fact_jetton_events.sql b/models/gold/core/core__fact_jetton_events.sql index 3a11d3b..7b11012 100644 --- a/models/gold/core/core__fact_jetton_events.sql +++ b/models/gold/core/core__fact_jetton_events.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'incremental', - unique_key = ['jetton_events_id'], + unique_key = ['fact_jetton_events_id'], incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], cluster_by = ['block_timestamp::DATE','source','destination'], diff --git a/models/gold/core/core__fact_jetton_metadata.sql b/models/gold/core/core__fact_jetton_metadata.sql new file mode 100644 index 0000000..c0ce4fd --- /dev/null +++ b/models/gold/core/core__fact_jetton_metadata.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_jetton_metadata_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['update_timestamp_onchain::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address,symbol);", + tags = ['scheduled_core'] +) }} + +SELECT + address, + TO_TIMESTAMP(update_time_onchain) AS update_timestamp_onchain, + symbol, + decimals, + NAME, + description, + tonapi_image_url, + image_data, + image, + admin_address, + mintable, + jetton_content_onchain, + jetton_wallet_code_hash, + code_hash, + adding_at, + sources, + metadata_status, + update_time_onchain, + update_time_metadata, + jetton_metadata_id AS fact_jetton_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__jetton_metadata') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_messages.sql b/models/gold/core/core__fact_messages.sql new file mode 100644 index 0000000..fbfb75f --- /dev/null +++ b/models/gold/core/core__fact_messages.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_messages_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,msg_hash,source,destination);", + tags = ['scheduled_core'] +) }} + +SELECT + TO_TIMESTAMP(tx_now) AS block_timestamp, + tx_hash, + msg_hash, + body_hash, + trace_id, + direction, + source, + destination, + _VALUE AS VALUE, + opcode, + created_at, + tx_now, + ihr_fee, + import_fee, + fwd_fee, + ihr_disabled, + bounced, + bounce, + COMMENT, + tx_lt, + created_lt, + init_state_hash, + init_state_boc, + body_boc, + messages_with_data_id AS fact_messages_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__messages_with_data') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql new file mode 100644 index 0000000..22ae25a --- /dev/null +++ b/models/gold/core/core__fact_transactions.sql @@ -0,0 +1,88 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_transactions_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,prev_tx_hash,account);", + tags = ['scheduled_core'] +) }} + +SELECT + TO_TIMESTAMP(now) AS block_timestamp, + HASH AS tx_hash, + prev_trans_hash AS prev_tx_hash, + CASE + WHEN aborted THEN FALSE + ELSE TRUE + END tx_succeeded, + aborted, + account, + orig_status, + end_status, + compute_success, + compute_skipped, + compute_gas_fees, + action_result_code, + action_success, + action_spec_actions, + action_result_arg, + action_skipped_actions, + action_valid, + action_tot_actions, + action_no_funds, + action_status_change, + compute_msg_state_used, + descr, + block_workchain, + block_seqno, + block_shard, + mc_block_seqno, + total_fees, + storage_fees_collected, + credit_due_fees_collected, + action_total_fwd_fees, + storage_fees_due, + action_total_action_fees, + account_state_balance_before, + account_state_balance_after, + account_state_hash_before, + account_state_hash_after, + account_state_code_hash_before, + account_state_code_hash_after, + installed, + destroyed, + is_tock, + credit_first, + compute_account_activated, + compute_vm_steps, + compute_exit_arg, + compute_gas_credit, + compute_gas_limit, + compute_gas_used, + compute_vm_init_state_hash, + compute_vm_final_state_hash, + skipped_reason, + compute_exit_code, + storage_status_change, + compute_mode, + credit, + trace_id, + lt, + prev_trans_lt, + now, + transactions_id AS fact_transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__transactions') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/defi/defi__fact_dex_pools.sql b/models/gold/defi/defi__fact_dex_pools.sql similarity index 100% rename from models/gold/core/defi/defi__fact_dex_pools.sql rename to models/gold/defi/defi__fact_dex_pools.sql diff --git a/models/gold/core/defi/defi__fact_dex_trades.sql b/models/gold/defi/defi__fact_dex_trades.sql similarity index 90% rename from models/gold/core/defi/defi__fact_dex_trades.sql rename to models/gold/defi/defi__fact_dex_trades.sql index 213abbe..99c4c92 100644 --- a/models/gold/core/defi/defi__fact_dex_trades.sql +++ b/models/gold/defi/defi__fact_dex_trades.sql @@ -3,13 +3,13 @@ unique_key = ['fact_dex_trades_id'], incremental_strategy = 'merge', merge_exclude_columns = ['inserted_timestamp'], - cluster_by = ['block_timestamp_event::DATE'], + cluster_by = ['block_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,pool_address,trader_address,token_bought_address,token_sold_address);", tags = ['scheduled_core'] ) }} SELECT - TO_TIMESTAMP(event_time) AS block_timestamp_event, + TO_TIMESTAMP(event_time) AS block_timestamp, event_type, tx_hash, project_type, diff --git a/models/gold/nft/nft__fact_nft_events.sql b/models/gold/nft/nft__fact_nft_events.sql new file mode 100644 index 0000000..a66bd07 --- /dev/null +++ b/models/gold/nft/nft__fact_nft_events.sql @@ -0,0 +1,57 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_nft_events_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,nft_item_address,sale_contract,marketplace_address );", + tags = ['scheduled_core'] +) }} + +SELECT + TO_TIMESTAMP(TIMESTAMP) AS block_timestamp, + tx_hash, + TYPE, + sale_type, + nft_item_address, + nft_item_index, + payment_asset, + sale_price, + forward_amount, + royalty_amount, + marketplace_fee, + auction_max_bid, + auction_min_bid, + sale_contract, + royalty_address, + marketplace_address, + marketplace_fee_address, + owner_address, + collection_address, + content_onchain, + trace_id, + query_id, + is_init, + custom_payload, + COMMENT, + sale_end_time, + forward_payload, + auction_min_step, + prev_owner, + TIMESTAMP, + lt, + nft_events_id AS fact_nft_events_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__nft_events') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/nft/nft__fact_nft_items.sql b/models/gold/nft/nft__fact_nft_items.sql new file mode 100644 index 0000000..80d0d48 --- /dev/null +++ b/models/gold/nft/nft__fact_nft_items.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_nft_items_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address,collection_address,owner_address);", + tags = ['scheduled_core'] +) }} + +SELECT + TO_TIMESTAMP(TIMESTAMP) AS block_timestamp, + address, + INDEX, + collection_address, + owner_address, + content_onchain, + is_init, + lt, + TIMESTAMP, + nft_items_id AS fact_nft_items_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__nft_items') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/nft/nft__fact_nft_metadata.sql b/models/gold/nft/nft__fact_nft_metadata.sql new file mode 100644 index 0000000..eab06ad --- /dev/null +++ b/models/gold/nft/nft__fact_nft_metadata.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_nft_metadata_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['update_timestamp_onchain::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address,parent_address);", + tags = ['scheduled_core'] +) }} + +SELECT + address, + parent_address, + TO_TIMESTAMP(update_time_onchain) AS update_timestamp_onchain, + TYPE, + NAME, + description, + metadata_status, + attributes, + image, + image_data, + tonapi_image_url, + content_onchain, + sources, + update_time_onchain, + update_time_metadata, + adding_at, + nft_metadata_id AS fact_nft_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__nft_metadata') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/nft/nft__fact_nft_sales.sql b/models/gold/nft/nft__fact_nft_sales.sql new file mode 100644 index 0000000..eea018e --- /dev/null +++ b/models/gold/nft/nft__fact_nft_sales.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_nft_sales_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address,nft_address,asset,nft_owner_address);", + tags = ['scheduled_core'] +) }} + +SELECT + TO_TIMESTAMP(TIMESTAMP) AS block_timestamp, + address, + nft_address, + TYPE, + asset, + price, + marketplace_fee, + royalty_amount, + nft_owner_address, + marketplace_address, + marketplace_fee_address, + royalty_address, + is_complete, + is_canceled, + min_bid, + max_bid, + min_step, + end_time, + last_bid_at, + last_member, + created_at, + TIMESTAMP, + lt, + nft_sales_id AS fact_nft_sales_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__nft_sales') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/nft/nft__fact_nft_transfers.sql b/models/gold/nft/nft__fact_nft_transfers.sql new file mode 100644 index 0000000..96fd4b6 --- /dev/null +++ b/models/gold/nft/nft__fact_nft_transfers.sql @@ -0,0 +1,47 @@ +{{ config( + materialized = 'incremental', + unique_key = ['fact_nft_transfers_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,old_owner,new_owner,nft_collection_address,nft_item_address);", + tags = ['scheduled_core'] +) }} + +SELECT + TO_TIMESTAMP(tx_now) AS block_timestamp, + tx_hash, + CASE + WHEN tx_aborted THEN FALSE + ELSE TRUE + END tx_succeeded, + tx_aborted, + old_owner, + new_owner, + nft_collection_address, + nft_item_address, + nft_item_index, + response_destination, + forward_amount, + custom_payload, + forward_payload, + COMMENT, + trace_id, + query_id, + tx_now, + tx_lt, + nft_transfers_id AS fact_nft_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__nft_transfers') }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/core/silver__messages.sql b/models/silver/core/silver__messages.sql index 0c61c85..17eee24 100644 --- a/models/silver/core/silver__messages.sql +++ b/models/silver/core/silver__messages.sql @@ -5,7 +5,8 @@ begin = '2024-01-01', batch_size = 'day', cluster_by = ['block_date::DATE','modified_timestamp::DATE'], - tags = ['scheduled_core'] + tags = ['scheduled_core'], + enabled = false ) }} WITH pre_final AS (