flip/stellar-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/stellar-models.git synced 2026-02-06 13:26:52 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

Silver incremental logic (#2)

Browse Source 
* check in

* check in, gte in progress

* bronze & silver
This commit is contained in:
eric-laurello 2025-02-04 16:12:04 -05:00 committed by Mike Stepanovic
parent 6e64c2bc77
commit 420df69933
23 changed files with 486 additions and 546 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
macros/streamline/models.sql
View File

@ -22,8 +22,7 @@ SELECT
{{ other_cols }},
value,
_inserted_timestamp,
s.{{ partition_name }},
s.value AS VALUE
s.{{ partition_name }}
FROM
{{ source(
"bronze_streamline",
@ -63,8 +62,7 @@ SELECT
{{ other_cols }},
value,
_inserted_timestamp,
s.{{ partition_name }},
s.value AS VALUE
s.{{ partition_name }}
FROM
{{ source(
"bronze_streamline",

6
models/bronze/core/bronze__accounts.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_query_v2(
model = "accounts",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "ACCOUNT_ID",
other_cols = "BALANCE, BUYING_LIABILITIES, SELLING_LIABILITIES, SEQUENCE_NUMBER, NUM_SUBENTRIES, INFLATION_DESTINATION, FLAGS, HOME_DOMAIN, MASTER_WEIGHT, THRESHOLD_LOW, THRESHOLD_MEDIUM, THRESHOLD_HIGH, LAST_MODIFIED_LEDGER, LEDGER_ENTRY_CHANGE, DELETED, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SPONSOR, NUM_SPONSORED, NUM_SPONSORING, SEQUENCE_LEDGER, SEQUENCE_TIME, CLOSED_AT, LEDGER_SEQUENCE"
other_cols = "partition_id"
) }}

6
models/bronze/core/bronze__accounts_FR.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_FR_query_v2(
model = "accounts",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "ACCOUNT_ID",
other_cols = "BALANCE, BUYING_LIABILITIES, SELLING_LIABILITIES, SEQUENCE_NUMBER, NUM_SUBENTRIES, INFLATION_DESTINATION, FLAGS, HOME_DOMAIN, MASTER_WEIGHT, THRESHOLD_LOW, THRESHOLD_MEDIUM, THRESHOLD_HIGH, LAST_MODIFIED_LEDGER, LEDGER_ENTRY_CHANGE, DELETED, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SPONSOR, NUM_SPONSORED, NUM_SPONSORING, SEQUENCE_LEDGER, SEQUENCE_TIME, CLOSED_AT, LEDGER_SEQUENCE"
other_cols = "partition_id"
) }}

8
models/bronze/core/bronze__assets.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_query_v2(
model = "history_assets",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
unique_key = "ID",
other_cols = "ASSET_TYPE, ASSET_CODE, ASSET_ISSUER, BATCH_RUN_DATE, BATCH_ID, BATCH_INSERT_TS, ASSET_ID"
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "asset_id",
other_cols = "partition_id"
) }}

8
models/bronze/core/bronze__assets_FR.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_FR_query_v2(
model = "history_assets",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
unique_key = "ID",
other_cols = " ASSET_TYPE, ASSET_CODE, ASSET_ISSUER, BATCH_RUN_DATE, BATCH_ID, BATCH_INSERT_TS, ASSET_ID"
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "asset_id",
other_cols = "partition_id"
) }}

7
models/bronze/core/bronze__ledgers.sql
View File

@ -3,8 +3,9 @@
) }}
{{ streamline_external_table_query_v2(
model = "history_ledgers",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "SEQUENCE",
other_cols = "LEDGER_HASH, PREVIOUS_LEDGER_HASH, TRANSACTION_COUNT, OPERATION_COUNT, CLOSED_AT, ID, TOTAL_COINS, FEE_POOL, BASE_FEE, BASE_RESERVE, MAX_TX_SET_SIZE, PROTOCOL_VERSION, LEDGER_HEADER, SUCCESSFUL_TRANSACTION_COUNT, FAILED_TRANSACTION_COUNT, TX_SET_OPERATION_COUNT, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SOROBAN_FEE_WRITE_1KB, NODE_ID, SIGNATURE, TOTAL_BYTE_SIZE_OF_BUCKET_LIST"
other_cols = "partition_id, LEDGER_HASH, PREVIOUS_LEDGER_HASH, TRANSACTION_COUNT, OPERATION_COUNT, CLOSED_AT, ID, TOTAL_COINS, FEE_POOL, BASE_FEE, BASE_RESERVE, MAX_TX_SET_SIZE, PROTOCOL_VERSION, SUCCESSFUL_TRANSACTION_COUNT, FAILED_TRANSACTION_COUNT, TX_SET_OPERATION_COUNT, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SOROBAN_FEE_WRITE_1KB, NODE_ID, SIGNATURE, TOTAL_BYTE_SIZE_OF_BUCKET_LIST"
) }}
--LEDGER_HEADER,

7
models/bronze/core/bronze__ledgers_FR.sql
View File

@ -3,8 +3,9 @@
) }}
{{ streamline_external_table_FR_query_v2(
model = "history_ledgers",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "SEQUENCE",
other_cols = "LEDGER_HASH, PREVIOUS_LEDGER_HASH, TRANSACTION_COUNT, OPERATION_COUNT, CLOSED_AT, ID, TOTAL_COINS, FEE_POOL, BASE_FEE, BASE_RESERVE, MAX_TX_SET_SIZE, PROTOCOL_VERSION, LEDGER_HEADER, SUCCESSFUL_TRANSACTION_COUNT, FAILED_TRANSACTION_COUNT, TX_SET_OPERATION_COUNT, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SOROBAN_FEE_WRITE_1KB, NODE_ID, SIGNATURE, TOTAL_BYTE_SIZE_OF_BUCKET_LIST"
other_cols = "partition_id, LEDGER_HASH, PREVIOUS_LEDGER_HASH, TRANSACTION_COUNT, OPERATION_COUNT, CLOSED_AT, ID, TOTAL_COINS, FEE_POOL, BASE_FEE, BASE_RESERVE, MAX_TX_SET_SIZE, PROTOCOL_VERSION, SUCCESSFUL_TRANSACTION_COUNT, FAILED_TRANSACTION_COUNT, TX_SET_OPERATION_COUNT, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SOROBAN_FEE_WRITE_1KB, NODE_ID, SIGNATURE, TOTAL_BYTE_SIZE_OF_BUCKET_LIST"
) }}
--LEDGER_HEADER,

6
models/bronze/core/bronze__liquidity_pools.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_query_v2(
model = "liquidity_pools",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "LIQUIDITY_POOL_ID",
other_cols = "TYPE, FEE, TRUSTLINE_COUNT, POOL_SHARE_COUNT, ASSET_A_TYPE, ASSET_A_CODE, ASSET_A_ISSUER, ASSET_A_ID, ASSET_A_AMOUNT, ASSET_B_TYPE, ASSET_B_CODE, ASSET_B_ISSUER, ASSET_B_ID, ASSET_B_AMOUNT, LAST_MODIFIED_LEDGER, LEDGER_ENTRY_CHANGE, DELETED, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, CLOSED_AT, LEDGER_SEQUENCE"
other_cols = "partition_id"
) }}

6
models/bronze/core/bronze__liquidity_pools_FR.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_FR_query_v2(
model = "liquidity_pools",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "LIQUIDITY_POOL_ID",
other_cols = "TYPE, FEE, TRUSTLINE_COUNT, POOL_SHARE_COUNT, ASSET_A_TYPE, ASSET_A_CODE, ASSET_A_ISSUER, ASSET_A_ID, ASSET_A_AMOUNT, ASSET_B_TYPE, ASSET_B_CODE, ASSET_B_ISSUER, ASSET_B_ID, ASSET_B_AMOUNT, LAST_MODIFIED_LEDGER, LEDGER_ENTRY_CHANGE, DELETED, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, CLOSED_AT, LEDGER_SEQUENCE"
other_cols = "partition_id"
) }}

10
models/bronze/core/bronze__operations.sql
View File

@ -2,9 +2,9 @@
materialized = 'view'
) }}
{{ streamline_external_table_query_v2(
model = "enriched_history_operations",
partition_function = "TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
unique_key = "OP_ID",
other_cols = 'ACCOUNT, AMOUNT, ASSET_CODE, ASSET_ISSUER, ASSET_TYPE, ASSET_ID, AUTHORIZE, BALANCE_ID, BUYING_ASSET_CODE, BUYING_ASSET_ISSUER, BUYING_ASSET_TYPE, BUYING_ASSET_ID,"from", FUNDER, HIGH_THRESHOLD, HOME_DOMAIN, INFLATION_DEST,"into","limit", LOW_THRESHOLD, MASTER_KEY_WEIGHT, MED_THRESHOLD, NAME, OFFER_ID, PATH, PRICE, D, N, SELLING_ASSET_CODE, SELLING_ASSET_ISSUER, SELLING_ASSET_TYPE, SELLING_ASSET_ID, SET_FLAGS, SET_FLAGS_S, SIGNER_KEY, SIGNER_WEIGHT, SOURCE_AMOUNT, SOURCE_ASSET_CODE, SOURCE_ASSET_ISSUER, SOURCE_ASSET_TYPE, SOURCE_ASSET_ID, SOURCE_MAX, STARTING_BALANCE,"to", TRUSTEE, TRUSTOR, TRUSTLINE_ASSET,"value", CLEAR_FLAGS, CLEAR_FLAGS_S, DESTINATION_MIN, BUMP_TO, SPONSOR, SPONSORED_ID, BEGIN_SPONSOR, AUTHORIZE_TO_MAINTAIN_LIABILITIES, CLAWBACK_ENABLED, LIQUIDITY_POOL_ID, RESERVE_A_ASSET_TYPE, RESERVE_A_ASSET_ID, RESERVE_A_ASSET_CODE, RESERVE_A_ASSET_ISSUER, RESERVE_A_MAX_AMOUNT, RESERVE_A_DEPOSIT_AMOUNT, RESERVE_B_ASSET_TYPE, RESERVE_B_ASSET_ID, RESERVE_B_ASSET_CODE, RESERVE_B_ASSET_ISSUER, RESERVE_B_MAX_AMOUNT, RESERVE_B_DEPOSIT_AMOUNT, MIN_PRICE, MIN_PRICE_R, MAX_PRICE, MAX_PRICE_R, SHARES_RECEIVED, RESERVE_A_MIN_AMOUNT, RESERVE_B_MIN_AMOUNT, SHARES, RESERVE_A_WITHDRAW_AMOUNT, RESERVE_B_WITHDRAW_AMOUNT, OP_SOURCE_ACCOUNT, OP_SOURCE_ACCOUNT_MUXED, TRANSACTION_ID, TYPE, TRANSACTION_HASH, LEDGER_SEQUENCE, TXN_ACCOUNT, ACCOUNT_SEQUENCE, MAX_FEE, TXN_OPERATION_COUNT, TXN_CREATED_AT, MEMO_TYPE, MEMO, TIME_BOUNDS, SUCCESSFUL, FEE_CHARGED, FEE_ACCOUNT, NEW_MAX_FEE, ACCOUNT_MUXED, FEE_ACCOUNT_MUXED, LEDGER_HASH, PREVIOUS_LEDGER_HASH, TRANSACTION_COUNT, LEDGER_OPERATION_COUNT, CLOSED_AT, LEDGER_ID, TOTAL_COINS, FEE_POOL, BASE_FEE, BASE_RESERVE, MAX_TX_SET_SIZE, PROTOCOL_VERSION, SUCCESSFUL_TRANSACTION_COUNT, FAILED_TRANSACTION_COUNT, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, LEDGER_BOUNDS, MIN_ACCOUNT_SEQUENCE, MIN_ACCOUNT_SEQUENCE_AGE, MIN_ACCOUNT_SEQUENCE_LEDGER_GAP, EXTRA_SIGNERS, ASSET_BALANCE_CHANGES, PARAMETERS, PARAMETERS_DECODED, FUNCTION, ADDRESS, SOROBAN_OPERATION_TYPE, EXTEND_TO, CONTRACT_ID, CONTRACT_CODE_HASH, RESOURCE_FEE, SOROBAN_RESOURCES_INSTRUCTIONS, SOROBAN_RESOURCES_READ_BYTES, SOROBAN_RESOURCES_WRITE_BYTES, TRANSACTION_RESULT_CODE, INCLUSION_FEE_BID, INCLUSION_FEE_CHARGED, RESOURCE_FEE_REFUND, OPERATION_RESULT_CODE, OPERATION_TRACE_CODE, OP_APPLICATION_ORDER, TXN_APPLICATION_ORDER'
model = "history_operations",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "ID",
other_cols = 'partition_id'
) }}

10
models/bronze/core/bronze__operations_FR.sql
View File

@ -2,9 +2,9 @@
materialized = 'view'
) }}
{{ streamline_external_table_FR_query_v2(
model = "enriched_history_operations",
partition_function = "TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
unique_key = "OP_ID",
other_cols = 'ACCOUNT, AMOUNT, ASSET_CODE, ASSET_ISSUER, ASSET_TYPE, ASSET_ID, AUTHORIZE, BALANCE_ID, BUYING_ASSET_CODE, BUYING_ASSET_ISSUER, BUYING_ASSET_TYPE, BUYING_ASSET_ID,"from", FUNDER, HIGH_THRESHOLD, HOME_DOMAIN, INFLATION_DEST,"into","limit", LOW_THRESHOLD, MASTER_KEY_WEIGHT, MED_THRESHOLD, NAME, OFFER_ID, PATH, PRICE, D, N, SELLING_ASSET_CODE, SELLING_ASSET_ISSUER, SELLING_ASSET_TYPE, SELLING_ASSET_ID, SET_FLAGS, SET_FLAGS_S, SIGNER_KEY, SIGNER_WEIGHT, SOURCE_AMOUNT, SOURCE_ASSET_CODE, SOURCE_ASSET_ISSUER, SOURCE_ASSET_TYPE, SOURCE_ASSET_ID, SOURCE_MAX, STARTING_BALANCE,"to", TRUSTEE, TRUSTOR, TRUSTLINE_ASSET,"value", CLEAR_FLAGS, CLEAR_FLAGS_S, DESTINATION_MIN, BUMP_TO, SPONSOR, SPONSORED_ID, BEGIN_SPONSOR, AUTHORIZE_TO_MAINTAIN_LIABILITIES, CLAWBACK_ENABLED, LIQUIDITY_POOL_ID, RESERVE_A_ASSET_TYPE, RESERVE_A_ASSET_ID, RESERVE_A_ASSET_CODE, RESERVE_A_ASSET_ISSUER, RESERVE_A_MAX_AMOUNT, RESERVE_A_DEPOSIT_AMOUNT, RESERVE_B_ASSET_TYPE, RESERVE_B_ASSET_ID, RESERVE_B_ASSET_CODE, RESERVE_B_ASSET_ISSUER, RESERVE_B_MAX_AMOUNT, RESERVE_B_DEPOSIT_AMOUNT, MIN_PRICE, MIN_PRICE_R, MAX_PRICE, MAX_PRICE_R, SHARES_RECEIVED, RESERVE_A_MIN_AMOUNT, RESERVE_B_MIN_AMOUNT, SHARES, RESERVE_A_WITHDRAW_AMOUNT, RESERVE_B_WITHDRAW_AMOUNT, OP_SOURCE_ACCOUNT, OP_SOURCE_ACCOUNT_MUXED, TRANSACTION_ID, TYPE, TRANSACTION_HASH, LEDGER_SEQUENCE, TXN_ACCOUNT, ACCOUNT_SEQUENCE, MAX_FEE, TXN_OPERATION_COUNT, TXN_CREATED_AT, MEMO_TYPE, MEMO, TIME_BOUNDS, SUCCESSFUL, FEE_CHARGED, FEE_ACCOUNT, NEW_MAX_FEE, ACCOUNT_MUXED, FEE_ACCOUNT_MUXED, LEDGER_HASH, PREVIOUS_LEDGER_HASH, TRANSACTION_COUNT, LEDGER_OPERATION_COUNT, CLOSED_AT, LEDGER_ID, TOTAL_COINS, FEE_POOL, BASE_FEE, BASE_RESERVE, MAX_TX_SET_SIZE, PROTOCOL_VERSION, SUCCESSFUL_TRANSACTION_COUNT, FAILED_TRANSACTION_COUNT, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, LEDGER_BOUNDS, MIN_ACCOUNT_SEQUENCE, MIN_ACCOUNT_SEQUENCE_AGE, MIN_ACCOUNT_SEQUENCE_LEDGER_GAP, EXTRA_SIGNERS, ASSET_BALANCE_CHANGES, PARAMETERS, PARAMETERS_DECODED, FUNCTION, ADDRESS, SOROBAN_OPERATION_TYPE, EXTEND_TO, CONTRACT_ID, CONTRACT_CODE_HASH, RESOURCE_FEE, SOROBAN_RESOURCES_INSTRUCTIONS, SOROBAN_RESOURCES_READ_BYTES, SOROBAN_RESOURCES_WRITE_BYTES, TRANSACTION_RESULT_CODE, INCLUSION_FEE_BID, INCLUSION_FEE_CHARGED, RESOURCE_FEE_REFUND, OPERATION_RESULT_CODE, OPERATION_TRACE_CODE, OP_APPLICATION_ORDER, TXN_APPLICATION_ORDER'
model = "history_operations",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "ID",
other_cols = 'partition_id'
) }}

6
models/bronze/core/bronze__trades.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_query_v2(
model = "history_trades",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "HISTORY_OPERATION_ID",
other_cols = '"order", LEDGER_CLOSED_AT, SELLING_ACCOUNT_ADDRESS, SELLING_ASSET_CODE, SELLING_ASSET_ISSUER, SELLING_ASSET_TYPE, SELLING_ASSET_ID, SELLING_AMOUNT, BUYING_ACCOUNT_ADDRESS, BUYING_ASSET_CODE, BUYING_ASSET_ISSUER, BUYING_ASSET_TYPE, BUYING_ASSET_ID, BUYING_AMOUNT, PRICE_N, PRICE_D, SELLING_OFFER_ID, BUYING_OFFER_ID, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SELLING_LIQUIDITY_POOL_ID, LIQUIDITY_POOL_FEE, TRADE_TYPE, ROUNDING_SLIPPAGE, SELLER_IS_EXACT'
other_cols = 'partition_id'
) }}

6
models/bronze/core/bronze__trades_FR.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_FR_query_v2(
model = "history_trades",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "HISTORY_OPERATION_ID",
other_cols = '"order", LEDGER_CLOSED_AT, SELLING_ACCOUNT_ADDRESS, SELLING_ASSET_CODE, SELLING_ASSET_ISSUER, SELLING_ASSET_TYPE, SELLING_ASSET_ID, SELLING_AMOUNT, BUYING_ACCOUNT_ADDRESS, BUYING_ASSET_CODE, BUYING_ASSET_ISSUER, BUYING_ASSET_TYPE, BUYING_ASSET_ID, BUYING_AMOUNT, PRICE_N, PRICE_D, SELLING_OFFER_ID, BUYING_OFFER_ID, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, SELLING_LIQUIDITY_POOL_ID, LIQUIDITY_POOL_FEE, TRADE_TYPE, ROUNDING_SLIPPAGE, SELLER_IS_EXACT'
other_cols = 'partition_id'
) }}

6
models/bronze/core/bronze__transactions.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_query_v2(
model = "history_transactions",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "ID",
other_cols = "TRANSACTION_HASH, LEDGER_SEQUENCE, ACCOUNT, ACCOUNT_SEQUENCE, MAX_FEE, OPERATION_COUNT, CREATED_AT, MEMO_TYPE, MEMO, TIME_BOUNDS, SUCCESSFUL, FEE_CHARGED, INNER_TRANSACTION_HASH, FEE_ACCOUNT, NEW_MAX_FEE, ACCOUNT_MUXED, FEE_ACCOUNT_MUXED, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, LEDGER_BOUNDS, MIN_ACCOUNT_SEQUENCE, MIN_ACCOUNT_SEQUENCE_AGE, MIN_ACCOUNT_SEQUENCE_LEDGER_GAP, TX_ENVELOPE, TX_RESULT, TX_META, TX_FEE_META, EXTRA_SIGNERS, RESOURCE_FEE, SOROBAN_RESOURCES_INSTRUCTIONS, SOROBAN_RESOURCES_READ_BYTES, SOROBAN_RESOURCES_WRITE_BYTES, CLOSED_AT, TRANSACTION_RESULT_CODE, INCLUSION_FEE_BID, INCLUSION_FEE_CHARGED, RESOURCE_FEE_REFUND, NON_REFUNDABLE_RESOURCE_FEE_CHARGED, REFUNDABLE_RESOURCE_FEE_CHARGED, RENT_FEE_CHARGED, TX_SIGNERS, REFUNDABLE_FEE"
other_cols = "partition_id"
) }}

6
models/bronze/core/bronze__transactions_FR.sql
View File

@ -3,8 +3,8 @@
) }}
{{ streamline_external_table_FR_query_v2(
model = "history_transactions",
partition_function = " TO_DATE( SPLIT_PART(SPLIT_PART(file_name, '/', 3), '=', 2)|| '01', 'YYYYMMDD')",
partition_name = "partition_id",
partition_function = "TRY_TO_DATE(left(split_part(split_part(file_name, '=', -1), '/', -1),8), 'YYYYMMDD')",
partition_name = "partition_gte_id",
unique_key = "ID",
other_cols = "TRANSACTION_HASH, LEDGER_SEQUENCE, ACCOUNT, ACCOUNT_SEQUENCE, MAX_FEE, OPERATION_COUNT, CREATED_AT, MEMO_TYPE, MEMO, TIME_BOUNDS, SUCCESSFUL, FEE_CHARGED, INNER_TRANSACTION_HASH, FEE_ACCOUNT, NEW_MAX_FEE, ACCOUNT_MUXED, FEE_ACCOUNT_MUXED, BATCH_ID, BATCH_RUN_DATE, BATCH_INSERT_TS, LEDGER_BOUNDS, MIN_ACCOUNT_SEQUENCE, MIN_ACCOUNT_SEQUENCE_AGE, MIN_ACCOUNT_SEQUENCE_LEDGER_GAP, TX_ENVELOPE, TX_RESULT, TX_META, TX_FEE_META, EXTRA_SIGNERS, RESOURCE_FEE, SOROBAN_RESOURCES_INSTRUCTIONS, SOROBAN_RESOURCES_READ_BYTES, SOROBAN_RESOURCES_WRITE_BYTES, CLOSED_AT, TRANSACTION_RESULT_CODE, INCLUSION_FEE_BID, INCLUSION_FEE_CHARGED, RESOURCE_FEE_REFUND, NON_REFUNDABLE_RESOURCE_FEE_CHARGED, REFUNDABLE_RESOURCE_FEE_CHARGED, RENT_FEE_CHARGED, TX_SIGNERS, REFUNDABLE_FEE"
other_cols = "partition_id"
) }}

89
models/silver/silver__accounts.sql
View File

@ -5,54 +5,78 @@
incremental_predicates = ["dynamic_range_predicate", "partition_id::date"],
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ['closed_at::DATE','partition_id','modified_timestamp::DATE'],
full_refresh = false,
tags = ['scheduled_core'],
) }}
{% if execute %}
{% if is_incremental() %}
{% set max_inserted_query %}
{% set max_is_query %}
SELECT
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(_inserted_timestamp) AS _inserted_timestamp,
MAX(partition_gte_id) AS partition__gte_id
FROM
{{ this }}
{% endset %}
{% set max_inserted_timestamp = run_query(max_inserted_query) [0] [0] %}
{% set result = run_query(max_is_query) %}
{% set max_is = result [0] [0] %}
{% set max_part = result [0] [1] %}
{% endif %}
{% endif %}
WITH pre_final AS (
SELECT
partition_id,
account_id,
balance,
buying_liabilities,
selling_liabilities,
sequence_number,
num_subentries,
inflation_destination,
flags,
home_domain,
master_weight,
threshold_low,
threshold_medium,
threshold_high,
last_modified_ledger,
ledger_entry_change,
deleted,
batch_id,
batch_run_date,
batch_insert_ts,
sponsor,
num_sponsored,
num_sponsoring,
sequence_ledger,
sequence_time,
closed_at,
ledger_sequence,
partition_gte_id,
VALUE :account_id :: STRING AS account_id,
TRY_CAST(
VALUE :balance :: STRING AS FLOAT
) AS balance,
TRY_CAST(
VALUE :buying_liabilities :: STRING AS FLOAT
) AS buying_liabilities,
TRY_CAST(
VALUE :selling_liabilities :: STRING AS FLOAT
) AS selling_liabilities,
VALUE :sequence_number :: INTEGER AS sequence_number,
VALUE :num_subentries :: INTEGER AS num_subentries,
VALUE :inflation_destination :: STRING AS inflation_destination,
VALUE :flags :: INTEGER AS flags,
VALUE :home_domain :: STRING AS home_domain,
VALUE :master_weight :: INTEGER AS master_weight,
VALUE :threshold_low :: INTEGER AS threshold_low,
VALUE :threshold_medium :: INTEGER AS threshold_medium,
VALUE :threshold_high :: INTEGER AS threshold_high,
VALUE :last_modified_ledger :: INTEGER AS last_modified_ledger,
VALUE :ledger_entry_change :: INTEGER AS ledger_entry_change,
VALUE :deleted :: BOOLEAN AS deleted,
TO_TIMESTAMP(
VALUE :batch_run_date :: INT,
6
) AS batch_run_date,
VALUE: batch_id :: STRING AS batch_id,
TO_TIMESTAMP(
VALUE :batch_insert_ts :: INT,
6
) AS batch_insert_ts,
VALUE :sponsor :: STRING AS sponsor,
VALUE :num_sponsored :: INTEGER AS num_sponsored,
VALUE :num_sponsoring :: INTEGER AS num_sponsoring,
VALUE :sequence_ledger :: INTEGER AS sequence_ledger,
TO_TIMESTAMP(
VALUE :sequence_time :: INT,
6
) AS sequence_time,
TO_TIMESTAMP(
VALUE :closed_at :: INT,
6
) AS closed_at,
TO_TIMESTAMP(
VALUE :ledger_sequence :: INT,
6
) AS ledger_sequence,
_inserted_timestamp
FROM
@ -64,18 +88,21 @@ WITH pre_final AS (
{% if is_incremental() %}
WHERE
_inserted_timestamp >= '{{ max_inserted_timestamp }}'
partition_gte_id >= '{{ max_part }}'
AND _inserted_timestamp > '{{ max_is }}'
{% endif %}
qualify ROW_NUMBER() over (
PARTITION BY account_id,
closed_at
ORDER BY
batch_insert_ts DESC,
_inserted_timestamp DESC
) = 1
)
SELECT
partition_id,
partition_gte_id,
account_id,
balance,
buying_liabilities,

36
models/silver/silver__assets.sql
View File

@ -11,29 +11,39 @@
{% if execute %}
{% if is_incremental() %}
{% set max_inserted_query %}
{% set max_is_query %}
SELECT
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(_inserted_timestamp) AS _inserted_timestamp,
MAX(partition_gte_id) AS partition__gte_id
FROM
{{ this }}
{% endset %}
{% set max_inserted_timestamp = run_query(max_inserted_query) [0] [0] %}
{% set result = run_query(max_is_query) %}
{% set max_is = result [0] [0] %}
{% set max_part = result [0] [1] %}
{% endif %}
{% endif %}
WITH pre_final AS (
SELECT
partition_id,
id,
asset_type,
asset_code,
asset_issuer,
batch_run_date,
batch_id,
batch_insert_ts,
asset_id,
partition_gte_id,
VALUE :id :: FLOAT AS id,
VALUE :asset_type :: STRING AS asset_type,
VALUE :asset_code :: STRING AS asset_code,
VALUE :asset_issuer :: STRING AS asset_issuer,
TO_TIMESTAMP(
VALUE :batch_run_date :: INT,
6
) AS batch_run_date,
VALUE: batch_id :: STRING AS batch_id,
TO_TIMESTAMP(
VALUE :batch_insert_ts :: INT,
6
) AS batch_insert_ts,
VALUE :asset_id :: INT AS asset_id,
_inserted_timestamp
FROM
@ -45,7 +55,8 @@ WITH pre_final AS (
{% if is_incremental() %}
WHERE
_inserted_timestamp >= '{{ max_inserted_timestamp }}'
partition_gte_id >= '{{ max_part }}'
AND _inserted_timestamp > '{{ max_is }}'
{% endif %}
qualify ROW_NUMBER() over (
@ -57,6 +68,7 @@ qualify ROW_NUMBER() over (
)
SELECT
partition_id,
partition_gte_id,
id,
asset_type,
asset_code,

70
models/silver/silver__ledgers.sql
View File

@ -5,52 +5,55 @@
incremental_predicates = ["dynamic_range_predicate", "partition_id::date"],
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ['closed_at::DATE','partition_id','modified_timestamp::DATE'],
full_refresh = false,
tags = ['scheduled_core'],
) }}
{% if execute %}
{% if is_incremental() %}
{% set max_inserted_query %}
{% set max_is_query %}
SELECT
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(_inserted_timestamp) AS _inserted_timestamp,
MAX(partition_gte_id) AS partition__gte_id
FROM
{{ this }}
{% endset %}
{% set max_inserted_timestamp = run_query(max_inserted_query) [0] [0] %}
{% set result = run_query(max_is_query) %}
{% set max_is = result [0] [0] %}
{% set max_part = result [0] [1] %}
{% endif %}
{% endif %}
WITH pre_final AS (
SELECT
partition_id,
SEQUENCE,
ledger_hash,
previous_ledger_hash,
transaction_count,
operation_count,
closed_at,
id,
total_coins,
fee_pool,
base_fee,
base_reserve,
max_tx_set_size,
protocol_version,
ledger_header,
successful_transaction_count,
failed_transaction_count,
tx_set_operation_count,
batch_id,
batch_run_date,
batch_insert_ts,
soroban_fee_write_1kb,
node_id,
signature,
total_byte_size_of_bucket_list,
partition_gte_id,
SEQUENCE :: INTEGER AS SEQUENCE,
ledger_hash :: STRING AS ledger_hash,
previous_ledger_hash :: STRING AS previous_ledger_hash,
transaction_count :: INTEGER AS transaction_count,
operation_count :: INTEGER AS operation_count,
closed_at :: TIMESTAMP AS closed_at,
id :: INTEGER AS id,
total_coins :: INTEGER AS total_coins,
fee_pool :: INTEGER AS fee_pool,
base_fee :: INTEGER AS base_fee,
base_reserve :: INTEGER AS base_reserve,
max_tx_set_size :: INTEGER AS max_tx_set_size,
protocol_version :: INTEGER AS protocol_version,
{# ledger_header :: STRING AS ledger_header, #}
successful_transaction_count :: INTEGER AS successful_transaction_count,
failed_transaction_count :: INTEGER AS failed_transaction_count,
tx_set_operation_count :: INTEGER AS tx_set_operation_count,
batch_id :: STRING AS batch_id,
batch_run_date :: TIMESTAMP AS batch_run_date,
batch_insert_ts :: TIMESTAMP AS batch_insert_ts,
soroban_fee_write_1kb :: INTEGER AS soroban_fee_write_1kb,
node_id :: STRING AS node_id,
signature :: STRING AS signature,
total_byte_size_of_bucket_list :: INTEGER AS total_byte_size_of_bucket_list,
_inserted_timestamp
FROM
@ -62,17 +65,20 @@ WITH pre_final AS (
{% if is_incremental() %}
WHERE
_inserted_timestamp >= '{{ max_inserted_timestamp }}'
partition_gte_id >= '{{ max_part }}'
AND _inserted_timestamp > '{{ max_is }}'
{% endif %}
qualify ROW_NUMBER() over (
PARTITION BY SEQUENCE
ORDER BY
batch_insert_ts DESC,
_inserted_timestamp DESC
) = 1
)
SELECT
partition_id,
partition_gte_id,
SEQUENCE,
ledger_hash,
previous_ledger_hash,
@ -86,7 +92,7 @@ SELECT
base_reserve,
max_tx_set_size,
protocol_version,
ledger_header,
{# ledger_header, #}
successful_transaction_count,
failed_transaction_count,
tx_set_operation_count,
@ -97,12 +103,12 @@ SELECT
node_id,
signature,
total_byte_size_of_bucket_list,
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['sequence']
) }} AS ledgers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
'{{ invocation_id }}' AS _invocation_id,
_inserted_timestamp
FROM
pre_final

70
models/silver/silver__liquidity_pools.sql
View File

@ -11,44 +11,57 @@
{% if execute %}
{% if is_incremental() %}
{% set max_inserted_query %}
{% set max_is_query %}
SELECT
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(_inserted_timestamp) AS _inserted_timestamp,
MAX(partition_gte_id) AS partition__gte_id
FROM
{{ this }}
{% endset %}
{% set max_inserted_timestamp = run_query(max_inserted_query) [0] [0] %}
{% set result = run_query(max_is_query) %}
{% set max_is = result [0] [0] %}
{% set max_part = result [0] [1] %}
{% endif %}
{% endif %}
WITH pre_final AS (
SELECT
partition_id,
partition_gte_id,
liquidity_pool_id,
TYPE,
fee,
trustline_count,
pool_share_count,
asset_a_type,
asset_a_code,
asset_a_issuer,
asset_a_id,
asset_a_amount,
asset_b_type,
asset_b_code,
asset_b_issuer,
asset_b_id,
asset_b_amount,
last_modified_ledger,
ledger_entry_change,
deleted,
batch_id,
batch_run_date,
batch_insert_ts,
closed_at,
ledger_sequence,
VALUE :TYPE :: STRING AS TYPE,
VALUE :fee :: INTEGER AS fee,
VALUE :trustline_count :: INTEGER AS trustline_count,
VALUE :pool_share_count :: FLOAT AS pool_share_count,
VALUE :asset_a_type :: STRING AS asset_a_type,
VALUE :asset_a_code :: STRING AS asset_a_code,
VALUE :asset_a_issuer :: STRING AS asset_a_issuer,
VALUE :asset_a_id :: INTEGER AS asset_a_id,
VALUE :asset_a_amount :: FLOAT AS asset_a_amount,
VALUE :asset_b_type :: STRING AS asset_b_type,
VALUE :asset_b_code :: STRING AS asset_b_code,
VALUE :asset_b_issuer :: STRING AS asset_b_issuer,
VALUE :asset_b_id :: INTEGER AS asset_b_id,
VALUE :asset_b_amount :: FLOAT AS asset_b_amount,
VALUE :last_modified_ledger :: INTEGER AS last_modified_ledger,
VALUE :ledger_entry_change :: INTEGER AS ledger_entry_change,
VALUE :deleted :: BOOLEAN AS deleted,
TO_TIMESTAMP(
VALUE :batch_run_date :: INT,
6
) AS batch_run_date,
VALUE: batch_id :: STRING AS batch_id,
TO_TIMESTAMP(
VALUE :batch_insert_ts :: INT,
6
) AS batch_insert_ts,
TO_TIMESTAMP(
VALUE :closed_at :: INT,
6
) AS closed_at,
VALUE :ledger_sequence :: INTEGER AS ledger_sequence,
_inserted_timestamp
FROM
@ -60,19 +73,22 @@ WITH pre_final AS (
{% if is_incremental() %}
WHERE
_inserted_timestamp >= '{{ max_inserted_timestamp }}'
partition_gte_id >= '{{ max_part }}'
AND _inserted_timestamp > '{{ max_is }}'
{% endif %}
qualify ROW_NUMBER() over (
PARTITION BY liquidity_pool_id,
closed_at
ORDER BY
batch_insert_ts DESC,
_inserted_timestamp DESC
) = 1
)
SELECT
partition_id,
liquidity_pool_id,
partition_gte_id,
liquidity_pool_id :: STRING AS liquidity_pool_id,
TYPE,
fee,
trustline_count,

453
models/silver/silver__operations.sql
View File

@ -1,8 +1,8 @@
-- depends_on: {{ ref('bronze__operations') }}
{{ config(
materialized = 'incremental',
unique_key = "op_id",
incremental_predicates = ["dynamic_range_predicate", "partition_id::date"],
unique_key = "id",
incremental_predicates = ["dynamic_range_predicate","partition_id::date"],
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ['closed_at::DATE','partition_id','modified_timestamp::DATE'],
tags = ['scheduled_core'],
@ -11,165 +11,165 @@
{% if execute %}
{% if is_incremental() %}
{% set max_inserted_query %}
{% set max_is_query %}
SELECT
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(_inserted_timestamp) AS _inserted_timestamp,
MAX(partition_gte_id) AS partition__gte_id
FROM
{{ this }}
{% endset %}
{% set max_inserted_timestamp = run_query(max_inserted_query) [0] [0] %}
{% set result = run_query(max_is_query) %}
{% set max_is = result [0] [0] %}
{% set max_part = result [0] [1] %}
{% endif %}
{% endif %}
WITH pre_final AS (
SELECT
partition_id,
op_id,
account,
amount,
asset_code,
asset_issuer,
asset_type,
asset_id,
authorize,
balance_id,
buying_asset_code,
buying_asset_issuer,
buying_asset_type,
buying_asset_id,
"from",
funder,
high_threshold,
home_domain,
inflation_dest,
"into",
"limit",
low_threshold,
master_key_weight,
med_threshold,
NAME,
offer_id,
path,
price,
d,
n,
selling_asset_code,
selling_asset_issuer,
selling_asset_type,
selling_asset_id,
set_flags,
set_flags_s,
signer_key,
signer_weight,
source_amount,
source_asset_code,
source_asset_issuer,
source_asset_type,
source_asset_id,
source_max,
starting_balance,
"to",
trustee,
trustor,
trustline_asset,
"value",
clear_flags,
clear_flags_s,
destination_min,
bump_to,
sponsor,
sponsored_id,
begin_sponsor,
authorize_to_maintain_liabilities,
clawback_enabled,
liquidity_pool_id,
reserve_a_asset_type,
reserve_a_asset_id,
reserve_a_asset_code,
reserve_a_asset_issuer,
reserve_a_max_amount,
reserve_a_deposit_amount,
reserve_b_asset_type,
reserve_b_asset_id,
reserve_b_asset_code,
reserve_b_asset_issuer,
reserve_b_max_amount,
reserve_b_deposit_amount,
min_price,
min_price_r,
max_price,
max_price_r,
shares_received,
reserve_a_min_amount,
reserve_b_min_amount,
shares,
reserve_a_withdraw_amount,
reserve_b_withdraw_amount,
op_source_account,
op_source_account_muxed,
transaction_id,
TYPE,
transaction_hash,
ledger_sequence,
txn_account,
account_sequence,
max_fee,
txn_operation_count,
txn_created_at,
memo_type,
memo,
time_bounds,
SUCCESSFUL,
fee_charged,
fee_account,
new_max_fee,
account_muxed,
fee_account_muxed,
ledger_hash,
previous_ledger_hash,
transaction_count,
ledger_operation_count,
closed_at,
ledger_id,
total_coins,
fee_pool,
base_fee,
base_reserve,
max_tx_set_size,
protocol_version,
successful_transaction_count,
failed_transaction_count,
batch_id,
batch_run_date,
batch_insert_ts,
ledger_bounds,
min_account_sequence,
min_account_sequence_age,
min_account_sequence_ledger_gap,
extra_signers,
asset_balance_changes,
PARAMETERS,
parameters_decoded,
FUNCTION,
address,
soroban_operation_type,
extend_to,
contract_id,
contract_code_hash,
resource_fee,
soroban_resources_instructions,
soroban_resources_read_bytes,
soroban_resources_write_bytes,
transaction_result_code,
inclusion_fee_bid,
inclusion_fee_charged,
resource_fee_refund,
operation_result_code,
operation_trace_code,
op_application_order,
txn_application_order,
partition_gte_id,
id :: INTEGER AS id,
VALUE :source_account :: STRING AS source_account,
VALUE :source_account_muxed :: STRING AS op_source_account_muxed,
VALUE :ledger_sequence :: INTEGER AS ledger_sequence,
VALUE :transaction_id :: INTEGER AS transaction_id,
VALUE :type :: INTEGER AS TYPE,
VALUE :type_string :: STRING AS type_string,
VALUE :details :account :: STRING AS account,
VALUE :details :account_muxed :: STRING AS op_account_muxed,
VALUE :details :account_muxed_id :: INTEGER AS op_account_muxed_id,
VALUE :details :account_id :: STRING AS op_account_id,
VALUE :details :amount :: FLOAT AS amount,
VALUE :details :asset :: STRING AS asset,
VALUE :details :asset_code :: STRING AS asset_code,
VALUE :details :asset_issuer :: STRING AS asset_issuer,
VALUE :details :asset_id :: STRING AS asset_id,
VALUE :details :asset_type :: STRING AS asset_type,
VALUE :details :authorize :: BOOLEAN AS authorize,
VALUE :details :balance_id :: STRING AS balance_id,
VALUE :details :buying_asset_code :: STRING AS buying_asset_code,
VALUE :details :buying_asset_issuer :: STRING AS buying_asset_issuer,
VALUE :details :buying_asset_id :: STRING AS buying_asset_id,
VALUE :details :buying_asset_type :: STRING AS buying_asset_type,
VALUE :details :claimable_balance_id :: STRING AS claimable_balance_id,
VALUE :details :claimant :: STRING AS claimant,
VALUE :details :claimant_muxed :: STRING AS claimant_muxed,
VALUE :details :claimant_muxed_id :: INTEGER AS claimant_muxed_id,
VALUE :details :claimants :: variant AS claimants,
VALUE :details :data_account_id :: STRING AS data_account_id,
VALUE :details :data_name :: STRING AS data_name,
VALUE :details :"from" :: STRING AS "from",
VALUE :details :from_muxed :: STRING AS from_muxed,
VALUE :details :from_muxed_id :: INTEGER AS from_muxed_id,
VALUE :details :funder :: STRING AS funder,
VALUE :details :funder_muxed :: STRING AS funder_muxed,
VALUE :details :funder_muxed_id :: INTEGER AS funder_muxed_id,
VALUE :details :high_threshold :: INTEGER AS high_threshold,
VALUE :details :home_domain :: STRING AS home_domain,
VALUE :details :inflation_dest :: STRING AS inflation_dest,
VALUE :details :"into" :: STRING AS "into",
VALUE :details :into_muxed :: STRING AS into_muxed,
VALUE :details :into_muxed_id :: INTEGER AS into_muxed_id,
VALUE :details :"limit" :: FLOAT AS "limit",
VALUE :details :low_threshold :: INTEGER AS low_threshold,
VALUE :details :master_key_weight :: INTEGER AS master_key_weight,
VALUE :details :med_threshold :: INTEGER AS med_threshold,
VALUE :details :name :: STRING AS NAME,
VALUE :details :offer_id :: INTEGER AS offer_id,
VALUE :details :path :: variant AS path,
VALUE :details :price :: ARRAY AS price,
VALUE :details :price_r :: variant AS price_r,
VALUE :details :selling_asset_code :: STRING AS selling_asset_code,
VALUE :details :selling_asset_issuer :: STRING AS selling_asset_issuer,
VALUE :details :selling_asset_id :: STRING AS selling_asset_id,
VALUE :details :selling_asset_type :: STRING AS selling_asset_type,
VALUE :details :set_flags :: ARRAY AS set_flags,
VALUE :details :set_flags_s :: ARRAY AS set_flags_s,
VALUE :details :signer_account_id :: STRING AS signer_account_id,
VALUE :details :signer_key :: STRING AS signer_key,
VALUE :details :signer_weight :: INTEGER AS signer_weight,
VALUE :details :source_amount :: FLOAT AS source_amount,
VALUE :details :source_asset_code :: STRING AS source_asset_code,
VALUE :details :source_asset_issuer :: STRING AS source_asset_issuer,
VALUE :details :source_asset_id :: STRING AS source_asset_id,
VALUE :details :source_asset_type :: STRING AS source_asset_type,
VALUE :details :source_max :: FLOAT AS source_max,
VALUE :details :starting_balance :: FLOAT AS starting_balance,
VALUE :details :"to" :: STRING AS "to",
VALUE :details :to_muxed :: STRING AS to_muxed,
VALUE :details :to_muxed_id :: INTEGER AS to_muxed_id,
VALUE :details :trustee :: STRING AS trustee,
VALUE :details :trustee_muxed :: STRING AS trustee_muxed,
VALUE :details :trustee_muxed_id :: INTEGER AS trustee_muxed_id,
VALUE :details :trustline_account_id :: STRING AS trustline_account_id,
VALUE :details :trustline_asset :: STRING AS trustline_asset,
VALUE :details :trustor :: STRING AS trustor,
VALUE :details :trustor_muxed :: STRING AS trustor_muxed,
VALUE :details :trustor_muxed_id :: INTEGER AS trustor_muxed_id,
VALUE :details :value :: STRING AS VALUE,
VALUE :details :clear_flags :: ARRAY AS clear_flags,
VALUE :details :clear_flags_s :: ARRAY AS clear_flags_s,
VALUE :details :destination_min :: STRING AS destination_min,
VALUE :details :bump_to :: STRING AS bump_to,
VALUE :details :authorize_to_maintain_liabilities :: BOOLEAN AS authorize_to_maintain_liabilities,
VALUE :details :clawback_enabled :: BOOLEAN AS clawback_enabled,
VALUE :details :sponsor :: STRING AS sponsor,
VALUE :details :sponsored_id :: STRING AS sponsored_id,
VALUE :details :begin_sponsor :: STRING AS begin_sponsor,
VALUE :details :begin_sponsor_muxed :: STRING AS begin_sponsor_muxed,
VALUE :details :begin_sponsor_muxed_id :: INTEGER AS begin_sponsor_muxed_id,
VALUE :details :liquidity_pool_id :: STRING AS liquidity_pool_id,
VALUE :details :reserve_a_asset_type :: STRING AS reserve_a_asset_type,
VALUE :details :reserve_a_asset_code :: STRING AS reserve_a_asset_code,
VALUE :details :reserve_a_asset_issuer :: STRING AS reserve_a_asset_issuer,
VALUE :details :reserve_a_asset_id :: STRING AS reserve_a_asset_id,
VALUE :details :reserve_a_max_amount :: FLOAT AS reserve_a_max_amount,
VALUE :details :reserve_a_deposit_amount :: FLOAT AS reserve_a_deposit_amount,
VALUE :details :reserve_b_asset_type :: STRING AS reserve_b_asset_type,
VALUE :details :reserve_b_asset_code :: STRING AS reserve_b_asset_code,
VALUE :details :reserve_b_asset_issuer :: STRING AS reserve_b_asset_issuer,
VALUE :details :reserve_b_asset_id :: STRING AS reserve_b_asset_id,
VALUE :details :reserve_b_max_amount :: FLOAT AS reserve_b_max_amount,
VALUE :details :reserve_b_deposit_amount :: FLOAT AS reserve_b_deposit_amount,
VALUE :details :min_price :: FLOAT AS min_price,
VALUE :details :min_price_r :: variant AS min_price_r,
VALUE :details :max_price :: FLOAT AS max_price,
VALUE :details :max_price_r :: variant AS max_price_r,
VALUE :details :shares_received :: FLOAT AS shares_received,
VALUE :details :reserve_a_min_amount :: FLOAT AS reserve_a_min_amount,
VALUE :details :reserve_a_withdraw_amount :: FLOAT AS reserve_a_withdraw_amount,
VALUE :details :reserve_b_min_amount :: FLOAT AS reserve_b_min_amount,
VALUE :details :reserve_b_withdraw_amount :: FLOAT AS reserve_b_withdraw_amount,
VALUE :details :shares :: FLOAT AS shares,
VALUE :details :asset_balance_changes :: variant AS asset_balance_changes,
VALUE :details :parameters :: variant AS PARAMETERS,
VALUE :details :parameters_decoded :: variant AS parameters_decoded,
VALUE :details :function :: STRING AS FUNCTION,
VALUE :details :address :: STRING AS address,
VALUE :details :type :: STRING AS soroban_operation_type,
VALUE :details :extend_to :: INTEGER AS extend_to,
VALUE :details :contract_id :: STRING AS contract_id,
VALUE :details :contract_code_hash :: STRING AS contract_code_hash,
VALUE :details :ledger_key_hash :: STRING AS ledger_key_hash,
VALUE :details :ledgers_to_expire :: INTEGER AS ledgers_to_expire,
VALUE :details :: variant AS details_json,
VALUE :operation_result_code :: STRING AS operation_result_code,
VALUE :operation_trace_code :: STRING AS operation_trace_code,
TO_TIMESTAMP(
VALUE :closed_at :: INT,
6
) AS closed_at,
VALUE :batch_id :: STRING AS batch_id,
TO_TIMESTAMP(
VALUE :batch_run_date :: INT,
6
) AS batch_run_date,
TO_TIMESTAMP(
VALUE :batch_insert_ts :: INT,
6
) AS batch_insert_ts,
_inserted_timestamp
FROM
@ -181,164 +181,21 @@ WITH pre_final AS (
{% if is_incremental() %}
WHERE
_inserted_timestamp >= '{{ max_inserted_timestamp }}'
partition_gte_id >= '{{ max_part }}'
AND _inserted_timestamp > '{{ max_is }}'
{% endif %}
qualify ROW_NUMBER() over (
PARTITION BY op_id
PARTITION BY id
ORDER BY
batch_insert_ts DESC,
_inserted_timestamp DESC
) = 1
)
SELECT
partition_id,
op_id,
account,
amount,
asset_code,
asset_issuer,
asset_type,
asset_id,
authorize,
balance_id,
buying_asset_code,
buying_asset_issuer,
buying_asset_type,
buying_asset_id,
"from",
funder,
high_threshold,
home_domain,
inflation_dest,
"into",
"limit",
low_threshold,
master_key_weight,
med_threshold,
NAME,
offer_id,
path,
price,
d,
n,
selling_asset_code,
selling_asset_issuer,
selling_asset_type,
selling_asset_id,
set_flags,
set_flags_s,
signer_key,
signer_weight,
source_amount,
source_asset_code,
source_asset_issuer,
source_asset_type,
source_asset_id,
source_max,
starting_balance,
"to",
trustee,
trustor,
trustline_asset,
"value",
clear_flags,
clear_flags_s,
destination_min,
bump_to,
sponsor,
sponsored_id,
begin_sponsor,
authorize_to_maintain_liabilities,
clawback_enabled,
liquidity_pool_id,
reserve_a_asset_type,
reserve_a_asset_id,
reserve_a_asset_code,
reserve_a_asset_issuer,
reserve_a_max_amount,
reserve_a_deposit_amount,
reserve_b_asset_type,
reserve_b_asset_id,
reserve_b_asset_code,
reserve_b_asset_issuer,
reserve_b_max_amount,
reserve_b_deposit_amount,
min_price,
min_price_r,
max_price,
max_price_r,
shares_received,
reserve_a_min_amount,
reserve_b_min_amount,
shares,
reserve_a_withdraw_amount,
reserve_b_withdraw_amount,
op_source_account,
op_source_account_muxed,
transaction_id,
TYPE,
transaction_hash,
ledger_sequence,
txn_account,
account_sequence,
max_fee,
txn_operation_count,
txn_created_at,
memo_type,
memo,
time_bounds,
SUCCESSFUL,
fee_charged,
fee_account,
new_max_fee,
account_muxed,
fee_account_muxed,
ledger_hash,
previous_ledger_hash,
transaction_count,
ledger_operation_count,
closed_at,
ledger_id,
total_coins,
fee_pool,
base_fee,
base_reserve,
max_tx_set_size,
protocol_version,
successful_transaction_count,
failed_transaction_count,
batch_id,
batch_run_date,
batch_insert_ts,
ledger_bounds,
min_account_sequence,
min_account_sequence_age,
min_account_sequence_ledger_gap,
extra_signers,
asset_balance_changes,
PARAMETERS,
parameters_decoded,
FUNCTION,
address,
soroban_operation_type,
extend_to,
contract_id,
contract_code_hash,
resource_fee,
soroban_resources_instructions,
soroban_resources_read_bytes,
soroban_resources_write_bytes,
transaction_result_code,
inclusion_fee_bid,
inclusion_fee_charged,
resource_fee_refund,
operation_result_code,
operation_trace_code,
op_application_order,
txn_application_order,
_inserted_timestamp,
*,
{{ dbt_utils.generate_surrogate_key(
['op_id']
['id']
) }} AS operations_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,

78
models/silver/silver__trades.sql
View File

@ -11,48 +11,61 @@
{% if execute %}
{% if is_incremental() %}
{% set max_inserted_query %}
{% set max_is_query %}
SELECT
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(_inserted_timestamp) AS _inserted_timestamp,
MAX(partition_gte_id) AS partition__gte_id
FROM
{{ this }}
{% endset %}
{% set max_inserted_timestamp = run_query(max_inserted_query) [0] [0] %}
{% set result = run_query(max_is_query) %}
{% set max_is = result [0] [0] %}
{% set max_part = result [0] [1] %}
{% endif %}
{% endif %}
WITH pre_final AS (
SELECT
partition_id,
history_operation_id,
"order",
ledger_closed_at,
selling_account_address,
selling_asset_code,
selling_asset_issuer,
selling_asset_type,
selling_asset_id,
selling_amount,
buying_account_address,
buying_asset_code,
buying_asset_issuer,
buying_asset_type,
buying_asset_id,
buying_amount,
price_n,
price_d,
selling_offer_id,
buying_offer_id,
batch_id,
batch_run_date,
batch_insert_ts,
selling_liquidity_pool_id,
liquidity_pool_fee,
trade_type,
rounding_slippage,
seller_is_exact,
partition_gte_id,
history_operation_id :: INTEGER AS history_operation_id,
VALUE: "order" :: INTEGER AS "order",
TO_TIMESTAMP(
VALUE :ledger_closed_at :: INT,
6
) AS ledger_closed_at,
VALUE: selling_account_address :: STRING AS selling_account_address,
VALUE: selling_asset_code :: STRING AS selling_asset_code,
VALUE: selling_asset_issuer :: STRING AS selling_asset_issuer,
VALUE: selling_asset_type :: STRING AS selling_asset_type,
VALUE: selling_asset_id :: INTEGER AS selling_asset_id,
VALUE: selling_amount :: FLOAT AS selling_amount,
VALUE: buying_account_address :: STRING AS buying_account_address,
VALUE: buying_asset_code :: STRING AS buying_asset_code,
VALUE: buying_asset_issuer :: STRING AS buying_asset_issuer,
VALUE: buying_asset_type :: STRING AS buying_asset_type,
VALUE: buying_asset_id :: INTEGER AS buying_asset_id,
VALUE: buying_amount :: FLOAT AS buying_amount,
VALUE: price_n :: INTEGER AS price_n,
VALUE: price_d :: INTEGER AS price_d,
VALUE: selling_offer_id :: INTEGER AS selling_offer_id,
VALUE: buying_offer_id :: INTEGER AS buying_offer_id,
VALUE :batch_id :: STRING AS batch_id,
TO_TIMESTAMP(
VALUE :batch_run_date :: INT,
6
) AS batch_run_date,
TO_TIMESTAMP(
VALUE :batch_insert_ts :: INT,
6
) AS batch_insert_ts,
VALUE: selling_liquidity_pool_id :: STRING AS selling_liquidity_pool_id,
VALUE: liquidity_pool_fee :: INTEGER AS liquidity_pool_fee,
VALUE: trade_type :: INTEGER AS trade_type,
VALUE: rounding_slippage :: INTEGER AS rounding_slippage,
VALUE: seller_is_exact :: BOOLEAN AS seller_is_exact,
_inserted_timestamp
FROM
@ -64,18 +77,21 @@ WITH pre_final AS (
{% if is_incremental() %}
WHERE
_inserted_timestamp >= '{{ max_inserted_timestamp }}'
partition_gte_id >= '{{ max_part }}'
AND _inserted_timestamp > '{{ max_is }}'
{% endif %}
qualify ROW_NUMBER() over (
PARTITION BY history_operation_id,
"order"
ORDER BY
batch_insert_ts DESC,
_inserted_timestamp DESC
) = 1
)
SELECT
partition_id,
partition_gte_id,
history_operation_id,
"order",
ledger_closed_at,

115
models/silver/silver__transactions.sql
View File

@ -11,65 +11,81 @@
{% if execute %}
{% if is_incremental() %}
{% set max_inserted_query %}
{% set max_is_query %}
SELECT
MAX(_inserted_timestamp) AS _inserted_timestamp
MAX(_inserted_timestamp) AS _inserted_timestamp,
MAX(partition_gte_id) AS partition__gte_id
FROM
{{ this }}
{% endset %}
{% set max_inserted_timestamp = run_query(max_inserted_query) [0] [0] %}
{% set result = run_query(max_is_query) %}
{% set max_is = result [0] [0] %}
{% set max_part = result [0] [1] %}
{% endif %}
{% endif %}
WITH pre_final AS (
SELECT
partition_id,
id,
transaction_hash,
ledger_sequence,
account,
account_sequence,
max_fee,
operation_count,
created_at,
memo_type,
memo,
time_bounds,
SUCCESSFUL,
fee_charged,
inner_transaction_hash,
fee_account,
new_max_fee,
account_muxed,
fee_account_muxed,
batch_id,
batch_run_date,
batch_insert_ts,
ledger_bounds,
min_account_sequence,
min_account_sequence_age,
min_account_sequence_ledger_gap,
tx_envelope,
tx_result,
tx_meta,
tx_fee_meta,
extra_signers,
resource_fee,
soroban_resources_instructions,
soroban_resources_read_bytes,
soroban_resources_write_bytes,
closed_at,
transaction_result_code,
inclusion_fee_bid,
inclusion_fee_charged,
resource_fee_refund,
non_refundable_resource_fee_charged,
refundable_resource_fee_charged,
rent_fee_charged,
tx_signers,
refundable_fee,
partition_gte_id,
id :: INTEGER AS id,
VALUE :transaction_hash :: STRING AS transaction_hash,
VALUE :ledger_sequence :: INTEGER AS ledger_sequence,
VALUE :account :: STRING AS account,
VALUE :account_sequence :: INTEGER AS account_sequence,
VALUE :max_fee :: INTEGER AS max_fee,
VALUE :operation_count :: INTEGER AS operation_count,
TO_TIMESTAMP(
VALUE :created_at :: INT,
6
) AS created_at,
VALUE :memo_type :: STRING AS memo_type,
VALUE :memo :: STRING AS memo,
VALUE :time_bounds :: STRING AS time_bounds,
VALUE :SUCCESSFUL :: BOOLEAN AS SUCCESSFUL,
VALUE :fee_charged :: INTEGER AS fee_charged,
VALUE :inner_transaction_hash :: STRING AS inner_transaction_hash,
VALUE :fee_account :: STRING AS fee_account,
VALUE :new_max_fee :: INTEGER AS new_max_fee,
VALUE :account_muxed :: STRING AS account_muxed,
VALUE :fee_account_muxed :: STRING AS fee_account_muxed,
VALUE :batch_id :: STRING AS batch_id,
TO_TIMESTAMP(
VALUE :batch_run_date :: INT,
6
) AS batch_run_date,
TO_TIMESTAMP(
VALUE :batch_insert_ts :: INT,
6
) AS batch_insert_ts,
VALUE :ledger_bounds :: STRING AS ledger_bounds,
VALUE :min_account_sequence :: INTEGER AS min_account_sequence,
VALUE :min_account_sequence_age :: INTEGER AS min_account_sequence_age,
VALUE :min_account_sequence_ledger_gap :: INTEGER AS min_account_sequence_ledger_gap,
VALUE :tx_envelope :: STRING AS tx_envelope,
VALUE :tx_result :: STRING AS tx_result,
VALUE :tx_meta :: STRING AS tx_meta,
VALUE :tx_fee_meta :: STRING AS tx_fee_meta,
VALUE :extra_signers :: ARRAY AS extra_signers,
VALUE :resource_fee :: INTEGER AS resource_fee,
VALUE :soroban_resources_instructions :: INTEGER AS soroban_resources_instructions,
VALUE :soroban_resources_read_bytes :: INTEGER AS soroban_resources_read_bytes,
VALUE :soroban_resources_write_bytes :: INTEGER AS soroban_resources_write_bytes,
TO_TIMESTAMP(
VALUE :closed_at :: INT,
6
) AS closed_at,
VALUE :transaction_result_code :: STRING AS transaction_result_code,
VALUE :inclusion_fee_bid :: INTEGER AS inclusion_fee_bid,
VALUE :inclusion_fee_charged :: INTEGER AS inclusion_fee_charged,
VALUE :resource_fee_refund :: INTEGER AS resource_fee_refund,
VALUE :non_refundable_resource_fee_charged :: INTEGER AS non_refundable_resource_fee_charged,
VALUE :refundable_resource_fee_charged :: INTEGER AS refundable_resource_fee_charged,
VALUE :rent_fee_charged :: INTEGER AS rent_fee_charged,
VALUE :tx_signers :: STRING AS tx_signers,
VALUE :refundable_fee :: INTEGER AS refundable_fee,
_inserted_timestamp
FROM
@ -81,17 +97,20 @@ WITH pre_final AS (
{% if is_incremental() %}
WHERE
_inserted_timestamp >= '{{ max_inserted_timestamp }}'
partition_gte_id >= '{{ max_part }}'
AND _inserted_timestamp > '{{ max_is }}'
{% endif %}
qualify ROW_NUMBER() over (
PARTITION BY id
ORDER BY
batch_insert_ts DESC,
_inserted_timestamp DESC
) = 1
)
SELECT
partition_id,
partition_gte_id,
id,
transaction_hash,
ledger_sequence,

17
models/sources.yml
View File

@ -3,14 +3,14 @@ version: 2
sources:
- name: bronze_streamline
database: streamline
schema: "{{ 'stellar' if target.database == 'STELLAR' else 'stellar_dev' }}"
schema: stellar
tables:
- name: accounts
- name: contract_data
- name: enriched_history_operations
- name: history_assets
- name: history_contract_events
- name: history_ledgers
- name: history_operations
- name: history_trades
- name: history_transactions
- name: liquidity_pools
@ -30,16 +30,3 @@ sources:
schema: github_actions
tables:
- name: workflows
- name: bronze_bq
database: STELLAR_SAMPLE
schema: SAMPLE_DATA
tables:
- name: ACCOUNTS
- name: CONTRACT_DATA
- name: ENRICHED_HISTORY_OPERATIONS
- name: HISTORY_ASSETS
- name: HISTORY_CONTRACT_EVENTS
- name: HISTORY_LEDGERS
- name: HISTORY_TRADES
- name: HISTORY_TRANSACTIONS
- name: LIQUIDITY_POOLS