From f5e502b7ba1d50b73a4ded33a4ec4861442262eb Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:17:58 -0500 Subject: [PATCH] An 4916/new columns (#174) * stash * stash * name * command * nft * dates * event index * typos * inserted --------- Co-authored-by: sam --- dbt_project.yml | 1 + .../doc_descriptions/general/deprecation.md | 15 ++- .../general/export_columns.md | 17 +++ .../nft/nft_intra_event_index.md | 5 + .../traces/arb_traces_index.md | 5 + models/gold/core/core__dim_contracts.sql | 13 ++- models/gold/core/core__dim_contracts.yml | 6 + models/gold/core/core__dim_labels.sql | 16 ++- models/gold/core/core__dim_labels.yml | 7 +- .../gold/core/core__ez_decoded_event_logs.sql | 15 ++- .../gold/core/core__ez_decoded_event_logs.yml | 8 +- models/gold/core/core__ez_eth_transfers.sql | 98 ++-------------- models/gold/core/core__ez_eth_transfers.yml | 2 +- .../gold/core/core__ez_native_transfers.sql | 38 ++++++ .../gold/core/core__ez_native_transfers.yml | 42 +++++++ models/gold/core/core__ez_token_transfers.sql | 16 ++- models/gold/core/core__ez_token_transfers.yml | 10 +- models/gold/core/core__fact_blocks.sql | 32 ++++- models/gold/core/core__fact_blocks.yml | 8 +- .../core/core__fact_decoded_event_logs.sql | 16 ++- .../core/core__fact_decoded_event_logs.yml | 6 + models/gold/core/core__fact_event_logs.sql | 16 ++- models/gold/core/core__fact_event_logs.yml | 8 +- .../gold/core/core__fact_token_transfers.sql | 16 ++- .../gold/core/core__fact_token_transfers.yml | 8 +- models/gold/core/core__fact_traces.sql | 25 +++- models/gold/core/core__fact_traces.yml | 20 +++- models/gold/core/core__fact_transactions.sql | 84 +++++--------- models/gold/core/core__fact_transactions.yml | 18 ++- .../defi/defi__dim_dex_liquidity_pools.sql | 16 ++- .../defi/defi__dim_dex_liquidity_pools.yml | 8 +- models/gold/defi/defi__ez_dex_swaps.sql | 16 ++- models/gold/defi/defi__ez_dex_swaps.yml | 9 +- models/gold/nft/nft__ez_nft_sales.sql | 17 ++- models/gold/nft/nft__ez_nft_sales.yml | 10 +- models/gold/nft/nft__ez_nft_transfers.sql | 17 ++- models/gold/nft/nft__ez_nft_transfers.yml | 9 +- .../gold/price/price__dim_asset_metadata.sql | 16 ++- .../gold/price/price__dim_asset_metadata.yml | 6 + .../gold/price/price__ez_asset_metadata.sql | 16 ++- .../gold/price/price__ez_asset_metadata.yml | 6 + .../price/price__ez_hourly_token_prices.sql | 16 ++- .../price/price__ez_hourly_token_prices.yml | 8 +- .../price/price__fact_hourly_token_prices.sql | 16 ++- .../price/price__fact_hourly_token_prices.yml | 8 +- models/silver/core/silver__blocks.sql | 12 +- models/silver/core/silver__contracts.sql | 9 +- .../silver/core/silver__created_contracts.sql | 9 +- models/silver/core/silver__decoded_logs.sql | 17 ++- models/silver/core/silver__logs.sql | 8 +- .../silver/core/silver__native_transfers.sql | 109 ++++++++++++++++++ models/silver/core/silver__receipts.sql | 5 +- models/silver/core/silver__traces.sql | 8 +- models/silver/core/silver__transactions.sql | 54 +++++++-- models/silver/core/silver__transfers.sql | 16 ++- models/silver/core/silver__tx_count.sql | 28 +++-- ...lver_dex__complete_dex_liquidity_pools.sql | 8 +- .../dex/silver_dex__complete_dex_swaps.sql | 8 +- models/silver/labels/silver__labels.sql | 9 +- .../silver/nft/silver__complete_nft_sales.sql | 46 +++++++- .../silver/nft/silver__complete_nft_sales.yml | 3 + models/silver/nft/silver__nft_transfers.sql | 65 ++++++++++- models/silver/nft/silver__nft_transfers.yml | 6 + .../silver__asset_metadata_all_providers.sql | 9 +- .../silver__asset_metadata_priority.sql | 9 +- .../silver__hourly_prices_all_providers.sql | 9 +- .../prices/silver__hourly_prices_priority.sql | 9 +- .../silver__hourly_prices_priority_eth.sql | 9 +- 68 files changed, 982 insertions(+), 248 deletions(-) create mode 100644 models/doc_descriptions/general/export_columns.md create mode 100644 models/doc_descriptions/nft/nft_intra_event_index.md create mode 100644 models/doc_descriptions/traces/arb_traces_index.md create mode 100644 models/gold/core/core__ez_native_transfers.sql create mode 100644 models/gold/core/core__ez_native_transfers.yml create mode 100644 models/silver/core/silver__native_transfers.sql diff --git a/dbt_project.yml b/dbt_project.yml index aba64a1..6d82eb2 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -26,6 +26,7 @@ clean-targets: # directories to be removed by `dbt clean` models: +copy_grants: true + +on_schema_change: "append_new_columns" tests: +store_failures: true # all tests diff --git a/models/doc_descriptions/general/deprecation.md b/models/doc_descriptions/general/deprecation.md index 903fe0a..004e578 100644 --- a/models/doc_descriptions/general/deprecation.md +++ b/models/doc_descriptions/general/deprecation.md @@ -1,12 +1,11 @@ -{% docs deprecation %} +{% docs internal_column %} -Deprecating soon: This is a notice that we're only removing the below columns. Please migrate queries using these columns to `fact_decoded_event_logs`, `ez_decoded_event_logs` or use manual parsing of topics and data. The following columns will be deprecated on 7/01/23: +Deprecated. This column is no longer used. Please remove from your query by Jan. 31 2024. -`Fact_event_logs` Columns: -- `event_name` -- `event_inputs` -- `contract_name` +{% enddocs %} + +{% docs amount_deprecation %} + +This column is being deprecated for standardization purposes on Jan. 31 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `eth_amount`. -`Fact_transactions` Columns: -- `tx_json` {% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/export_columns.md b/models/doc_descriptions/general/export_columns.md new file mode 100644 index 0000000..726edb1 --- /dev/null +++ b/models/doc_descriptions/general/export_columns.md @@ -0,0 +1,17 @@ +{% docs pk %} + +The unique identifier for each row in the table. + +{% enddocs %} + +{% docs inserted_timestamp %} + +The utc timestamp at which the row was inserted into the table. + +{% enddocs %} + +{% docs modified_timestamp %} + +The utc timestamp at which the row was last modified. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/nft/nft_intra_event_index.md b/models/doc_descriptions/nft/nft_intra_event_index.md new file mode 100644 index 0000000..4ce24fc --- /dev/null +++ b/models/doc_descriptions/nft/nft_intra_event_index.md @@ -0,0 +1,5 @@ +{% docs nft_intra_event_index %} + +The order of events within a single event index. This is primarily used for ERC1155 NFT batch transfer events. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/arb_traces_index.md b/models/doc_descriptions/traces/arb_traces_index.md new file mode 100644 index 0000000..bab4cd8 --- /dev/null +++ b/models/doc_descriptions/traces/arb_traces_index.md @@ -0,0 +1,5 @@ +{% docs arb_trace_index %} + +The index of the trace within the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_contracts.sql b/models/gold/core/core__dim_contracts.sql index 30bdd92..2fa3be3 100644 --- a/models/gold/core/core__dim_contracts.sql +++ b/models/gold/core/core__dim_contracts.sql @@ -12,7 +12,18 @@ SELECT c0.block_number AS created_block_number, c0.block_timestamp AS created_block_timestamp, c0.tx_hash AS created_tx_hash, - c0.creator_address AS creator_address + c0.creator_address AS creator_address, + c0.created_contracts_id AS dim_contracts_id, + GREATEST( + c0.inserted_timestamp, + c1.inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + GREATEST( + c0.modified_timestamp, + c1.modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__created_contracts') }} c0 diff --git a/models/gold/core/core__dim_contracts.yml b/models/gold/core/core__dim_contracts.yml index c531409..bb27102 100644 --- a/models/gold/core/core__dim_contracts.yml +++ b/models/gold/core/core__dim_contracts.yml @@ -20,3 +20,9 @@ models: description: 'The transaction hash when the contract was created' - name: CREATOR_ADDRESS description: 'The address of the creator of the contract' + - name: DIM_CONTRACTS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__dim_labels.sql b/models/gold/core/core__dim_labels.sql index e21ee86..00a2ba1 100644 --- a/models/gold/core/core__dim_labels.sql +++ b/models/gold/core/core__dim_labels.sql @@ -11,6 +11,20 @@ SELECT address_name, label_type, label_subtype, - project_name + project_name, + COALESCE ( + labels_id, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} + ) AS dim_labels_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__labels') }} diff --git a/models/gold/core/core__dim_labels.yml b/models/gold/core/core__dim_labels.yml index a11e623..21c4331 100644 --- a/models/gold/core/core__dim_labels.yml +++ b/models/gold/core/core__dim_labels.yml @@ -52,4 +52,9 @@ models: column_type_list: - STRING - VARCHAR - + - name: DIM_LABELS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__ez_decoded_event_logs.sql b/models/gold/core/core__ez_decoded_event_logs.sql index 7848433..1e1e3f6 100644 --- a/models/gold/core/core__ez_decoded_event_logs.sql +++ b/models/gold/core/core__ez_decoded_event_logs.sql @@ -20,7 +20,16 @@ SELECT topics, DATA, event_removed, - tx_status + tx_status, + COALESCE ( + decoded_logs_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_decoded_event_logs_id, + GREATEST(COALESCE(l.inserted_timestamp, '2000-01-01'), COALESCE(C.inserted_timestamp, '2000-01-01')) AS inserted_timestamp, + GREATEST(COALESCE(l.modified_timestamp, '2000-01-01'), COALESCE(C.modified_timestamp, '2000-01-01')) AS modified_timestamp FROM - {{ ref('silver__decoded_logs') }} - LEFT JOIN {{ ref('silver__contracts') }} using (contract_address) + {{ ref('silver__decoded_logs') }} + l + LEFT JOIN {{ ref('silver__contracts') }} C USING (contract_address) diff --git a/models/gold/core/core__ez_decoded_event_logs.yml b/models/gold/core/core__ez_decoded_event_logs.yml index 9805482..f69ff55 100644 --- a/models/gold/core/core__ez_decoded_event_logs.yml +++ b/models/gold/core/core__ez_decoded_event_logs.yml @@ -56,4 +56,10 @@ models: - name: EVENT_REMOVED description: '{{ doc("arb_event_removed") }}' - name: TX_STATUS - description: '{{ doc("arb_tx_status") }}' \ No newline at end of file + description: '{{ doc("arb_tx_status") }}' + - name: EZ_DECODED_EVENT_LOGS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__ez_eth_transfers.sql b/models/gold/core/core__ez_eth_transfers.sql index b8f2f34..2fd6fdc 100644 --- a/models/gold/core/core__ez_eth_transfers.sql +++ b/models/gold/core/core__ez_eth_transfers.sql @@ -1,104 +1,26 @@ {{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - unique_key = 'block_number', - cluster_by = ['block_timestamp::DATE'], - tags = ['core','non_realtime','reorg'], + materialized = 'view', persist_docs ={ "relation": true, "columns": true } ) }} -WITH eth_base AS ( - - SELECT - tx_hash, - block_number, - block_timestamp, - identifier, - from_address, - to_address, - eth_value, - _call_id, - _inserted_timestamp, - eth_value_precise_raw, - eth_value_precise, - tx_position, - trace_index - FROM - {{ ref('silver__traces') }} - WHERE - eth_value > 0 - AND tx_status = 'SUCCESS' - AND trace_status = 'SUCCESS' - AND TYPE NOT IN ( - 'DELEGATECALL', - 'STATICCALL' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '72 hours' - FROM - {{ this }} -) -{% endif %} -), -tx_table AS ( - SELECT - block_number, - tx_hash, - from_address AS origin_from_address, - to_address AS origin_to_address, - origin_function_signature - FROM - {{ ref('silver__transactions') }} - WHERE - tx_hash IN ( - SELECT - DISTINCT tx_hash - FROM - eth_base - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '72 hours' - FROM - {{ this }} -) -{% endif %} -) SELECT - tx_hash AS tx_hash, - block_number AS block_number, - block_timestamp AS block_timestamp, - identifier AS identifier, + tx_hash, + block_number, + block_timestamp, + identifier, origin_from_address, origin_to_address, origin_function_signature, from_address AS eth_from_address, to_address AS eth_to_address, - eth_value AS amount, - eth_value_precise_raw AS amount_precise_raw, - eth_value_precise AS amount_precise, - ROUND( - eth_value * price, - 2 - ) AS amount_usd, + amount, + amount_precise_raw, + amount_precise, + amount_usd, _call_id, _inserted_timestamp, tx_position, trace_index FROM - eth_base A - LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }} - ON DATE_TRUNC( - 'hour', - A.block_timestamp - ) = HOUR - JOIN tx_table USING ( - tx_hash, - block_number - ) + {{ ref('silver__native_transfers') }} \ No newline at end of file diff --git a/models/gold/core/core__ez_eth_transfers.yml b/models/gold/core/core__ez_eth_transfers.yml index b39aa56..16dc7e9 100644 --- a/models/gold/core/core__ez_eth_transfers.yml +++ b/models/gold/core/core__ez_eth_transfers.yml @@ -1,7 +1,7 @@ version: 2 models: - name: core__ez_eth_transfers - description: '{{ doc("arb_ez_eth_transfers_table_doc") }}' + description: 'Deprecating soon! Migrate to `core.ez_native_transfers` by Jan. 31 2024.' columns: - name: BLOCK_NUMBER diff --git a/models/gold/core/core__ez_native_transfers.sql b/models/gold/core/core__ez_native_transfers.sql new file mode 100644 index 0000000..10fbdf0 --- /dev/null +++ b/models/gold/core/core__ez_native_transfers.sql @@ -0,0 +1,38 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + tx_hash, + block_number, + block_timestamp, + tx_position, + trace_index, + identifier, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address, + to_address, + amount, + amount_precise_raw, + amount_precise, + amount_usd, + COALESCE ( + native_transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} + ) AS ez_native_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp +FROM + {{ ref('silver__native_transfers') }} \ No newline at end of file diff --git a/models/gold/core/core__ez_native_transfers.yml b/models/gold/core/core__ez_native_transfers.yml new file mode 100644 index 0000000..e179b62 --- /dev/null +++ b/models/gold/core/core__ez_native_transfers.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: core__ez_native_transfers + description: '{{ doc("arb_ez_eth_transfers_table_doc") }}' + + columns: + - name: TX_HASH + description: '{{ doc("arb_transfer_tx_hash") }}' + - name: BLOCK_NUMBER + description: '{{ doc("arb_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("arb_block_timestamp") }}' + - name: TX_POSITION + description: '{{ doc("arb_tx_position") }}' + - name: TRACE_INDEX + description: '{{ doc("arb_trace_index") }}' + - name: IDENTIFIER + description: '{{ doc("arb_traces_identifier") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("arb_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("arb_origin_to") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("arb_origin_sig") }}' + - name: FROM_ADDRESS + description: '{{ doc("arb_transfer_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("arb_transfer_to_address") }}' + - name: AMOUNT + description: '{{ doc("arb_eth_amount") }}' + - name: AMOUNT_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: AMOUNT_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: AMOUNT_USD + description: '{{ doc("arb_eth_amount_usd") }}' + - name: EZ_NATIVE_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index 37ed8eb..11def73 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -26,6 +26,20 @@ SELECT has_decimal, has_price, _log_id, - _inserted_timestamp + _inserted_timestamp, + COALESCE ( + transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS ez_token_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__transfers') }} diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml index 8b59dfa..e473f8c 100644 --- a/models/gold/core/core__ez_token_transfers.yml +++ b/models/gold/core/core__ez_token_transfers.yml @@ -47,4 +47,12 @@ models: - name: HAS_PRICE description: '{{ doc("arb_transfer_has_price") }}' - name: _LOG_ID - description: '{{ doc("arb_log_id_transfers") }}' \ No newline at end of file + description: '{{ doc("internal_column") }}' + - name: _INSERTED_TIMESTAMP + description: '{{ doc("internal_column") }}' + - name: EZ_TOKEN_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index a2395ce..dd171ca 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -60,7 +60,33 @@ SELECT transactions_root, 'uncles', uncles - ) AS block_header_json + ) AS block_header_json, + COALESCE ( + blocks_id, + {{ dbt_utils.generate_surrogate_key( + ['a.block_number'] + ) }} + ) AS fact_blocks_id, + GREATEST( + COALESCE( + A.inserted_timestamp, + '2000-01-01' + ), + COALESCE( + d.inserted_timestamp, + '2000-01-01' + ) + ) AS inserted_timestamp, + GREATEST( + COALESCE( + A.modified_timestamp, + '2000-01-01' + ), + COALESCE( + d.modified_timestamp, + '2000-01-01' + ) + ) AS modified_timestamp FROM - {{ ref('silver__blocks') }} - LEFT JOIN {{ ref('silver__tx_count') }} USING (block_number) + {{ ref('silver__blocks') }} a + LEFT JOIN {{ ref('silver__tx_count') }} d USING (block_number) diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index e56f251..7923806 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -37,4 +37,10 @@ models: - name: UNCLE_BLOCKS description: '{{ doc("arb_uncle_blocks") }}' - name: BLOCK_HEADER_JSON - description: '{{ doc("arb_block_header_json") }}' \ No newline at end of file + description: '{{ doc("arb_block_header_json") }}' + - name: FACT_BLOCKS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_decoded_event_logs.sql b/models/gold/core/core__fact_decoded_event_logs.sql index 33f85a6..aab1f0c 100644 --- a/models/gold/core/core__fact_decoded_event_logs.sql +++ b/models/gold/core/core__fact_decoded_event_logs.sql @@ -12,6 +12,20 @@ SELECT contract_address, event_name, decoded_flat AS decoded_log, - decoded_data AS full_decoded_log + decoded_data AS full_decoded_log, + COALESCE ( + decoded_logs_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS fact_decoded_event_logs_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__decoded_logs') }} diff --git a/models/gold/core/core__fact_decoded_event_logs.yml b/models/gold/core/core__fact_decoded_event_logs.yml index c10b54a..2cd16a3 100644 --- a/models/gold/core/core__fact_decoded_event_logs.yml +++ b/models/gold/core/core__fact_decoded_event_logs.yml @@ -42,3 +42,9 @@ models: description: 'The flattened decoded log, where the keys are the names of the event parameters, and the values are the values of the event parameters.' - name: FULL_DECODED_LOG description: 'The full decoded log, including the event name, the event parameters, and the data type of the event parameters.' + - name: FACT_DECODED_EVENT_LOGS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_event_logs.sql b/models/gold/core/core__fact_event_logs.sql index f57c5bf..e21eda0 100644 --- a/models/gold/core/core__fact_event_logs.sql +++ b/models/gold/core/core__fact_event_logs.sql @@ -17,6 +17,20 @@ SELECT DATA, event_removed, tx_status, - _log_id + _log_id, + COALESCE ( + logs_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS fact_event_logs_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__logs') }} diff --git a/models/gold/core/core__fact_event_logs.yml b/models/gold/core/core__fact_event_logs.yml index 750dc0d..76689c2 100644 --- a/models/gold/core/core__fact_event_logs.yml +++ b/models/gold/core/core__fact_event_logs.yml @@ -21,7 +21,7 @@ models: - name: EVENT_REMOVED description: '{{ doc("arb_event_removed") }}' - name: _LOG_ID - description: '{{ doc("arb_log_id_events") }}' + description: '{{ doc("internal_column") }}' - name: TX_STATUS description: '{{ doc("arb_tx_status") }}' - name: ORIGIN_FUNCTION_SIGNATURE @@ -30,3 +30,9 @@ models: description: '{{ doc("arb_origin_from") }}' - name: ORIGIN_TO_ADDRESS description: '{{ doc("arb_origin_to") }}' + - name: FACT_EVENT_LOGS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_token_transfers.sql b/models/gold/core/core__fact_token_transfers.sql index 9dffa5a..8908c54 100644 --- a/models/gold/core/core__fact_token_transfers.sql +++ b/models/gold/core/core__fact_token_transfers.sql @@ -17,6 +17,20 @@ SELECT to_address, raw_amount, raw_amount_precise, - _log_id + _log_id, + COALESCE ( + transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} + ) AS fact_token_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__transfers') }} diff --git a/models/gold/core/core__fact_token_transfers.yml b/models/gold/core/core__fact_token_transfers.yml index 7f84c1e..86439af 100644 --- a/models/gold/core/core__fact_token_transfers.yml +++ b/models/gold/core/core__fact_token_transfers.yml @@ -29,4 +29,10 @@ models: - name: RAW_AMOUNT_PRECISE description: '{{ doc("arb_transfer_raw_amount_precise") }}' - name: _LOG_ID - description: '{{ doc("arb_log_id_transfers") }}' \ No newline at end of file + description: '{{ doc("internal_column") }}' + - name: FACT_TOKEN_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index 639a530..93b322c 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -10,9 +10,9 @@ SELECT block_timestamp, from_address, to_address, - eth_value, - eth_value_precise_raw, - eth_value_precise, + eth_value AS VALUE, + eth_value_precise_raw AS value_precise_raw, + eth_value_precise AS value_precise, gas, gas_used, input, @@ -26,6 +26,23 @@ SELECT error_reason, trace_index, before_evm_transfers, - after_evm_transfers + after_evm_transfers, + COALESCE ( + traces_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} + ) AS fact_traces_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp, + eth_value, + eth_value_precise_raw, + eth_value_precise FROM {{ ref('silver__traces') }} diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml index c9bbd2a..8e11e35 100644 --- a/models/gold/core/core__fact_traces.yml +++ b/models/gold/core/core__fact_traces.yml @@ -15,10 +15,16 @@ models: - name: TO_ADDRESS description: '{{ doc("arb_traces_to") }}' - name: ETH_VALUE - description: '{{ doc("arb_traces_value") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE_RAW - description: '{{ doc("precise_amount_unadjusted") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE + description: '{{ doc("amount_deprecation") }}' + - name: VALUE + description: '{{ doc("arb_traces_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE description: '{{ doc("precise_amount_adjusted") }}' - name: GAS description: '{{ doc("arb_traces_gas") }}' @@ -48,7 +54,9 @@ models: description: The state of all ETH transfers to be executed for a given transaction. - name: AFTER_EVM_TRANSFERS description: The state of all ETH transfers executed for a given transaction. - - - - \ No newline at end of file + - name: FACT_TRACES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 390bbd4..4d01ead 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -14,74 +14,42 @@ SELECT origin_function_signature, from_address, to_address, - eth_value, - eth_value_precise_raw, - eth_value_precise, + VALUE, + value_precise_raw, + value_precise, tx_fee, tx_fee_precise, - gas_price_bid, - gas_price_paid, - gas_limit, + gas_price AS gas_price_bid, + effective_gas_price AS gas_price_paid, + gas AS gas_limit, gas_used, cumulative_gas_used, max_fee_per_gas, max_priority_fee_per_gas, input_data, - status, + tx_status AS status, r, s, v, tx_type, l1_block_number, - gas_used_for_l1 + gas_used_for_l1, + COALESCE ( + transactions_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} + ) AS fact_transactions_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp, + VALUE AS eth_value, + value_precise_raw AS eth_value_precise_raw, + value_precise AS eth_value_precise FROM - ( - SELECT - block_number, - block_timestamp, - block_hash, - tx_hash, - nonce, - POSITION, - origin_function_signature, - from_address, - to_address, - VALUE AS eth_value, - tx_fee, - tx_fee_precise, - gas_price AS gas_price_bid, - effective_gas_price AS gas_price_paid, - gas AS gas_limit, - gas_used, - cumulative_gas_used, - max_fee_per_gas, - max_priority_fee_per_gas, - input_data, - tx_status AS status, - r, - s, - v, - tx_type, - l1_block_number, - gas_used_for_l1, - CASE - WHEN tx_hash <> '0xefc4b1845f162fd61b496766c69fc0da9ee1317f0153efa30b3cb30d8f7884ba' THEN to_varchar( - TO_NUMBER(REPLACE(DATA :value :: STRING, '0x'), REPEAT('X', LENGTH(REPLACE(DATA :value :: STRING, '0x')))) - ) - ELSE NULL - END AS eth_value_precise_raw, - IFF(LENGTH(eth_value_precise_raw) > 18, LEFT(eth_value_precise_raw, LENGTH(eth_value_precise_raw) - 18) || '.' || RIGHT(eth_value_precise_raw, 18), '0.' || LPAD(eth_value_precise_raw, 18, '0')) AS rough_conversion, - IFF( - POSITION( - '.000000000000000000' IN rough_conversion - ) > 0, - LEFT(rough_conversion, LENGTH(rough_conversion) - 19), - REGEXP_REPLACE( - rough_conversion, - '0*$', - '' - ) - ) AS eth_value_precise - FROM - {{ ref('silver__transactions') }} - ) + {{ ref('silver__transactions') }} diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index abc7dae..ac5ee31 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -21,10 +21,16 @@ models: - name: TO_ADDRESS description: '{{ doc("arb_to_address") }}' - name: ETH_VALUE - description: '{{ doc("arb_value") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE_RAW - description: '{{ doc("precise_amount_unadjusted") }}' + description: '{{ doc("amount_deprecation") }}' - name: ETH_VALUE_PRECISE + description: '{{ doc("amount_deprecation") }}' + - name: VALUE + description: '{{ doc("arb_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE description: '{{ doc("precise_amount_adjusted") }}' - name: TX_FEE description: '{{ doc("arb_tx_fee") }}' @@ -61,4 +67,10 @@ models: - name: L1_BLOCK_NUMBER description: The block number of the transaction on Ethereum Mainnet. - name: GAS_USED_FOR_L1 - description: The gas used by the transaction on Ethereum Mainnet. \ No newline at end of file + description: The gas used by the transaction on Ethereum Mainnet. + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/defi/defi__dim_dex_liquidity_pools.sql b/models/gold/defi/defi__dim_dex_liquidity_pools.sql index a57848b..4b91e24 100644 --- a/models/gold/defi/defi__dim_dex_liquidity_pools.sql +++ b/models/gold/defi/defi__dim_dex_liquidity_pools.sql @@ -22,6 +22,20 @@ SELECT pool_name, tokens, symbols, - decimals + decimals, + COALESCE ( + complete_dex_liquidity_pools_id, + {{ dbt_utils.generate_surrogate_key( + ['block_number','platform','version'] + ) }} + ) AS dim_dex_liquidity_pools_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver_dex__complete_dex_liquidity_pools') }} \ No newline at end of file diff --git a/models/gold/defi/defi__dim_dex_liquidity_pools.yml b/models/gold/defi/defi__dim_dex_liquidity_pools.yml index 9c828f1..99262fe 100644 --- a/models/gold/defi/defi__dim_dex_liquidity_pools.yml +++ b/models/gold/defi/defi__dim_dex_liquidity_pools.yml @@ -23,4 +23,10 @@ models: - name: SYMBOLS description: '{{ doc("eth_dex_lp_symbols") }}' - name: DECIMALS - description: '{{ doc("eth_dex_lp_decimals") }}' \ No newline at end of file + description: '{{ doc("eth_dex_lp_decimals") }}' + - name: DIM_DEX_LIQUIDITY_POOLS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index 7d5df1b..26ece34 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -36,5 +36,19 @@ SELECT token_out, symbol_in, symbol_out, - _log_id + _log_id, + COALESCE ( + complete_dex_swaps_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index'] + ) }} + ) AS ez_dex_swaps_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver_dex__complete_dex_swaps') }} \ No newline at end of file diff --git a/models/gold/defi/defi__ez_dex_swaps.yml b/models/gold/defi/defi__ez_dex_swaps.yml index 738107f..e011d1c 100644 --- a/models/gold/defi/defi__ez_dex_swaps.yml +++ b/models/gold/defi/defi__ez_dex_swaps.yml @@ -39,7 +39,7 @@ models: - name: EVENT_INDEX description: '{{ doc("arb_event_index") }}' - name: _LOG_ID - description: '{{ doc("arb_log_id_events") }}' + description: '{{ doc("internal_column") }}' - name: ORIGIN_FUNCTION_SIGNATURE description: '{{ doc("arb_tx_origin_sig") }}' - name: ORIGIN_FROM_ADDRESS @@ -50,4 +50,9 @@ models: description: '{{ doc("eth_dex_swaps_amount_in_unadj") }}' - name: AMOUNT_OUT_UNADJ description: '{{ doc("eth_dex_swaps_amount_out_unadj") }}' - + - name: EZ_DEX_SWAPS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/nft/nft__ez_nft_sales.sql b/models/gold/nft/nft__ez_nft_sales.sql index ab86312..565e482 100644 --- a/models/gold/nft/nft__ez_nft_sales.sql +++ b/models/gold/nft/nft__ez_nft_sales.sql @@ -15,6 +15,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -37,6 +38,20 @@ SELECT creator_fee_usd, origin_from_address, origin_to_address, - origin_function_signature + origin_function_signature, + COALESCE ( + complete_nft_sales_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index', 'nft_address','tokenId','platform_exchange_version'] + ) }} + ) AS ez_nft_sales_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__complete_nft_sales') }} diff --git a/models/gold/nft/nft__ez_nft_sales.yml b/models/gold/nft/nft__ez_nft_sales.yml index 7616579..cd0f5bc 100644 --- a/models/gold/nft/nft__ez_nft_sales.yml +++ b/models/gold/nft/nft__ez_nft_sales.yml @@ -9,7 +9,9 @@ models: - name: BLOCK_TIMESTAMP description: '{{ doc("nft_blocktime") }}' - name: TX_HASH - description: '{{ doc("nft_tx_hash") }}' + description: '{{ doc("nft_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("nft_event_index") }}' - name: EVENT_TYPE description: '{{ doc("nft_event_type") }}' - name: PLATFORM_ADDRESS @@ -54,3 +56,9 @@ models: description: '{{ doc("nft_origin_to") }}' - name: ORIGIN_FUNCTION_SIGNATURE description: '{{ doc("nft_origin_sig") }}' + - name: EZ_NFT_SALES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/nft/nft__ez_nft_transfers.sql b/models/gold/nft/nft__ez_nft_transfers.sql index 58faec4..54be5a8 100644 --- a/models/gold/nft/nft__ez_nft_transfers.sql +++ b/models/gold/nft/nft__ez_nft_transfers.sql @@ -13,13 +13,28 @@ SELECT block_number, tx_hash, event_index, + intra_event_index, event_type, contract_address AS nft_address, project_name, from_address AS nft_from_address, to_address AS nft_to_address, tokenId, - erc1155_value + erc1155_value, + COALESCE ( + nft_transfers_id, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index','intra_event_index'] + ) }} + ) AS ez_nft_transfers_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__nft_transfers') }} diff --git a/models/gold/nft/nft__ez_nft_transfers.yml b/models/gold/nft/nft__ez_nft_transfers.yml index fc46b2b..dfc0c45 100644 --- a/models/gold/nft/nft__ez_nft_transfers.yml +++ b/models/gold/nft/nft__ez_nft_transfers.yml @@ -12,6 +12,8 @@ models: description: '{{ doc("nft_tx_hash") }}' - name: EVENT_INDEX description: '{{ doc("nft_event_index") }}' + - name: INTRA_EVENT_INDEX + description: '{{ doc("nft_intra_event_index") }}' - name: EVENT_TYPE description: '{{ doc("nft_event_type") }}' - name: NFT_ADDRESS @@ -26,4 +28,9 @@ models: description: '{{ doc("nft_tokenid") }}' - name: ERC1155_VALUE description: '{{ doc("nft_erc1155_value") }}' - \ No newline at end of file + - name: EZ_NFT_TRANSFERS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql index 8b6c904..794c615 100644 --- a/models/gold/price/price__dim_asset_metadata.sql +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -10,6 +10,20 @@ SELECT symbol, NAME, decimals, - provider + provider, + COALESCE ( + asset_metadata_all_providers_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address','symbol','id','provider'] + ) }} + ) AS dim_asset_metadata_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__asset_metadata_all_providers') }} \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.yml b/models/gold/price/price__dim_asset_metadata.yml index 866753d..fe7022b 100644 --- a/models/gold/price/price__dim_asset_metadata.yml +++ b/models/gold/price/price__dim_asset_metadata.yml @@ -23,3 +23,9 @@ models: description: The specific address representing the asset in a specific platform. - name: DECIMALS description: The number of decimal places the token needs adjusted where token values exist. + - name: DIM_ASSET_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql index aa4a3df..6788bc7 100644 --- a/models/gold/price/price__ez_asset_metadata.sql +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -9,6 +9,20 @@ SELECT id, symbol, NAME, - decimals + decimals, + COALESCE ( + asset_metadata_priority_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address'] + ) }} + ) AS ez_asset_metadata_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__asset_metadata_priority') }} \ No newline at end of file diff --git a/models/gold/price/price__ez_asset_metadata.yml b/models/gold/price/price__ez_asset_metadata.yml index eda50f7..a7a2c3e 100644 --- a/models/gold/price/price__ez_asset_metadata.yml +++ b/models/gold/price/price__ez_asset_metadata.yml @@ -18,3 +18,9 @@ models: description: The specific address representing the asset in a specific platform. - name: DECIMALS description: The number of decimal places the token needs adjusted where token values exist. + - name: EZ_ASSET_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__ez_hourly_token_prices.sql b/models/gold/price/price__ez_hourly_token_prices.sql index a584b9e..088ec86 100644 --- a/models/gold/price/price__ez_hourly_token_prices.sql +++ b/models/gold/price/price__ez_hourly_token_prices.sql @@ -10,6 +10,20 @@ SELECT symbol, decimals, price, - is_imputed + is_imputed, + COALESCE ( + hourly_prices_priority_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour'] + ) }} + ) AS ez_hourly_token_prices_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__hourly_prices_priority') }} diff --git a/models/gold/price/price__ez_hourly_token_prices.yml b/models/gold/price/price__ez_hourly_token_prices.yml index 3e15547..e705476 100644 --- a/models/gold/price/price__ez_hourly_token_prices.yml +++ b/models/gold/price/price__ez_hourly_token_prices.yml @@ -20,4 +20,10 @@ models: - name: PRICE description: Closing price of the recorded hour in USD - name: IS_IMPUTED - description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) \ No newline at end of file + description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) + - name: EZ_HOURLY_TOKEN_PRICES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/price/price__fact_hourly_token_prices.sql b/models/gold/price/price__fact_hourly_token_prices.sql index 1894e4a..8bb363e 100644 --- a/models/gold/price/price__fact_hourly_token_prices.sql +++ b/models/gold/price/price__fact_hourly_token_prices.sql @@ -9,6 +9,20 @@ SELECT token_address, price, is_imputed, - provider + provider, + COALESCE ( + hourly_prices_all_providers_id, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour', 'provider'] + ) }} + ) AS fact_hourly_token_prices_id, + COALESCE( + inserted_timestamp, + '2000-01-01' + ) AS inserted_timestamp, + COALESCE( + modified_timestamp, + '2000-01-01' + ) AS modified_timestamp FROM {{ ref('silver__hourly_prices_all_providers') }} diff --git a/models/gold/price/price__fact_hourly_token_prices.yml b/models/gold/price/price__fact_hourly_token_prices.yml index 9337e7b..69f452a 100644 --- a/models/gold/price/price__fact_hourly_token_prices.yml +++ b/models/gold/price/price__fact_hourly_token_prices.yml @@ -13,4 +13,10 @@ models: - name: PRICE description: Closing price of the recorded hour in USD - name: IS_IMPUTED - description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) \ No newline at end of file + description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) + - name: FACT_HOURLY_TOKEN_PRICES_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index e14f6f8..2067905 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -3,7 +3,9 @@ materialized = 'incremental', unique_key = "block_number", cluster_by = "block_timestamp::date", - tags = ['core','non_realtime'] + tags = ['core','non_realtime'], + merge_exclude_columns = ["inserted_timestamp"], + full_refresh = false ) }} SELECT @@ -49,7 +51,13 @@ SELECT ) :: INT AS total_difficulty, DATA :transactionsRoot :: STRING AS transactions_root, DATA :uncles AS uncles, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS blocks_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {% if is_incremental() %} diff --git a/models/silver/core/silver__contracts.sql b/models/silver/core/silver__contracts.sql index 7282fda..45be2cd 100644 --- a/models/silver/core/silver__contracts.sql +++ b/models/silver/core/silver__contracts.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'contract_address', + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -81,7 +82,13 @@ token_names AS ( token_name, token_decimals :: INTEGER AS token_decimals, token_symbol, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['c1.contract_address'] + ) }} AS contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM contracts c1 LEFT JOIN token_names diff --git a/models/silver/core/silver__created_contracts.sql b/models/silver/core/silver__created_contracts.sql index ed3e3f3..7457d05 100644 --- a/models/silver/core/silver__created_contracts.sql +++ b/models/silver/core/silver__created_contracts.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "incremental", unique_key = "created_contract_address", + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -11,7 +12,13 @@ SELECT to_address AS created_contract_address, from_address AS creator_address, input AS created_contract_input, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['to_address'] + ) }} AS created_contracts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('silver__traces') }} WHERE diff --git a/models/silver/core/silver__decoded_logs.sql b/models/silver/core/silver__decoded_logs.sql index a93f976..d85fddd 100644 --- a/models/silver/core/silver__decoded_logs.sql +++ b/models/silver/core/silver__decoded_logs.sql @@ -6,6 +6,7 @@ incremental_predicates = ["dynamic_range", "block_number"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", full_refresh = false, + merge_exclude_columns = ["inserted_timestamp"], tags = ['decoded_logs','reorg'] ) }} @@ -191,7 +192,13 @@ SELECT DATA, event_removed, tx_status, - is_pending + is_pending, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS decoded_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM new_records @@ -216,7 +223,13 @@ SELECT DATA, event_removed, tx_status, - is_pending + is_pending, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS decoded_logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM missing_data {% endif %} diff --git a/models/silver/core/silver__logs.sql b/models/silver/core/silver__logs.sql index 34fccbf..e4487c0 100644 --- a/models/silver/core/silver__logs.sql +++ b/models/silver/core/silver__logs.sql @@ -176,7 +176,13 @@ FROM {% endif %} ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, event_index ORDER BY diff --git a/models/silver/core/silver__native_transfers.sql b/models/silver/core/silver__native_transfers.sql new file mode 100644 index 0000000..4a39c20 --- /dev/null +++ b/models/silver/core/silver__native_transfers.sql @@ -0,0 +1,109 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'block_number', + cluster_by = ['block_timestamp::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core','non_realtime','reorg'] +) }} + +WITH eth_base AS ( + + SELECT + tx_hash, + block_number, + block_timestamp, + identifier, + from_address, + to_address, + eth_value, + _call_id, + _inserted_timestamp, + eth_value_precise_raw, + eth_value_precise, + tx_position, + trace_index + FROM + {{ ref('silver__traces') }} + WHERE + eth_value > 0 + AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND TYPE NOT IN ( + 'DELEGATECALL', + 'STATICCALL' + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '72 hours' + FROM + {{ this }} +) +{% endif %} +), +tx_table AS ( + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + origin_function_signature + FROM + {{ ref('silver__transactions') }} + WHERE + tx_hash IN ( + SELECT + DISTINCT tx_hash + FROM + eth_base + ) + +{% if is_incremental() %} +AND _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) - INTERVAL '72 hours' + FROM + {{ this }} +) +{% endif %} +) +SELECT + tx_hash AS tx_hash, + block_number AS block_number, + block_timestamp AS block_timestamp, + identifier AS identifier, + origin_from_address, + origin_to_address, + origin_function_signature, + from_address, + to_address, + eth_value AS amount, + eth_value_precise_raw AS amount_precise_raw, + eth_value_precise AS amount_precise, + ROUND( + eth_value * price, + 2 + ) AS amount_usd, + _call_id, + _inserted_timestamp, + tx_position, + trace_index, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS native_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + eth_base A + LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }} + ON DATE_TRUNC( + 'hour', + A.block_timestamp + ) = HOUR + JOIN tx_table USING ( + tx_hash, + block_number + ) \ No newline at end of file diff --git a/models/silver/core/silver__receipts.sql b/models/silver/core/silver__receipts.sql index db02972..9d70802 100644 --- a/models/silver/core/silver__receipts.sql +++ b/models/silver/core/silver__receipts.sql @@ -46,10 +46,7 @@ FINAL AS ( ) :: INT AS cumulative_gas_used, utils.udf_hex_to_int( DATA :effectiveGasPrice :: STRING - ) :: INT / pow( - 10, - 9 - ) AS effective_gas_price, + ) :: bigint AS effective_gas_price, DATA :from :: STRING AS from_address, utils.udf_hex_to_int( DATA :gasUsed :: STRING diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index c2b5a92..954f26c 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -554,7 +554,13 @@ FROM {% endif %} ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index ORDER BY diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index d2cf248..b8ecf5d 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -104,7 +104,12 @@ base_tx AS ( A.data :v :: STRING AS v, utils.udf_hex_to_int( A.data :value :: STRING - ) :: FLOAT AS VALUE, + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, A._INSERTED_TIMESTAMP, A.data FROM @@ -130,6 +135,8 @@ new_records AS ( t.position, t.type, t.v, + t.value_precise_raw, + t.value_precise, t.value, block_timestamp, CASE @@ -141,12 +148,18 @@ new_records AS ( tx_success, tx_status, cumulative_gas_used, - effective_gas_price, utils.udf_decimal_adjust( - effective_gas_price * r.gas_used, + r.effective_gas_price, 9 + ) AS effective_gas_price, + utils.udf_decimal_adjust( + r.effective_gas_price * r.gas_used, + 18 ) AS tx_fee_precise, - tx_fee_precise :: FLOAT AS tx_fee, + COALESCE( + tx_fee_precise :: FLOAT, + 0 + ) AS tx_fee, r.type AS tx_type, r.l1BlockNumber AS l1_block_number, r.gas_used_for_l1, @@ -193,6 +206,8 @@ missing_data AS ( t.position, t.type, t.v, + t.value_precise_raw, + t.value_precise, t.value, b.block_timestamp, FALSE AS is_pending, @@ -200,12 +215,18 @@ missing_data AS ( r.tx_success, r.tx_status, r.cumulative_gas_used, - r.effective_gas_price, + utils.udf_decimal_adjust( + r.effective_gas_price, + 9 + ) AS effective_gas_price, utils.udf_decimal_adjust( r.effective_gas_price * r.gas_used, - 9 + 18 ) AS tx_fee_precise_heal, - tx_fee_precise_heal :: FLOAT AS tx_fee, + COALESCE( + tx_fee_precise_heal :: FLOAT, + 0 + ) AS tx_fee, r.type AS tx_type, r.l1BlockNumber AS l1_block_number, r.gas_used_for_l1, @@ -259,6 +280,8 @@ FINAL AS ( POSITION, TYPE, v, + value_precise_raw, + value_precise, VALUE, block_timestamp, is_pending, @@ -298,6 +321,8 @@ SELECT POSITION, TYPE, v, + value_precise_raw, + value_precise, VALUE, block_timestamp, is_pending, @@ -336,10 +361,9 @@ SELECT POSITION, TYPE, v, - VALUE / pow( - 10, - 18 - ) AS VALUE, + VALUE, + value_precise_raw, + value_precise, block_timestamp, CASE WHEN CONCAT( @@ -365,7 +389,13 @@ SELECT l1_block_number, gas_used_for_l1, _inserted_timestamp, - DATA + DATA, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash'] + ) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL WHERE diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index b2e8c2d..f4ee332 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -239,7 +239,13 @@ heal_model AS ( has_decimal, has_price, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM token_transfers @@ -269,7 +275,13 @@ SELECT has_decimal, has_price, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index'] + ) }} AS transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM heal_model {% endif %} diff --git a/models/silver/core/silver__tx_count.sql b/models/silver/core/silver__tx_count.sql index e90e919..926198a 100644 --- a/models/silver/core/silver__tx_count.sql +++ b/models/silver/core/silver__tx_count.sql @@ -1,15 +1,18 @@ {{ config( materialized = 'incremental', unique_key = "block_number", + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} -SELECT - block_number, - MIN(_inserted_timestamp) AS _inserted_timestamp, - COUNT(*) AS tx_count -FROM - {{ ref('silver__transactions') }} +WITH base AS ( + + SELECT + block_number, + MIN(_inserted_timestamp) AS _inserted_timestamp, + COUNT(*) AS tx_count + FROM + {{ ref('silver__transactions') }} {% if is_incremental() %} WHERE @@ -21,4 +24,15 @@ WHERE ) {% endif %} GROUP BY - block_number \ No newline at end of file + block_number +) +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS tx_count_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + base \ No newline at end of file diff --git a/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql b/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql index 19caea8..13575bb 100644 --- a/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql +++ b/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql @@ -586,5 +586,11 @@ SELECT symbols, decimals, _id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number','platform','version'] + ) }} AS complete_dex_liquidity_pools_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL \ No newline at end of file diff --git a/models/silver/dex/silver_dex__complete_dex_swaps.sql b/models/silver/dex/silver_dex__complete_dex_swaps.sql index 98d8634..cd26b80 100644 --- a/models/silver/dex/silver_dex__complete_dex_swaps.sql +++ b/models/silver/dex/silver_dex__complete_dex_swaps.sql @@ -1994,7 +1994,13 @@ SELECT symbol_in, symbol_out, f._log_id, - f._inserted_timestamp + f._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['f.tx_hash','f.event_index'] + ) }} AS complete_dex_swaps_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL f LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} p diff --git a/models/silver/labels/silver__labels.sql b/models/silver/labels/silver__labels.sql index 6955fde..891d25c 100644 --- a/models/silver/labels/silver__labels.sql +++ b/models/silver/labels/silver__labels.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'address', + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -13,7 +14,13 @@ SELECT label_type, label_subtype, address_name, - project_name + project_name, + {{ dbt_utils.generate_surrogate_key( + ['address'] + ) }} AS labels_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__labels') }} WHERE diff --git a/models/silver/nft/silver__complete_nft_sales.sql b/models/silver/nft/silver__complete_nft_sales.sql index 232594d..7b999a1 100644 --- a/models/silver/nft/silver__complete_nft_sales.sql +++ b/models/silver/nft/silver__complete_nft_sales.sql @@ -12,6 +12,7 @@ WITH nft_base_models AS ( block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -51,6 +52,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -90,6 +92,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -129,6 +132,7 @@ SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -238,6 +242,7 @@ final_base AS ( block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -347,6 +352,7 @@ label_fill_sales AS ( block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -392,6 +398,30 @@ label_fill_sales AS ( WHERE t.project_name IS NULL AND C.token_name IS NOT NULL +), +blocks_fill AS ( + SELECT + * exclude ( + complete_nft_sales_id, + inserted_timestamp, + modified_timestamp, + _invocation_id + ) + FROM + {{ this }} + WHERE + block_number IN ( + SELECT + block_number + FROM + label_fill_sales + ) + AND nft_log_id NOT IN ( + SELECT + nft_log_id + FROM + label_fill_sales + ) ) {% endif %}, final_joins AS ( @@ -401,17 +431,23 @@ final_joins AS ( final_base {% if is_incremental() %} -UNION +UNION ALL SELECT * FROM label_fill_sales +UNION ALL +SELECT + * +FROM + blocks_fill {% endif %} ) SELECT block_number, block_timestamp, tx_hash, + event_index, event_type, platform_address, platform_name, @@ -445,7 +481,13 @@ SELECT nft_log_id, input_data, _log_id, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'event_index', 'nft_address','tokenId','platform_exchange_version'] + ) }} AS complete_nft_sales_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM final_joins qualify(ROW_NUMBER() over(PARTITION BY nft_log_id ORDER BY diff --git a/models/silver/nft/silver__complete_nft_sales.yml b/models/silver/nft/silver__complete_nft_sales.yml index 6eb5eb3..4341d5d 100644 --- a/models/silver/nft/silver__complete_nft_sales.yml +++ b/models/silver/nft/silver__complete_nft_sales.yml @@ -28,6 +28,9 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ + - name: EVENT_INDEX + tests: + - not_null - name: PLATFORM_ADDRESS tests: - not_null diff --git a/models/silver/nft/silver__nft_transfers.sql b/models/silver/nft/silver__nft_transfers.sql index a895d17..0357c3c 100644 --- a/models/silver/nft/silver__nft_transfers.sql +++ b/models/silver/nft/silver__nft_transfers.sql @@ -43,7 +43,7 @@ AND TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( SELECT MAX( _inserted_timestamp - ) + ) - INTERVAL '24 hours' FROM {{ this }} ) @@ -234,6 +234,7 @@ all_transfers AS ( erc1155_value, _inserted_timestamp, event_index, + 1 AS intra_event_index, 'erc721_Transfer' AS token_transfer_type, CONCAT( _log_id, @@ -256,6 +257,7 @@ all_transfers AS ( erc1155_value, _inserted_timestamp, event_index, + 1 AS intra_event_index, 'erc1155_TransferSingle' AS token_transfer_type, CONCAT( _log_id, @@ -280,6 +282,7 @@ all_transfers AS ( erc1155_value, _inserted_timestamp, event_index, + intra_event_index, 'erc1155_TransferBatch' AS token_transfer_type, CONCAT( _log_id, @@ -301,6 +304,7 @@ transfer_base AS ( block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, C.token_name AS project_name, from_address, @@ -328,6 +332,7 @@ fill_transfers AS ( t.block_timestamp, t.tx_hash, t.event_index, + t.intra_event_index, t.contract_address, C.token_name AS project_name, t.from_address, @@ -348,6 +353,30 @@ fill_transfers AS ( WHERE t.project_name IS NULL AND C.token_name IS NOT NULL +), +blocks_fill AS ( + SELECT + * exclude ( + nft_transfers_id, + inserted_timestamp, + modified_timestamp, + _invocation_id + ) + FROM + {{ this }} + WHERE + block_number IN ( + SELECT + block_number + FROM + fill_transfers + ) + AND _log_id NOT IN ( + SELECT + _log_id + FROM + fill_transfers + ) ) {% endif %}, final_base AS ( @@ -356,6 +385,7 @@ final_base AS ( block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, project_name, from_address, @@ -370,12 +400,13 @@ final_base AS ( transfer_base {% if is_incremental() %} -UNION +UNION ALL SELECT block_number, block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, project_name, from_address, @@ -388,13 +419,13 @@ SELECT _inserted_timestamp FROM fill_transfers -{% endif %} -) +UNION ALL SELECT block_number, block_timestamp, tx_hash, event_index, + intra_event_index, contract_address, project_name, from_address, @@ -405,6 +436,32 @@ SELECT token_transfer_type, _log_id, _inserted_timestamp +FROM + blocks_fill +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + intra_event_index, + contract_address, + project_name, + from_address, + to_address, + tokenId, + erc1155_value, + event_type, + token_transfer_type, + _log_id, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash','event_index','intra_event_index'] + ) }} AS nft_transfers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM final_base qualify ROW_NUMBER() over ( PARTITION BY _log_id diff --git a/models/silver/nft/silver__nft_transfers.yml b/models/silver/nft/silver__nft_transfers.yml index 6cdc8fa..6ba35b3 100644 --- a/models/silver/nft/silver__nft_transfers.yml +++ b/models/silver/nft/silver__nft_transfers.yml @@ -28,6 +28,12 @@ models: - not_null - dbt_expectations.expect_column_values_to_match_regex: regex: 0[xX][0-9a-fA-F]+ + - name: EVENT_INDEX + tests: + - not_null + - name: INTRA_EVENT_INDEX + tests: + - not_null - name: CONTRACT_ADDRESS tests: - not_null diff --git a/models/silver/prices/silver__asset_metadata_all_providers.sql b/models/silver/prices/silver__asset_metadata_all_providers.sql index e8aaec2..8ccb422 100644 --- a/models/silver/prices/silver__asset_metadata_all_providers.sql +++ b/models/silver/prices/silver__asset_metadata_all_providers.sql @@ -1,5 +1,6 @@ {{ config( materialized = 'incremental', + merge_exclude_columns = ["inserted_timestamp"], unique_key = ['token_address','symbol','id','provider'], tags = ['non_realtime'] ) }} @@ -14,7 +15,13 @@ SELECT token_name AS NAME, token_decimals AS decimals, provider, - p._inserted_timestamp AS _inserted_timestamp + p._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address','symbol','id','provider'] + ) }} AS asset_metadata_all_providers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__asset_metadata_all_providers') }} p diff --git a/models/silver/prices/silver__asset_metadata_priority.sql b/models/silver/prices/silver__asset_metadata_priority.sql index 0c9a8c8..984147e 100644 --- a/models/silver/prices/silver__asset_metadata_priority.sql +++ b/models/silver/prices/silver__asset_metadata_priority.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'token_address', + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -18,7 +19,13 @@ SELECT WHEN p.provider = 'coingecko' THEN 1 WHEN p.provider = 'coinmarketcap' THEN 2 END AS priority, - p._inserted_timestamp + p._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['p.token_address'] + ) }} AS asset_metadata_priority_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__asset_metadata_priority') }} p diff --git a/models/silver/prices/silver__hourly_prices_all_providers.sql b/models/silver/prices/silver__hourly_prices_all_providers.sql index 892f3a9..a9a4b05 100644 --- a/models/silver/prices/silver__hourly_prices_all_providers.sql +++ b/models/silver/prices/silver__hourly_prices_all_providers.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = ['token_address', 'hour', 'provider'], + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -10,7 +11,13 @@ SELECT provider, price, is_imputed, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour', 'provider'] + ) }} AS hourly_prices_all_providers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__hourly_prices_all_providers') }} WHERE diff --git a/models/silver/prices/silver__hourly_prices_priority.sql b/models/silver/prices/silver__hourly_prices_priority.sql index bb31417..d38afa6 100644 --- a/models/silver/prices/silver__hourly_prices_priority.sql +++ b/models/silver/prices/silver__hourly_prices_priority.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = ['token_address', 'hour'], + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -14,7 +15,13 @@ SELECT C.token_symbol, m.symbol ) AS symbol, - C.token_decimals AS decimals + C.token_decimals AS decimals, + {{ dbt_utils.generate_surrogate_key( + ['p.token_address', 'p.hour'] + ) }} AS hourly_prices_priority_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__hourly_prices_priority') }} p diff --git a/models/silver/prices/silver__hourly_prices_priority_eth.sql b/models/silver/prices/silver__hourly_prices_priority_eth.sql index 9ed40ef..9bae884 100644 --- a/models/silver/prices/silver__hourly_prices_priority_eth.sql +++ b/models/silver/prices/silver__hourly_prices_priority_eth.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', unique_key = ['token_address', 'hour'], + merge_exclude_columns = ["inserted_timestamp"], tags = ['non_realtime'] ) }} @@ -9,7 +10,13 @@ SELECT token_address, price, is_imputed, - _inserted_timestamp + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['token_address', 'hour'] + ) }} AS hourly_prices_priority_eth_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__hourly_prices_priority_eth') }} WHERE