diff --git a/models/main_package/core/gold/core/core__fact_transactions_logs.sql b/models/main_package/core/gold/core/core__fact_transactions_logs.sql index 87108a3..3b41e19 100644 --- a/models/main_package/core/gold/core/core__fact_transactions_logs.sql +++ b/models/main_package/core/gold/core/core__fact_transactions_logs.sql @@ -11,7 +11,8 @@ cluster_by = ['block_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_id, tx_id)", incremental_predicates = [fsc_ibc.standard_predicate()], - tags = ['gold', 'core', 'phase_2'] + tags = ['gold', 'core', 'phase_2'], + enabled = false ) }} SELECT diff --git a/models/main_package/core/silver/core/silver__blocks.sql b/models/main_package/core/silver/core/silver__blocks.sql index 1c3e174..e20a8b9 100644 --- a/models/main_package/core/silver/core/silver__blocks.sql +++ b/models/main_package/core/silver/core/silver__blocks.sql @@ -81,6 +81,8 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM bronze_blocks +WHERE + block_id is not null QUALIFY ROW_NUMBER() OVER ( PARTITION BY chain_id, block_id diff --git a/models/main_package/core/silver/core/silver__transactions.sql b/models/main_package/core/silver/core/silver__transactions.sql index 9c6d31c..84500e0 100644 --- a/models/main_package/core/silver/core/silver__transactions.sql +++ b/models/main_package/core/silver/core/silver__transactions.sql @@ -1,7 +1,6 @@ {# Get variables #} {% set vars = return_vars() %} --- depends_on: {{ ref('bronze__transactions') }} --- depends_on: {{ ref('bronze__transactions_fr') }} + {{ config ( materialized = "incremental", incremental_strategy = 'merge', @@ -11,12 +10,15 @@ tags = ['silver', 'core', 'phase_2'] ) }} +-- depends_on: {{ ref('bronze__transactions') }} +-- depends_on: {{ ref('bronze__transactions_fr') }} + WITH bronze_transactions AS ( SELECT - VALUE :BLOCK_ID :: INT AS block_id, + VALUE :BLOCK_ID_REQUESTED AS block_id, TO_TIMESTAMP_NTZ( - DATA:BLOCK_TIMESTAMP::STRING, + VALUE:BLOCK_TIMESTAMP::STRING, 'YYYY_MM_DD_HH_MI_SS_FF3' ) AS block_timestamp, DATA :hash :: STRING AS tx_id, @@ -38,10 +40,9 @@ WITH bronze_transactions AS ( DATA :tx_result :events AS msgs, DATA, partition_key, - DATA :BLOCK_ID_REQUESTED AS block_id_requested, inserted_timestamp AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['block_id_requested', 'tx_id'] + ['block_id', 'tx_id'] ) }} AS transactions_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, @@ -53,8 +54,10 @@ WITH bronze_transactions AS ( {% else %} {{ ref('bronze__transactions_fr') }} {% endif %} -{% if is_incremental() %} WHERE + DATA <> PARSE_JSON('[]') +{% if is_incremental() %} +AND inserted_timestamp >= ( SELECT MAX(_inserted_timestamp) @@ -68,6 +71,6 @@ SELECT FROM bronze_transactions QUALIFY ROW_NUMBER() OVER ( - PARTITION BY block_id_requested, tx_id + PARTITION BY block_id, tx_id ORDER BY _inserted_timestamp DESC ) = 1 diff --git a/models/main_package/core/silver/core/silver__transactions_logs.sql b/models/main_package/core/silver/core/silver__transactions_logs.sql index 2f753ee..2ab53ce 100644 --- a/models/main_package/core/silver/core/silver__transactions_logs.sql +++ b/models/main_package/core/silver/core/silver__transactions_logs.sql @@ -29,6 +29,7 @@ WITH event_attributes AS ( LATERAL FLATTEN(input => e.value:attributes) a WHERE t.tx_log IS NOT NULL AND t.tx_succeeded = TRUE + AND IS_OBJECT(PARSE_JSON(t.tx_log)) {% if is_incremental() %} AND t._inserted_timestamp >= ( SELECT MAX(_inserted_timestamp) diff --git a/models/main_package/core/streamline/realtime/streamline__blocks_realtime.sql b/models/main_package/core/streamline/realtime/streamline__blocks_realtime.sql index c3226b6..eef13f6 100644 --- a/models/main_package/core/streamline/realtime/streamline__blocks_realtime.sql +++ b/models/main_package/core/streamline/realtime/streamline__blocks_realtime.sql @@ -29,8 +29,7 @@ SELECT 'POST', '{{ vars.GLOBAL_NODE_URL }}', OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'streamline' + 'Content-Type', 'application/json' ), OBJECT_CONSTRUCT( 'id', block_id, diff --git a/models/main_package/core/streamline/realtime/streamline__transactions_realtime.sql b/models/main_package/core/streamline/realtime/streamline__transactions_realtime.sql index 92e35b9..78ddd1a 100644 --- a/models/main_package/core/streamline/realtime/streamline__transactions_realtime.sql +++ b/models/main_package/core/streamline/realtime/streamline__transactions_realtime.sql @@ -68,8 +68,7 @@ numbers AS ( 'POST', '{{ vars.GLOBAL_NODE_URL }}', OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'streamline' + 'Content-Type', 'application/json' ), OBJECT_CONSTRUCT( 'id', block_id, diff --git a/models/main_package/core/streamline/realtime/streamline__tx_counts_realtime.sql b/models/main_package/core/streamline/realtime/streamline__tx_counts_realtime.sql index 5a3f1b4..b69f964 100644 --- a/models/main_package/core/streamline/realtime/streamline__tx_counts_realtime.sql +++ b/models/main_package/core/streamline/realtime/streamline__tx_counts_realtime.sql @@ -50,8 +50,7 @@ SELECT 'POST', '{{ vars.GLOBAL_NODE_URL }}', OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'streamline' + 'Content-Type', 'application/json' ), OBJECT_CONSTRUCT( 'id', block_id,