From 8039a92698704141573db5628b681a8fb5eb6969 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 11:46:44 -0600 Subject: [PATCH 1/6] updated join criteria and revised incremental logic to modified_timestamp --- models/gold/stats/stats__ez_core_metrics_hourly.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 539ea44..338404e 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -35,11 +35,11 @@ FROM LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON s.block_timestamp_hour = p.hour + AND p.is_native WHERE - p.is_native {% if is_incremental() %} - AND s.block_timestamp_hour >= ( + s.modified_timestamp >= ( SELECT - MAX(block_timestamp_hour) + MAX(modified_timestamp) FROM {{ this }}) {% endif %} \ No newline at end of file From 71d49b89201b0e923acea39c0d3998e994f126f3 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 13:34:56 -0600 Subject: [PATCH 2/6] trying different stats approach --- .../gold/stats/stats__ez_core_metrics_hourly.sql | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 338404e..17bc8c5 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -37,9 +37,11 @@ FROM ON s.block_timestamp_hour = p.hour AND p.is_native WHERE - {% if is_incremental() %} - s.modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM {{ this }}) - {% endif %} \ No newline at end of file + block_timestamp_hour < DATE_TRUNC('hour', CURRENT_TIMESTAMP) +{% if is_incremental() %} + AND block_timestamp_hour >= DATEADD('hour', -{{ var('HOURLY_METRICS_LOOKBACK_HOURS', 24) }}, ( + SELECT + DATE_TRUNC('hour', MAX(modified_timestamp)) + FROM {{ this }} + )) +{% endif %} \ No newline at end of file From e3880f26186f8d86f6719a3d3a642eed36cc42d4 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 14:19:34 -0600 Subject: [PATCH 3/6] added stats and prices to overview file --- models/descriptions/__overview__.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 653298a..f8b4700 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -25,7 +25,21 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_transactions_block_metadata](#!/model/model.movement_models.core__fact_transactions_block_metadata) - [fact_transactions_state_checkpoint](#!/model/model.movement_models.core__fact_transactions_state_checkpoint) -## **Data Model Overview** +### Price Tables (`movement`.`PRICE`.``) + +**Dimension Tables:** +- [price.dim_asset_metadata](https://flipsidecrypto.github.io/stellar-models/#!/model/model.movement_models.price__dim_asset_metadata) + +**Fact Tables:** +- [price.fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/stellar-models/#!/model/model.movement_models.price__fact_prices_ohlc_hourly) + +**Convenience Views:** +- [price.ez_prices_hourly](https://flipsidecrypto.github.io/stellar-models/#!/model/model.movement_models.price__ez_prices_hourly) +- [price.ez_asset_metadata](https://flipsidecrypto.github.io/stellar-models/#!/model/model.movement_models.price__ez_asset_metadata) + +### Stats Tables (`stellar`.`STATS`.``) + +- [stats.ez_core_metrics_hourly](https://flipsidecrypto.github.io/stellar-models/#!/model/model.movement_models.stats__ez_core_metrics_hourly) The movement models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** From 206e36832798755f15e3fca532fa14ebba79796a Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 14:29:17 -0600 Subject: [PATCH 4/6] updated logic with VAR --- models/gold/stats/stats__ez_core_metrics_hourly.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 17bc8c5..4cc821a 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -37,7 +37,8 @@ FROM ON s.block_timestamp_hour = p.hour AND p.is_native WHERE - block_timestamp_hour < DATE_TRUNC('hour', CURRENT_TIMESTAMP) + p.price IS NOT NULL + AND block_timestamp_hour < DATE_TRUNC('hour', CURRENT_TIMESTAMP) {% if is_incremental() %} AND block_timestamp_hour >= DATEADD('hour', -{{ var('HOURLY_METRICS_LOOKBACK_HOURS', 24) }}, ( SELECT From 1080ea135848edecd402d06b22d6395f3acd81bd Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 14:41:38 -0600 Subject: [PATCH 5/6] modified logic --- models/gold/stats/stats__ez_core_metrics_hourly.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 4cc821a..44e5abe 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -32,13 +32,12 @@ SELECT FROM {{ ref('silver_stats__core_metrics_hourly') }} s - LEFT JOIN {{ ref('price__ez_prices_hourly') }} + JOIN {{ ref('price__ez_prices_hourly') }} p ON s.block_timestamp_hour = p.hour AND p.is_native WHERE - p.price IS NOT NULL - AND block_timestamp_hour < DATE_TRUNC('hour', CURRENT_TIMESTAMP) + block_timestamp_hour < DATE_TRUNC('hour', CURRENT_TIMESTAMP) {% if is_incremental() %} AND block_timestamp_hour >= DATEADD('hour', -{{ var('HOURLY_METRICS_LOOKBACK_HOURS', 24) }}, ( SELECT From 370282ec89e00cffe5c159be4c9fec8bee38bbc6 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 15:00:30 -0600 Subject: [PATCH 6/6] switched to VAR lookup --- .../stats/stats__ez_core_metrics_hourly.sql | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index 44e5abe..de07a18 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -8,6 +8,33 @@ tags = ['noncore'] ) }} +-- depends_on: {{ ref('core__fact_transactions') }} + +{% if execute %} + +{% if is_incremental() %} +{% set query %} + +SELECT + DATE_TRUNC('hour', MIN(block_timestamp)) AS block_timestamp_hour + FROM + {{ ref('core__fact_transactions') }} + WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endset %} + {% set min_block_timestamp_hour = run_query(query).columns [0].values() [0] %} +{% endif %} + +{% if not min_block_timestamp_hour or min_block_timestamp_hour == 'None' %} + {% set min_block_timestamp_hour = '2099-01-01' %} +{% endif %} +{% endif %} + SELECT block_timestamp_hour, block_number_min, @@ -32,16 +59,16 @@ SELECT FROM {{ ref('silver_stats__core_metrics_hourly') }} s - JOIN {{ ref('price__ez_prices_hourly') }} + LEFT JOIN {{ ref('price__ez_prices_hourly') }} p ON s.block_timestamp_hour = p.hour AND p.is_native WHERE block_timestamp_hour < DATE_TRUNC('hour', CURRENT_TIMESTAMP) {% if is_incremental() %} - AND block_timestamp_hour >= DATEADD('hour', -{{ var('HOURLY_METRICS_LOOKBACK_HOURS', 24) }}, ( - SELECT - DATE_TRUNC('hour', MAX(modified_timestamp)) - FROM {{ this }} - )) +AND + block_timestamp_hour >= COALESCE( + DATEADD('hour', -4, '{{ min_block_timestamp_hour }}'), + '2025-01-01 00:00:00' + ) {% endif %} \ No newline at end of file