An-4550_stats (#151)

* On branch An-4550_stats
Changes to be committed:
	modified:   models/descriptions/__overview__.md
	modified:   models/descriptions/inserted_timestamp.md
	new file:   models/descriptions/modified_timestamp.md
	new file:   models/descriptions/pk.md
	new file:   models/descriptions/stats_core.md
	new file:   models/gold/stats/stats__ez_core_metrics_hourly.sql
	new file:   models/gold/stats/stats__ez_core_metrics_hourly.yml
	new file:   models/silver/stats/silver_stats__core_metrics_block_hourly.sql
	new file:   models/silver/stats/silver_stats__core_metrics_block_hourly.yml
	new file:   models/silver/stats/silver_stats__core_metrics_hourly.sql
	new file:   models/silver/stats/silver_stats__core_metrics_hourly.yml

* decimals

* blockchain 100 per

* speed up

* 5-> 50

macros/snowflake_api/get_blockchain_api.sql

@@ -39,7 +39,7 @@ INSERT INTO
               *,
               MOD(ROW_NUMBER() over(
             ORDER BY
-              min_block), 200) rn_mod_out
+              min_block), 100) rn_mod_out
             FROM
               (
                 SELECT
@@ -114,10 +114,7 @@ SELECT
         osmosis._internal.api_keys
       WHERE
         provider = 'allthatnode'
-    ),
-    {
-      'Content-Type': 'application/json'
-    },
+    ),{ 'Content-Type': 'application/json' },
     call
   ) AS DATA,
   SYSDATE()

models/descriptions/__overview__.md

@@ -21,56 +21,59 @@ There is more information on how to use dbt docs in the last section of this doc
 ### DeFi Tables (`OSMOSIS`.`DEFI`.`<table_name>`)
 ### Governance Tables (`OSMOSIS`.`GOV`.`<table_name>`)
 ### Prices Tables (`OSMOSIS`.`PRICE`.`<table_name>`)
+### Statistics/Analytics Tables (`OSMOSIS`.`STATS`.`<table_name>`)
 
 
 
 **Core Dimension Tables:**
-- [dim_labels](#!/model/model.osmosis.core__dim_labels)
-- [dim_tokens](#!/model/model.osmosis.core__dim_tokens)
+- [dim_labels](#!/model/model.osmosis_models.core__dim_labels)
+- [dim_tokens](#!/model/model.osmosis_models.core__dim_tokens)
 
 **Core Fact Tables:**
-- [fact_airdrop](#!/model/model.osmosis.core__fact_airdrop)
-- [fact_blocks](#!/model/model.osmosis.core__fact_blocks)
-- [fact_daily_balances](#!/model/model.osmosis.core__fact_daily_balances)
-- [fact_msg_attributes](#!/model/model.osmosis.core__fact_msg_attributes)
-- [fact_msgs](#!/model/model.osmosis.core__fact_msgs)
-- [fact_token_day](#!/model/model.osmosis.core__fact_token_day)
-- [fact_transactions](#!/model/model.osmosis.core__fact_transactions)
-- [fact_transfers](#!/model/model.osmosis.core__fact_transfers)
+- [fact_airdrop](#!/model/model.osmosis_models.core__fact_airdrop)
+- [fact_blocks](#!/model/model.osmosis_models.core__fact_blocks)
+- [fact_daily_balances](#!/model/model.osmosis_models.core__fact_daily_balances)
+- [fact_msg_attributes](#!/model/model.osmosis_models.core__fact_msg_attributes)
+- [fact_msgs](#!/model/model.osmosis_models.core__fact_msgs)
+- [fact_token_day](#!/model/model.osmosis_models.core__fact_token_day)
+- [fact_transactions](#!/model/model.osmosis_models.core__fact_transactions)
+- [fact_transfers](#!/model/model.osmosis_models.core__fact_transfers)
 
 **Core Convenience Tables:**
-- [ez_icns](#!/model/model.osmosis.core__ez_icns)
+- [ez_icns](#!/model/model.osmosis_models.core__ez_icns)
 
 **DeFi Dimension Tables:**
-- [dim_liquidity_pools](#!/model/model.osmosis.defi__dim_liquidity_pools)
+- [dim_liquidity_pools](#!/model/model.osmosis_models.defi__dim_liquidity_pools)
 
 **DeFi Fact Tables:**
-- [fact_liquidity_provider_actions](#!/model/model.osmosis.defi__fact_liquidity_provider_actions)
-- [fact_locked_liquidity_actions](#!/model/model.osmosis.defi__fact_locked_liquidity_actions)
-- [fact_pool_fee_day](#!/model/model.osmosis.defi__fact_pool_fee_day)
-- [fact_pool_hour](#!/model/model.osmosis.defi__fact_pool_hour)
-- [fact_superfluid_staking](#!/model/model.osmosis.defi__fact_superfluid_staking)
-- [fact_swaps](#!/model/model.osmosis.defi__fact_swaps)
+- [fact_liquidity_provider_actions](#!/model/model.osmosis_models.defi__fact_liquidity_provider_actions)
+- [fact_locked_liquidity_actions](#!/model/model.osmosis_models.defi__fact_locked_liquidity_actions)
+- [fact_pool_fee_day](#!/model/model.osmosis_models.defi__fact_pool_fee_day)
+- [fact_pool_hour](#!/model/model.osmosis_models.defi__fact_pool_hour)
+- [fact_superfluid_staking](#!/model/model.osmosis_models.defi__fact_superfluid_staking)
+- [fact_swaps](#!/model/model.osmosis_models.defi__fact_swaps)
 
 **Governance Dimension Tables:**
-- [dim_dim_vote_options](#!/model/model.osmosis.core__dim_vote_options)
+- [dim_dim_vote_options](#!/model/model.osmosis_models.core__dim_vote_options)
 
 **Governance Fact Tables:**
-- [fact_governance_proposal_deposits](#!/model/model.osmosis.gov__fact_governance_proposal_deposits)
-- [fact_governance_submit_proposal](#!/model/model.osmosis.gov__fact_governance_submit_proposal)
-- [fact_governance_validator_votes](#!/model/model.osmosis.gov__fact_governance_validator_votes)
-- [fact_governance_votes](#!/model/model.osmosis.cgov__fact_governance_votes)
-- [fact_staking](#!/model/model.osmosis.cgov__fact_staking)
-- [fact_staking_rewards](#!/model/model.osmosis.gov__fact_staking_rewards)
-- [fact_validators](#!/model/model.osmosis.gov__fact_validators)
+- [fact_governance_proposal_deposits](#!/model/model.osmosis_models.gov__fact_governance_proposal_deposits)
+- [fact_governance_submit_proposal](#!/model/model.osmosis_models.gov__fact_governance_submit_proposal)
+- [fact_governance_validator_votes](#!/model/model.osmosis_models.gov__fact_governance_validator_votes)
+- [fact_governance_votes](#!/model/model.osmosis_models.cgov__fact_governance_votes)
+- [fact_staking](#!/model/model.osmosis_models.cgov__fact_staking)
+- [fact_staking_rewards](#!/model/model.osmosis_models.gov__fact_staking_rewards)
+- [fact_validators](#!/model/model.osmosis_models.gov__fact_validators)
 
 
 **Prices FacDimensiont Tables:**
-- [dim_prices ](#!/model/model.osmosis.core__dim_prices)
+- [dim_prices ](#!/model/model.osmosis_models.core__dim_prices)
 
 **Prices Convenience Tables:**
-- [ez_prices](#!/model/model.osmosis.core__ez_prices)
+- [ez_prices](#!/model/model.osmosis_models.core__ez_prices)
 
+**Stats EZ Tables:**
+- [ez_core_metrics_hourly](#!/model/model.osmosis_models.ez_core_metrics_hourly)
 
 
 ## **Data Model Overview**

models/descriptions/inserted_timestamp.md

@@ -1,5 +1,5 @@
 {% docs inserted_timestamp %}
 
-The time the block was inserted into to the Flipside database. 
+The utc timestamp at which the row was inserted into the table.
 
 {% enddocs %}

models/descriptions/modified_timestamp.md

@@ -0,0 +1,5 @@
+{% docs modified_timestamp %}
+
+The utc timestamp at which the row was last modified.
+
+{% enddocs %}

models/descriptions/pk.md

@@ -0,0 +1,5 @@
+{% docs pk %}
+
+The unique identifier for each row in the table.
+
+{% enddocs %}

models/descriptions/stats_core.md

@@ -0,0 +1,65 @@
+{% docs ez_core_metrics_hourly_table_doc %}
+
+A convenience table that aggregates block and transaction related metrics using various aggregate functions such as SUM, COUNT, MIN and MAX from the fact_transactions table, on an hourly basis. Stats for the current hour will be updated as new data arrives.
+
+{% enddocs %}
+
+{% docs block_timestamp_hour %}
+
+The hour of the timestamp of the block.
+
+{% enddocs %}
+
+{% docs block_id_min %}
+
+The minimum block id in the hour.
+
+{% enddocs %}
+
+{% docs block_id_max %}
+
+The maximum block id in the hour.
+
+{% enddocs %}
+
+{% docs block_count %}
+
+The number of blocks in the hour.
+
+{% enddocs %}
+
+{% docs transaction_count %}
+
+The number of transactions in the hour.
+
+{% enddocs %}
+
+{% docs transaction_count_success %}
+
+The number of successful transactions in the hour.
+
+{% enddocs %}
+
+{% docs transaction_count_failed %}
+
+The number of failed transactions in the hour.
+
+{% enddocs %}
+
+{% docs unique_from_count %}
+
+The number of unique tx_from addresses in the hour.
+
+{% enddocs %}
+
+{% docs total_fees_native %}
+
+The sum of all fees in the hour, in the native fee currency.
+
+{% enddocs %}
+
+{% docs total_fees_usd %}
+
+The sum of all fees in the hour, in USD.
+
+{% enddocs %}

models/gold/stats/stats__ez_core_metrics_hourly.sql

@@ -0,0 +1,60 @@
+{{ config(
+    materialized = 'view',
+    tags = ['noncore'],
+    meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STATS, METRICS, CORE, HOURLY',
+    }} }
+) }}
+
+WITH txs AS (
+
+    SELECT
+        block_timestamp_hour,
+        transaction_count,
+        transaction_count_success,
+        transaction_count_failed,
+        unique_from_count,
+        total_fees AS total_fees_native,
+        LAST_VALUE(
+            p.price ignore nulls
+        ) over (
+            ORDER BY
+                block_timestamp_hour rows unbounded preceding
+        ) AS imputed_price,
+        core_metrics_hourly_id AS ez_core_metrics_hourly_id,
+        s.inserted_timestamp AS inserted_timestamp,
+        s.modified_timestamp AS modified_timestamp
+    FROM
+        {{ ref('silver_stats__core_metrics_hourly') }}
+        s
+        LEFT JOIN {{ ref('price__ez_prices') }}
+        p
+        ON s.block_timestamp_hour = p.recorded_hour
+        AND p.currency = 'uosmo'
+)
+SELECT
+    A.block_timestamp_hour,
+    A.block_number_min,
+    A.block_number_max,
+    A.block_count,
+    b.transaction_count,
+    b.transaction_count_success,
+    b.transaction_count_failed,
+    b.unique_from_count,
+    b.total_fees_native,
+    ROUND(
+        b.total_fees_native * b.imputed_price,
+        2
+    ) AS total_fees_usd,
+    A.core_metrics_block_hourly_id AS ez_core_metrics_hourly_id,
+    GREATEST(
+        A.inserted_timestamp,
+        b.inserted_timestamp
+    ) AS inserted_timestamp,
+    GREATEST(
+        A.modified_timestamp,
+        b.modified_timestamp
+    ) AS modified_timestamp
+FROM
+    {{ ref('silver_stats__core_metrics_block_hourly') }} A
+    JOIN txs b
+    ON A.block_timestamp_hour = b.block_timestamp_hour

models/gold/stats/stats__ez_core_metrics_hourly.yml

@@ -0,0 +1,32 @@
+version: 2
+models:
+  - name: stats__ez_core_metrics_hourly
+    description: '{{ doc("ez_core_metrics_hourly_table_doc") }}'
+
+    columns:
+      - name: BLOCK_TIMESTAMP_HOUR
+        description: '{{ doc("block_timestamp_hour") }}'
+      - name: BLOCK_NUMBER_MIN
+        description: '{{ doc("block_id_min") }}'
+      - name: BLOCK_NUMBER_MAX
+        description: '{{ doc("block_id_max") }}'
+      - name: BLOCK_COUNT
+        description: '{{ doc("block_count") }}'
+      - name: TRANSACTION_COUNT
+        description: '{{ doc("transaction_count") }}'
+      - name: TRANSACTION_COUNT_SUCCESS
+        description: '{{ doc("transaction_count_success") }}'
+      - name: TRANSACTION_COUNT_FAILED
+        description: '{{ doc("transaction_count_failed") }}'
+      - name: UNIQUE_FROM_COUNT
+        description: '{{ doc("unique_from_count") }}'
+      - name: TOTAL_FEES_NATIVE
+        description: '{{ doc("total_fees_native") }}'
+      - name: TOTAL_FEES_USD
+        description: '{{ doc("total_fees_usd") }}'
+      - name: EZ_CORE_METRICS_HOURLY_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 

models/silver/stats/silver_stats__core_metrics_block_hourly.sql

@@ -0,0 +1,43 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = "block_timestamp_hour",
+    cluster_by = ['block_timestamp_hour::DATE'],
+    tags = ['noncore']
+) }}
+
+SELECT
+    DATE_TRUNC(
+        'hour',
+        block_timestamp
+    ) AS block_timestamp_hour,
+    MIN(block_id) AS block_number_min,
+    MAX(block_id) AS block_number_max,
+    COUNT(
+        1
+    ) AS block_count,
+    MAX(_inserted_timestamp) AS _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['block_timestamp_hour']
+    ) }} AS core_metrics_block_hourly_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    {{ ref('silver__blocks') }}
+WHERE
+    block_timestamp_hour < DATE_TRUNC('hour', systimestamp())
+
+{% if is_incremental() %}
+AND DATE_TRUNC(
+    'hour',
+    _inserted_timestamp
+) >= (
+    SELECT
+        MAX(DATE_TRUNC('hour', _inserted_timestamp)) - INTERVAL '12 hours'
+    FROM
+        {{ this }}
+)
+{% endif %}
+GROUP BY
+    1

models/silver/stats/silver_stats__core_metrics_block_hourly.yml

@@ -0,0 +1,41 @@
+version: 2
+models:
+  - name: silver_stats__core_metrics_block_hourly
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_TIMESTAMP_HOUR
+    columns:
+      - name: BLOCK_TIMESTAMP_HOUR
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ
+      - name: BLOCK_NUMBER_MIN
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: BLOCK_NUMBER_MAX
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: BLOCK_COUNT
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1

models/silver/stats/silver_stats__core_metrics_hourly.sql

@@ -0,0 +1,74 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = "block_timestamp_hour",
+    cluster_by = ['block_timestamp_hour::DATE'],
+    tags = ['noncore']
+) }}
+
+SELECT
+    DATE_TRUNC(
+        'hour',
+        block_timestamp
+    ) AS block_timestamp_hour,
+    COUNT(
+        DISTINCT tx_id
+    ) AS transaction_count,
+    COUNT(
+        DISTINCT CASE
+            WHEN tx_succeeded THEN tx_id
+        END
+    ) AS transaction_count_success,
+    COUNT(
+        DISTINCT CASE
+            WHEN NOT tx_succeeded THEN tx_id
+        END
+    ) AS transaction_count_failed,
+    COUNT(
+        DISTINCT tx_from
+    ) AS unique_from_count,
+    SUM(
+        CASE
+            WHEN fee ILIKE '%uosmo' THEN TRY_CAST(
+                SPLIT_PART(
+                    TRIM(
+                        REGEXP_REPLACE(
+                            fee,
+                            '[^[:digit:]]',
+                            ' '
+                        )
+                    ),
+                    ' ',
+                    0
+                ) AS INT
+            )
+        END / pow(
+            10,
+            6
+        )
+    ) AS total_fees,
+    MAX(inserted_timestamp) AS _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['block_timestamp_hour']
+    ) }} AS core_metrics_hourly_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    {{ ref('silver__transactions_final') }}
+WHERE
+    block_timestamp_hour < DATE_TRUNC('hour', systimestamp())
+
+{% if is_incremental() %}
+AND DATE_TRUNC(
+    'hour',
+    modified_timestamp
+) >= (
+    SELECT
+        MAX(DATE_TRUNC('hour', modified_timestamp)) - INTERVAL '12 hours'
+    FROM
+        {{ this }}
+)
+{% endif %}
+GROUP BY
+    1

models/silver/stats/silver_stats__core_metrics_hourly.yml

@@ -0,0 +1,56 @@
+version: 2
+models:
+  - name: silver_stats__core_metrics_hourly
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_TIMESTAMP_HOUR
+    columns:
+      - name: BLOCK_TIMESTAMP_HOUR
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ
+      - name: TRANSACTION_COUNT
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: TRANSACTION_COUNT_SUCCESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: TRANSACTION_COUNT_FAILED
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: UNIQUE_FROM_COUNT
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: TOTAL_FEES
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DECIMAL
+                - FLOAT
+                - NUMBER
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1

models/streamline/core/realtime/streamline__tx_search_realtime.sql

@@ -6,24 +6,14 @@
     )
 ) }}
 -- depends_on: {{ ref('streamline__complete_tx_search') }}
-WITH blocks AS (
+WITH transactions_counts_by_block AS (
 
-    SELECT
-        block_number
-    FROM
-        {{ ref("streamline__complete_txcount") }}
-    WHERE
-        block_number >= 11975572
-),
-transactions_counts_by_block AS (
     SELECT
         tc.block_number,
         tc.txcount
     FROM
         {{ ref("streamline__complete_txcount") }}
         tc
-        INNER JOIN blocks b
-        ON tc.block_number = b.block_number
 ),
 numbers AS (
     -- Recursive CTE to generate numbers. We'll use the maximum txcount value to limit our recursion.