diff --git a/macros/tests/date_gaps.sql b/macros/tests/date_gaps.sql new file mode 100644 index 0000000..0f298a2 --- /dev/null +++ b/macros/tests/date_gaps.sql @@ -0,0 +1,58 @@ +{% macro date_gaps( + table, + partition_by, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + {{ table }} + ) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column }}, + DATEDIFF( + days, + {{ previous_column }}, + {{ column }} + ) - 1 AS gap +FROM + source + {% if varargs -%} +LEFT JOIN ( + {% for x in varargs %} + ( + {{ dbt_utils.date_spine( + datepart = "day", + start_date = x.start_date, + end_date = x.end_date + ) }} + ) + {{- "UNION ALL" if not loop.last -}} + {% endfor %} +) exclude + ON source.day = exclude.date_day + {%- endif %} +WHERE + DATEDIFF( + days, + {{ previous_column }}, + {{ column }} + ) > 1 + {{ "AND source.day != exclude.date_day" if varargs }} +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/negative_one.sql b/macros/tests/negative_one.sql new file mode 100644 index 0000000..22193aa --- /dev/null +++ b/macros/tests/negative_one.sql @@ -0,0 +1,10 @@ +{% test negative_one( + model, + column_name +) %} +SELECT + * +FROM + {{ model }} +WHERE + {{ column_name }} = '-1' {% endtest %} diff --git a/macros/tests/sequence_distinct_gaps.sql b/macros/tests/sequence_distinct_gaps.sql new file mode 100644 index 0000000..6cd0ccc --- /dev/null +++ b/macros/tests/sequence_distinct_gaps.sql @@ -0,0 +1,34 @@ +{% macro sequence_distinct_gaps( + table, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM ( + SELECT DISTINCT {{ column }} FROM {{ table }} + ) + + ) +SELECT + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/sequence_distinct_gaps_dim_block_id.sql b/macros/tests/sequence_distinct_gaps_dim_block_id.sql new file mode 100644 index 0000000..955ca66 --- /dev/null +++ b/macros/tests/sequence_distinct_gaps_dim_block_id.sql @@ -0,0 +1,43 @@ +{% macro sequence_distinct_gaps_dim_block_id( + table, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + ( + SELECT + DISTINCT {{ column }} + FROM + {{ table }} A + JOIN {{ ref('core__dim_block') }} + b + ON A.dim_block_id = b.dim_block_id + WHERE + A.dim_block_id <> '-1' + AND b.block_timestamp :: DATE < CURRENT_DATE + ) + ) +SELECT + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/sequence_gaps_buffered_look_back.sql b/macros/tests/sequence_gaps_buffered_look_back.sql new file mode 100644 index 0000000..ea9deeb --- /dev/null +++ b/macros/tests/sequence_gaps_buffered_look_back.sql @@ -0,0 +1,52 @@ +{% macro sequence_gaps_buffered_look_back( + table, + partition_by, + column, + delayed_column, + delayed_period + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }}, + LAG( + {{ delayed_column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ delayed_column }} + FROM + {{ table }} + ) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +AND + {{ delayed_column }} < ( + SELECT + MAX( + {{ delayed_column }} + ) + FROM + {{ this }} + ) - INTERVAL '{{ delayed_period }}' +{% endmacro %} diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md new file mode 100644 index 0000000..6f7d34b --- /dev/null +++ b/models/descriptions/__overview__.md @@ -0,0 +1,123 @@ +{% docs __overview__ %} + +# Welcome to the Flipside Crypto Maya Models Documentation + +## **What does this documentation cover?** + +The documentation included here details the design of the Maya +tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/insights/dashboards/maya) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/maya-models) + +## **How do I use these docs?** + +The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. + +If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them. + +There is more information on how to use dbt docs in the last section of this document. + +## **Quick Links to Table Documentation** + +**Click on the links below to jump to the documentation for each schema.** + +### Core Schema (`maya`.`CORE`.``) + +- [core.dim_block](#!/model/model.maya_models.core__dim_block) +- [core.dim_midgard](#!/model/model.maya_models.core__dim_midgard) +- [core.fact_mayaname_change_events](#!/model/model.maya_models.core__fact_mayaname_change_events) +- [core.fact_set_mimir_events](#!/model/model.maya_models.core__fact_set_mimir_events) +- [core.fact_transfer_events](#!/model/model.maya_models.core__fact_transfer_events) +- [core.fact_transfers](#!/model/model.maya_models.core__fact_transfers) + +### Defi Schema + +- [defi.fact_active_vault_events](#!/model/model.maya_models.defi__fact_active_vault_events) +- [defi.fact_add_events](#!/model/model.maya_models.defi__fact_add_events) +- [defi.fact_block_pool_depths](#!/model/model.maya_models.defi__fact_block_pool_depths) +- [defi.fact_block_rewards](#!/model/model.maya_models.defi__fact_block_rewards) +- [defi.fact_bond_actions](#!/model/model.maya_models.defi__fact_bond_actions) +- [defi.fact_bond_events](#!/model/model.maya_models.defi__fact_bond_events) +- [defi.fact_daily_earnings](#!/model/model.maya_models.defi__fact_daily_earnings) +- [defi.fact_daily_pool_stats](#!/model/model.maya_models.defi__fact_daily_pool_stats) +- [defi.fact_daily_tvl](#!/model/model.maya_models.defi__fact_daily_tvl) +- [defi.fact_failed_deposit_messages](#!/model/model.maya_models.defi__fact_failed_deposit_messages) +- [defi.fact_fee_events](#!/model/model.maya_models.defi__fact_fee_events) +- [defi.fact_gas_events](#!/model/model.maya_models.defi__fact_gas_events) +- [defi.fact_inactive_vault_events](#!/model/model.maya_models.defi__fact_inactive_vault_events) +- [defi.fact_liquidity_actions](#!/model/model.maya_models.defi__fact_liquidity_actions) +- [defi.fact_outbound_events](#!/model/model.maya_models.defi__fact_outbound_events) +- [defi.fact_pending_liquidity_events](#!/model/model.maya_models.defi__fact_pending_liquidity_events) +- [defi.fact_pool_balance_change_events](#!/model/model.maya_models.defi__fact_pool_balance_change_events) +- [defi.fact_pool_block_balances](#!/model/model.maya_models.defi__fact_pool_block_balances) +- [defi.fact_pool_block_fees](#!/model/model.maya_models.defi__fact_pool_block_fees) +- [defi.fact_pool_block_statistics](#!/model/model.maya_models.defi__fact_pool_block_statistics) +- [defi.fact_pool_events](#!/model/model.maya_models.defi__fact_pool_events) +- [defi.fact_refund_events](#!/model/model.maya_models.defi__fact_refund_events) +- [defi.fact_reserve_events](#!/model/model.maya_models.defi__fact_reserve_events) +- [defi.fact_rewards_event_entries](#!/model/model.maya_models.defi__fact_rewards_event_entries) +- [defi.fact_rewards_events](#!/model/model.maya_models.defi__fact_rewards_events) +- [defi.fact_send_message_events](#!/model/model.maya_models.defi__fact_send_message_events) +- [defi.fact_slash_liquidity_events](#!/model/model.maya_models.defi__fact_slash_liquidity_events) +- [defi.fact_stake_events](#!/model/model.maya_models.defi__fact_stake_events) +- [defi.fact_streamling_swap_details_events](#!/model/model.maya_models.defi__fact_streamling_swap_details_events) +- [defi.fact_swaps](#!/model/model.maya_models.defi__fact_swaps) +- [defi.fact_swaps_events](#!/model/model.maya_models.defi__fact_swaps_events) +- [defi.fact_total_block_rewards](#!/model/model.maya_models.defi__fact_total_block_rewards) +- [defi.fact_total_value_locked](#!/model/model.maya_models.defi__fact_total_value_locked) +- [defi.fact_update_node_account_status_events](#!/model/model.maya_models.defi__fact_update_node_account_status_events) +- [defi.fact_withdraw_events](#!/model/model.maya_models.defi__fact_withdraw_events) + +### Governance Schema + +- [gov.fact_new_node_events](#!/model/model.maya_models.gov__fact_new_node_events) +- [gov.fact_set_ip_address_events](#!/model/model.maya_models.gov__fact_set_ip_address_events) +- [gov.fact_set_node_keys_events](#!/model/model.maya_models.gov__fact_set_node_keys_events) +- [gov.fact_set_version_events](#!/model/model.maya_models.gov__fact_set_version_events) +- [gov.fact_slash_amounts](#!/model/model.maya_models.gov__fact_slash_amounts) +- [gov.fact_slash_points](#!/model/model.maya_models.gov__fact_slash_points) +- [gov.fact_validator_request_leave_events](#!/model/model.maya_models.gov__fact_validator_request_leave_events) + +### Price Schema + + - [price.factcacao_price](#!/model/model.maya_models.price__fact_cacao_price) + + +## **Data Model Overview** + +While maya models are built a few different ways, they are organized into three layers of sql models: **bronze, silver, and gold (or core).** + +- Bronze: Data is loaded in from the source as a view +- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here +- Gold (or core): Final views and tables that are available publicly + +The dimension tables are sourced from a variety of on-chain and off-chain sources. + +Convenience views (denoted ez\_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. + +## **Using dbt docs** + +### Navigation + +You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models in the project. + +### Database Tab + +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown in this interface, as they do not exist in the database. + +### Graph Exploration + +You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models. + +On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. + +Once expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). + +Note that you can also right-click on models to interactively filter and explore the graph. + +### **More information** + +- [maya on Flipside Crypto](https://flipsidecrypto.xyz/insights/dashboards/maya) +- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) +- [Github](https://github.com/FlipsideCrypto/maya-models) +- [What is dbt?](https://docs.getdbt.com/docs/introduction) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/address.md b/models/descriptions/address.md new file mode 100644 index 0000000..8ee2615 --- /dev/null +++ b/models/descriptions/address.md @@ -0,0 +1,5 @@ +{% docs address %} + +The account public key + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/asset.md b/models/descriptions/asset.md new file mode 100644 index 0000000..22564e6 --- /dev/null +++ b/models/descriptions/asset.md @@ -0,0 +1,5 @@ +{% docs asset %} + +Asset name or pool name + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_date.md b/models/descriptions/block_date.md new file mode 100644 index 0000000..3bc3977 --- /dev/null +++ b/models/descriptions/block_date.md @@ -0,0 +1,5 @@ +{% docs block_date %} + +Date of block minting(without a timezone) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_id.md b/models/descriptions/block_id.md new file mode 100644 index 0000000..86b272d --- /dev/null +++ b/models/descriptions/block_id.md @@ -0,0 +1,5 @@ +{% docs block_id %} + +ID of the confirmed block + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md new file mode 100644 index 0000000..1792c06 --- /dev/null +++ b/models/descriptions/block_timestamp.md @@ -0,0 +1,5 @@ +{% docs block_timestamp %} + +Timestamp of block minting(without a timezone) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/blockchain.md b/models/descriptions/blockchain.md new file mode 100644 index 0000000..b5098dc --- /dev/null +++ b/models/descriptions/blockchain.md @@ -0,0 +1,5 @@ +{% docs blockchain %} + +The name of the blockchain + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/bond_type.md b/models/descriptions/bond_type.md new file mode 100644 index 0000000..6675b27 --- /dev/null +++ b/models/descriptions/bond_type.md @@ -0,0 +1,5 @@ +{% docs bond_type %} + +There are 4 different types of bond:bond_reward, bond_paid, bond_cost, bond_returned + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/from_address.md b/models/descriptions/from_address.md new file mode 100644 index 0000000..97a82e3 --- /dev/null +++ b/models/descriptions/from_address.md @@ -0,0 +1,5 @@ +{% docs from_address %} + +The sending address for this event + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/id.md b/models/descriptions/id.md new file mode 100644 index 0000000..4075e01 --- /dev/null +++ b/models/descriptions/id.md @@ -0,0 +1,5 @@ +{% docs id %} + +The primary key for the table. + +{% enddocs %} diff --git a/models/descriptions/inserted_timestamp.md b/models/descriptions/inserted_timestamp.md new file mode 100644 index 0000000..14703c3 --- /dev/null +++ b/models/descriptions/inserted_timestamp.md @@ -0,0 +1,5 @@ +{% docs inserted_timestamp %} + +The utc timestamp at which the row was inserted into the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/memo.md b/models/descriptions/memo.md new file mode 100644 index 0000000..59ef667 --- /dev/null +++ b/models/descriptions/memo.md @@ -0,0 +1,5 @@ +{% docs memo %} + +The memo for this event + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/modified_timestamp.md b/models/descriptions/modified_timestamp.md new file mode 100644 index 0000000..c4d18ee --- /dev/null +++ b/models/descriptions/modified_timestamp.md @@ -0,0 +1,5 @@ +{% docs modified_timestamp %} + +The utc timestamp at which the row was last modified. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/pk.md b/models/descriptions/pk.md new file mode 100644 index 0000000..8dfdfb5 --- /dev/null +++ b/models/descriptions/pk.md @@ -0,0 +1,5 @@ +{% docs pk %} + +The unique identifier for each row in the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/pool_name.md b/models/descriptions/pool_name.md new file mode 100644 index 0000000..3a171b3 --- /dev/null +++ b/models/descriptions/pool_name.md @@ -0,0 +1,5 @@ +{% docs pool_name %} + +Name of the pool -- also asset name in other tables + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/prices.md b/models/descriptions/prices.md new file mode 100644 index 0000000..7c7f6a1 --- /dev/null +++ b/models/descriptions/prices.md @@ -0,0 +1,137 @@ +{% docs prices_dim_asset_metadata_table_doc %} + +A comprehensive dimensional table holding asset metadata and other relevant details pertaining to each id, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this dimensional model. If you are interested in using a curated data set instead, please utilize ez_asset_metadata. + +{% enddocs %} + +{% docs prices_ez_asset_metadata_table_doc %} + +A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address and native asset. This data set is highly curated and contains metadata for one unique asset per blockchain. + +{% enddocs %} + +{% docs prices_fact_prices_ohlc_hourly_table_doc %} + +A comprehensive fact table holding id and provider specific open, high, low, close hourly prices, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this fact based model. If you are interested in using a curated data set instead, please utilize ez_prices_hourly. + +{% enddocs %} + +{% docs prices_ez_prices_hourly_table_doc %} + +A convenience table for determining token prices by address and blockchain, and native asset prices by symbol and blockchain. This data set is highly curated and contains metadata for one price per hour per unique asset and blockchain. + +{% enddocs %} + +{% docs prices_provider %} + +The provider or source of the data. + +{% enddocs %} + +{% docs prices_asset_id %} + +The unique identifier representing the asset. + +{% enddocs %} + +{% docs prices_name %} + +The name of asset. + +{% enddocs %} + +{% docs prices_symbol %} + +The symbol of asset. + +{% enddocs %} + +{% docs prices_token_address %} + +The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. + +{% enddocs %} + +{% docs prices_blockchain %} + +The Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_blockchain_id %} + +The unique identifier of the Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_decimals %} + +The number of decimals for the asset. May be NULL. + +{% enddocs %} + +{% docs prices_is_native %} + +A flag indicating assets native to the respective blockchain. + +{% enddocs %} + +{% docs prices_is_deprecated %} + +A flag indicating if the asset is deprecated or no longer supported by the provider. + +{% enddocs %} + +{% docs prices_id_deprecation %} + +Deprecating soon! Please use the `asset_id` column instead. + +{% enddocs %} + +{% docs prices_decimals_deprecation %} + +Deprecating soon! Please use the decimals column in `ez_asset_metadata` or join in `dim_contracts` instead. + +{% enddocs %} + +{% docs prices_hour %} + +Hour that the price was recorded at. + +{% enddocs %} + +{% docs prices_price %} + +Closing price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_is_imputed %} + +A flag indicating if the price was imputed, or derived, from the last arriving record. This is generally used for tokens with low-liquidity or inconsistent reporting. + +{% enddocs %} + +{% docs prices_open %} + +Opening price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_high %} + +Highest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_low %} + +Lowest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_close %} + +Closing price of the recorded hour in USD + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/sk.md b/models/descriptions/sk.md new file mode 100644 index 0000000..7a011e4 --- /dev/null +++ b/models/descriptions/sk.md @@ -0,0 +1,5 @@ +{% docs sk %} + +The surrogate key for the table. Will be unique and is used as a foreign key in other tables + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/to_address.md b/models/descriptions/to_address.md new file mode 100644 index 0000000..cda3c1b --- /dev/null +++ b/models/descriptions/to_address.md @@ -0,0 +1,5 @@ +{% docs to_address %} + +The receiving address for this event + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_id.md b/models/descriptions/tx_id.md new file mode 100644 index 0000000..18ac5e1 --- /dev/null +++ b/models/descriptions/tx_id.md @@ -0,0 +1,5 @@ +{% docs tx_id %} + +The unique transaction id + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_block.sql b/models/gold/core/core__dim_block.sql new file mode 100644 index 0000000..1f15154 --- /dev/null +++ b/models/gold/core/core__dim_block.sql @@ -0,0 +1,86 @@ +{{ config( + materialized = 'incremental', + unique_key = 'dim_block_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_id >= (select min(block_id) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['height'] + ) }} AS dim_block_id, + height AS block_id, + block_timestamp, + block_date, + block_hour, + block_week, + block_month, + block_quarter, + block_year, + block_DAYOFMONTH, + block_DAYOFWEEK, + block_DAYOFYEAR, + TIMESTAMP, + HASH, + agg_state, + _INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__block_log') }} + +{% if is_incremental() %} +WHERE + block_id >= ( + SELECT + MAX( + block_id - 600 --about 1 hour + ) + FROM + {{ this }} + ) +UNION ALL +SELECT + '-1' AS dim_block_id, + -1 AS block_id, + '1900-01-01' :: datetime AS block_timestamp, + NULL AS block_date, + NULL AS block_hour, + NULL AS block_week, + NULL AS block_month, + NULL AS block_quarter, + NULL AS block_year, + NULL AS block_DAYOFMONTH, + NULL AS block_DAYOFWEEK, + NULL AS block_DAYOFYEAR, + NULL AS TIMESTAMP, + NULL AS HASH, + NULL AS agg_state, + '1900-01-01' :: DATE AS _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + '1900-01-01' :: DATE AS inserted_timestamp, + '1900-01-01' :: DATE AS modified_timestamp +UNION ALL +SELECT + '-2' AS dim_block_id, + -2 AS block_id, + NULL AS block_timestamp, + NULL AS block_date, + NULL AS block_hour, + NULL AS block_week, + NULL AS block_month, + NULL AS block_quarter, + NULL AS block_year, + NULL AS block_DAYOFMONTH, + NULL AS block_DAYOFWEEK, + NULL AS block_DAYOFYEAR, + NULL AS TIMESTAMP, + NULL AS HASH, + NULL AS agg_state, + '1900-01-01' :: DATE AS _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + '1900-01-01' :: DATE AS inserted_timestamp, + '1900-01-01' :: DATE AS modified_timestamp +{% endif %} diff --git a/models/gold/core/core__dim_block.yml b/models/gold/core/core__dim_block.yml new file mode 100644 index 0000000..7d9b23f --- /dev/null +++ b/models/gold/core/core__dim_block.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: core__dim_block + description: "Records of all blocks that have occurred on Thorchain, dating back to the genesis block. " + columns: + - name: DIM_BLOCK_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_DATE + description: "{{ doc('block_date') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_HOUR + description: "{{ doc('block_date') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_WEEK + description: "{{ doc('block_date') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_MONTH + description: "The numeric month of block minting(without a timezone)" + - name: BLOCK_QUARTER + description: "The numeric quarter of block minting(without a timezone)" + - name: BLOCK_YEAR + description: "The numeric year of block minting(without a timezone)" + - name: BLOCK_DAYOFMONTH + description: "The numeric day of month of block minting(without a timezone)" + - name: BLOCK_DAYOFWEEK + description: "The numeric day of week of block minting(without a timezone)" + - name: BLOCK_DAYOFYEAR + description: "The numeric day of year of block minting(without a timezone)" + - name: TIMESTAMP + description: "integer value of the block_teimstamp" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: HASH + description: "block hash" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: agg_state + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: DIM_BLOCK_ID diff --git a/models/gold/core/core__dim_midgard.sql b/models/gold/core/core__dim_midgard.sql new file mode 100644 index 0000000..73df47a --- /dev/null +++ b/models/gold/core/core__dim_midgard.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + '2.10.0' AS midgard_version diff --git a/models/gold/core/core__fact_mayaname_change_events.sql b/models/gold/core/core__fact_mayaname_change_events.sql new file mode 100644 index 0000000..01e89f0 --- /dev/null +++ b/models/gold/core/core__fact_mayaname_change_events.sql @@ -0,0 +1,77 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_mayaname_change_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + NAME, + chain, + address, + registration_fee_e8, + fund_amount_e8, + height, + expire, + owner, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__mayaname_change_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.name'] + ) }} AS fact_mayaname_change_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + NAME, + chain, + address, + registration_fee_e8, + fund_amount_e8, + height, + expire, + owner, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + event_id, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_mayaname_change_events.yml b/models/gold/core/core__fact_mayaname_change_events.yml new file mode 100644 index 0000000..3764602 --- /dev/null +++ b/models/gold/core/core__fact_mayaname_change_events.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: core__fact_mayaname_change_events + description: "Fact table that shows name change events" + columns: + - name: FACT_MAYANAME_CHANGE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: OWNER + description: "" + - name: CHAIN + description: "{{ doc('blockchain') }}" + - name: ADDRESS + description: "{{ doc('address') }}" + - name: expire + description: "" + - name: NAME + description: "" + - name: FUND_AMOUNT_E8 + description: "" + - name: REGISTRATION_FEE_E8 + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_MAYANAME_CHANGE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/core/core__fact_set_mimir_events.sql b/models/gold/core/core__fact_set_mimir_events.sql new file mode 100644 index 0000000..958d8b0 --- /dev/null +++ b/models/gold/core/core__fact_set_mimir_events.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_mimir_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + key, + VALUE, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_mimir_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.key','a.block_timestamp'] + ) }} AS fact_set_mimir_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + key, + VALUE, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_set_mimir_events.yml b/models/gold/core/core__fact_set_mimir_events.yml new file mode 100644 index 0000000..e20ab40 --- /dev/null +++ b/models/gold/core/core__fact_set_mimir_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: core__fact_set_mimir_events + description: "Fact table that shows set mimir events" + columns: + - name: FACT_SET_MIMIR_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: KEY + description: "" + - name: VALUE + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_MIMIR_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/core/core__fact_transfer_events.sql b/models/gold/core/core__fact_transfer_events.sql new file mode 100644 index 0000000..3c10ba5 --- /dev/null +++ b/models/gold/core/core__fact_transfer_events.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_transfer_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + from_address, + to_address, + asset, + amount_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__transfer_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.from_address', 'a.to_address', 'a.asset', 'a.amount_e8'] + ) }} AS fact_transfer_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + from_address, + to_address, + asset, + amount_e8, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_transfer_events.yml b/models/gold/core/core__fact_transfer_events.yml new file mode 100644 index 0000000..b7ef274 --- /dev/null +++ b/models/gold/core/core__fact_transfer_events.yml @@ -0,0 +1,48 @@ +version: 2 +models: + - name: core__fact_transfer_events + description: "Fact table containing stake events" + columns: + - name: FACT_TRANSFER_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: FROM_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: AMOUNT_E8 + description: "The asset amount for this event" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TRANSFER_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql new file mode 100644 index 0000000..a669a72 --- /dev/null +++ b/models/gold/core/core__fact_transfers.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_transfers_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + from_address, + to_address, + asset, + cacao_amount, + cacao_amount_usd, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__transfers') }} + +{% if is_incremental() %} +WHERE + block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_transfers_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + from_address, + to_address, + asset, + cacao_amount, + cacao_amount_usd, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id diff --git a/models/gold/core/core__fact_transfers.yml b/models/gold/core/core__fact_transfers.yml new file mode 100644 index 0000000..6b41db7 --- /dev/null +++ b/models/gold/core/core__fact_transfers.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: core__fact_transfers + description: "Fact table shows the transfer action between different address" + columns: + - name: FACT_TRANSFERS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: FROM_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: cacao_AMOUNT + description: "The transferred cacao amount" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "The transferred cacao amount in USD" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TRANSFERS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_active_vault_events.sql b/models/gold/defi/defi__fact_active_vault_events.sql new file mode 100644 index 0000000..60f7b2b --- /dev/null +++ b/models/gold/defi/defi__fact_active_vault_events.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_active_vault_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + add_asgard_addr, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__active_vault_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp','a.add_asgard_addr'] + ) }} AS fact_active_vault_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + add_asgard_addr, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_active_vault_events.yml b/models/gold/defi/defi__fact_active_vault_events.yml new file mode 100644 index 0000000..bf40224 --- /dev/null +++ b/models/gold/defi/defi__fact_active_vault_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: defi__fact_active_vault_events + description: "Fact table containing the events triggered by the churning activities" + columns: + - name: FACT_ACTIVE_VAULT_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ADD_ASGARD_ADDR + description: "The asgard address added to the vault" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_ACTIVE_VAULT_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_add_events.sql b/models/gold/defi/defi__fact_add_events.sql new file mode 100644 index 0000000..d6ef799 --- /dev/null +++ b/models/gold/defi/defi__fact_add_events.sql @@ -0,0 +1,75 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_add_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + e.block_timestamp, + e.tx_id, + e.cacao_e8, + e.blockchain, + e.asset_e8, + e.pool_name, + e.memo, + e.to_address, + e.from_address, + e.asset, + e.event_id, + _inserted_timestamp + FROM + {{ ref('silver__add_events') }} + e +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.from_address','a.to_address','a.asset','a.memo','a.block_timestamp'] + ) }} AS fact_add_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + A.tx_id, + A.cacao_e8, + A.blockchain, + A.asset_e8, + A.pool_name, + A.memo, + A.to_address, + A.from_address, + A.asset, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_add_events.yml b/models/gold/defi/defi__fact_add_events.yml new file mode 100644 index 0000000..701cc9d --- /dev/null +++ b/models/gold/defi/defi__fact_add_events.yml @@ -0,0 +1,58 @@ +version: 2 +models: + - name: defi__fact_add_events + description: "Fact table containing add events" + columns: + - name: FACT_ADD_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: cacao_E8 + description: "The amount of cacao for this add event" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: ASSET_E8 + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_ADD_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_block_pool_depths.sql b/models/gold/defi/defi__fact_block_pool_depths.sql new file mode 100644 index 0000000..7bbcb36 --- /dev/null +++ b/models/gold/defi/defi__fact_block_pool_depths.sql @@ -0,0 +1,63 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_depths_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset_e8, + cacao_e8, + synth_e8, + block_timestamp, + _inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.pool_name','a.block_timestamp'] + ) }} AS fact_pool_depths_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + cacao_e8, + asset_e8, + synth_e8, + pool_name, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR pool_name IN ( + SELECT + pool_name + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_block_pool_depths.yml b/models/gold/defi/defi__fact_block_pool_depths.yml new file mode 100644 index 0000000..3713ed4 --- /dev/null +++ b/models/gold/defi/defi__fact_block_pool_depths.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_block_pool_depths + description: "Fact table containing all the available pools and its cacao/asset depth at each block interval" + columns: + - name: FACT_POOL_DEPTHS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: cacao_E8 + description: "The cacao depth for this pool at this block" + tests: + - not_null + - name: ASSET_E8 + description: "The asset depth for this pool at this block" + tests: + - not_null + - name: SYNTH_E8 + description: "" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_DEPTHS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_block_rewards.sql b/models/gold/defi/defi__fact_block_rewards.sql new file mode 100644 index 0000000..e5be0a7 --- /dev/null +++ b/models/gold/defi/defi__fact_block_rewards.sql @@ -0,0 +1,51 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_block_rewards_id', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_strategy = 'merge' +) }} + +WITH base AS ( + + SELECT + DAY, + liquidity_fee, + block_rewards, + earnings, + bonding_earnings, + liquidity_earnings, + avg_node_count, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__block_rewards') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_block_rewards_id, + DAY, + liquidity_fee, + block_rewards, + earnings, + bonding_earnings, + liquidity_earnings, + avg_node_count, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_block_rewards.yml b/models/gold/defi/defi__fact_block_rewards.yml new file mode 100644 index 0000000..3266bbb --- /dev/null +++ b/models/gold/defi/defi__fact_block_rewards.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: defi__fact_block_rewards + description: "The summarized rewards information for each block per day" + columns: + - name: FACT_BLOCK_REWARDS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The recorded day" + tests: + - not_null + - unique + - name: LIQUIDITY_FEE + description: "The summarized liquidity cost fee within this day" + tests: + - not_null + - name: BLOCK_REWARDS + description: "The summarized total block rewards within this day" + tests: + - not_null + - name: EARNINGS + description: "The summarized earnings within this day" + tests: + - not_null + - name: BONDING_EARNINGS + description: "The summarized bonding earnings within this day" + tests: + - not_null + - name: LIQUIDITY_EARNINGS + description: "The summarized liquidity earnings fee within this day" + tests: + - not_null + - name: AVG_NODE_COUNT + description: "The summarized average node operators number within this day" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_BLOCK_REWARDS_ID + + diff --git a/models/gold/defi/defi__fact_bond_actions.sql b/models/gold/defi/defi__fact_bond_actions.sql new file mode 100644 index 0000000..779921b --- /dev/null +++ b/models/gold/defi/defi__fact_bond_actions.sql @@ -0,0 +1,88 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = "fact_bond_actions_id", + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH block_prices AS ( + + SELECT + AVG(cacao_usd) AS cacao_usd, + block_id + FROM + {{ ref('silver__prices') }} + GROUP BY + block_id +), +bond_events AS ( + SELECT + block_timestamp, + tx_id, + from_address, + to_address, + asset, + blockchain, + bond_type, + asset_e8, + e8, + memo, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__bond_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['be.event_id'] + ) }} AS fact_bond_actions_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + from_address, + to_address, + asset, + blockchain, + bond_type, + COALESCE(e8 / pow(10, 8), 0) AS asset_amount, + COALESCE( + cacao_usd * asset_e8, + 0 + ) AS asset_usd, + memo, + be._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + bond_events be + JOIN {{ ref('core__dim_block') }} + b + ON be.block_timestamp = b.timestamp + LEFT JOIN block_prices p + ON b.block_id = p.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_bond_actions.yml b/models/gold/defi/defi__fact_bond_actions.yml new file mode 100644 index 0000000..b06cbc6 --- /dev/null +++ b/models/gold/defi/defi__fact_bond_actions.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: defi__fact_bond_actions + description: "Fact table containing the clean bond action table to record the node operators' behaviors" + columns: + - name: FACT_BOND_ACTIONS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 4 + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: ASSET + description: "Same as pool name, which pool this bond events happens" + - name: bond_type + description: "{{ doc('bond_type') }}" + tests: + - not_null + - name: ASSET_AMOUNT + description: "The asset amount for this bond event" + tests: + - not_null + - name: ASSET_USD + description: "Used the price table to calculate the asset in the usd" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_BOND_ACTIONS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_bond_events.sql b/models/gold/defi/defi__fact_bond_events.sql new file mode 100644 index 0000000..e33da15 --- /dev/null +++ b/models/gold/defi/defi__fact_bond_events.sql @@ -0,0 +1,74 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_bond_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + bond_type, + e8, + block_timestamp, + event_id, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__bond_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id'] + ) }} AS fact_bond_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + bond_type, + e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_bond_events.yml b/models/gold/defi/defi__fact_bond_events.yml new file mode 100644 index 0000000..656279d --- /dev/null +++ b/models/gold/defi/defi__fact_bond_events.yml @@ -0,0 +1,71 @@ +version: 2 +models: + - name: defi__fact_bond_events + description: "Fact table containing all the information about the bond activities. THORChain uses a spinoff of the Proof of Stake consensus mechanism called Proof of Bond." + columns: + - name: FACT_BOND_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 4 + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: ASSET + description: "Same as pool name, which pool this bond events happens" + - name: ASSET_E8 + description: "The asset amount for this bond, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: bond_type + description: "{{ doc('bond_type') }}" + tests: + - not_null + - name: E8 + description: "The cacao amount for this bond event" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_BOND_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_daily_earnings.sql b/models/gold/defi/defi__fact_daily_earnings.sql new file mode 100644 index 0000000..5852666 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_earnings.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_daily_earnings_id', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_strategy = 'merge' +) }} + +WITH base AS ( + + SELECT + DAY, + liquidity_fees, + liquidity_fees_usd, + block_rewards, + block_rewards_usd, + total_earnings, + total_earnings_usd, + earnings_to_nodes, + earnings_to_nodes_usd, + earnings_to_pools, + earnings_to_pools_usd, + avg_node_count, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__daily_earnings') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_daily_earnings_id, + DAY, + liquidity_fees, + liquidity_fees_usd, + block_rewards, + block_rewards_usd, + total_earnings, + total_earnings_usd, + earnings_to_nodes, + earnings_to_nodes_usd, + earnings_to_pools, + earnings_to_pools_usd, + avg_node_count, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_daily_earnings.yml b/models/gold/defi/defi__fact_daily_earnings.yml new file mode 100644 index 0000000..0818383 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_earnings.yml @@ -0,0 +1,66 @@ +version: 2 +models: + - name: defi__fact_daily_earnings + description: "The earnings information for the whole THORChain network broken down by daily" + columns: + - name: FACT_DAILY_EARNINGS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The recorded day" + tests: + - not_null + - unique + - name: LIQUIDITY_FEES + description: "The summarized liquidity cost fee within this day" + tests: + - not_null + - name: LIQUIDITY_FEES_USD + description: "The summarized liquidity cost fee within this day in USD" + tests: + - not_null + - name: BLOCK_REWARDS + description: "The summarized total block rewards within this day" + tests: + - not_null + - name: BLOCK_REWARDS_USD + description: "The summarized total block rewards within this day in USD" + tests: + - not_null + - name: TOTAL_EARNINGS + description: "The summarized total earnings within this day" + tests: + - not_null + - name: TOTAL_EARNINGS_USD + description: "The summarized total earnings within this day in USD" + tests: + - not_null + - name: EARNINGS_TO_NODES + description: "The summarized bonding earnings within this day" + tests: + - not_null + - name: EARNINGS_TO_NODES_USD + description: "The summarized bonding earnings within this day in USD" + tests: + - not_null + - name: EARNINGS_TO_POOLS + description: "The summarized liquidity earnings fee within this day" + tests: + - not_null + - name: EARNINGS_TO_POOLS_USD + description: "The summarized liquidity earnings fee within this day in USD" + tests: + - not_null + - name: AVG_NODE_COUNT + description: "The summarized average node operators number within this day" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_DAILY_EARNINGS_ID + + diff --git a/models/gold/defi/defi__fact_daily_pool_stats.sql b/models/gold/defi/defi__fact_daily_pool_stats.sql new file mode 100644 index 0000000..aa43991 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_pool_stats.sql @@ -0,0 +1,115 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_daily_pool_stats_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + pool_name, + system_rewards, + system_rewards_usd, + asset_liquidity, + asset_price, + asset_price_usd, + cacao_liquidity, + cacao_price, + cacao_price_usd, + add_liquidity_count, + add_asset_liquidity, + add_asset_liquidity_usd, + add_cacao_liquidity, + add_cacao_liquidity_usd, + withdraw_count, + withdraw_asset_liquidity, + withdraw_asset_liquidity_usd, + withdraw_cacao_liquidity, + withdraw_cacao_liquidity_usd, + il_protection_paid, + il_protection_paid_usd, + average_slip, + to_asset_average_slip, + to_cacao_average_slip, + swap_count, + to_asset_swap_count, + to_cacao_swap_count, + swap_volume_cacao, + swap_volume_cacao_usd, + to_asset_swap_volume, + to_cacao_swap_volume, + total_swap_fees_cacao, + total_swap_fees_usd, + total_asset_swap_fees, + total_asset_cacao_fees, + unique_member_count, + unique_swapper_count, + liquidity_units, + _UNIQUE_KEY + FROM + {{ ref('silver__daily_pool_stats') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day','a.pool_name'] + ) }} AS fact_daily_pool_stats_id, + DAY, + pool_name, + system_rewards, + system_rewards_usd, + asset_liquidity, + asset_price, + asset_price_usd, + cacao_liquidity, + cacao_price, + cacao_price_usd, + add_liquidity_count, + add_asset_liquidity, + add_asset_liquidity_usd, + add_cacao_liquidity, + add_cacao_liquidity_usd, + withdraw_count, + withdraw_asset_liquidity, + withdraw_asset_liquidity_usd, + withdraw_cacao_liquidity, + withdraw_cacao_liquidity_usd, + il_protection_paid, + il_protection_paid_usd, + average_slip, + to_asset_average_slip, + to_cacao_average_slip, + swap_count, + to_asset_swap_count, + to_cacao_swap_count, + swap_volume_cacao, + swap_volume_cacao_usd, + to_asset_swap_volume, + to_cacao_swap_volume, + total_swap_fees_cacao, + total_swap_fees_usd, + total_asset_swap_fees, + total_asset_cacao_fees, + unique_member_count, + unique_swapper_count, + liquidity_units, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_daily_pool_stats.yml b/models/gold/defi/defi__fact_daily_pool_stats.yml new file mode 100644 index 0000000..31999e7 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_pool_stats.yml @@ -0,0 +1,97 @@ +version: 2 +models: + - name: defi__fact_daily_pool_stats + description: "Fact table that shows the total valued locked and the total value bonded/pooled" + columns: + - name: FACT_DAILY_POOL_STATS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: SYSTEM_REWARDS + description: "The total daily system rewards in cacao to the pool" + - name: SYSTEM_REWARDS_USD + description: "The total daily system rewards in USD to the pool" + - name: ASSET_LIQUIDITY + description: "The total asset liquidity for this pool" + - name: ASSET_PRICE + description: "Current asset price" + - name: ASSET_PRICE_USD + description: "Current asset price in USD" + - name: cacao_LIQUIDITY + description: "The total cacao liquidity for this pool" + - name: cacao_PRICE + description: "Current cacao price" + - name: cacao_PRICE_USD + description: "Current cacao price in USD" + - name: ADD_LIQUIDITY_COUNT + description: "How many times to add liquidity to the pool" + - name: ADD_ASSET_LIQUIDITY + description: "The total amount of asset liquidity added to the pool" + - name: ADD_ASSET_LIQUIDITY_USD + description: "The total amount in usd of asset liquidity added to the pool" + - name: ADD_cacao_LIQUIDITY + description: "The total amount of cacao liquidity added to the pool" + - name: ADD_cacao_LIQUIDITY_USD + description: "The total amount in usd of cacao liquidity added to the pool" + - name: WITHDRAW_COUNT + description: "Time to withdraw from the pool" + - name: WITHDRAW_ASSET_LIQUIDITY + description: "The total amount of asset withdrawn from the pool" + - name: WITHDRAW_ASSET_LIQUIDITY_USD + description: "The total amount in USD of asset withdrawn from the pool" + - name: WITHDRAW_cacao_LIQUIDITY + description: "The total amount of cacao withdrawn from the pool" + - name: WITHDRAW_cacao_LIQUIDITY_USD + description: "The total amount in USD of cacao withdrawn from the pool" + - name: IL_PROTECTION_PAID + description: "Impermanent loss protection cost paid to the network" + - name: IL_PROTECTION_PAID_USD + description: "Impermanent loss protection cost in USD paid to the network" + - name: AVERAGE_SLIP + description: "Average slippage " + - name: TO_ASSET_AVERAGE_SLIP + description: "When swap from cacao to Asset, the average slippage" + - name: TO_cacao_AVERAGE_SLIP + description: "When swap from Asset to cacao, the average slippage" + - name: SWAP_COUNT + description: "Total number of swaps" + - name: TO_ASSET_SWAP_COUNT + description: "Total number of swaps from cacao TO Asset" + - name: TO_cacao_SWAP_COUNT + description: "Total number of swaps from Asset TO cacao" + - name: SWAP_VOLUME_cacao + description: "The swap amount of cacao" + - name: SWAP_VOLUME_cacao_USD + description: "The swap amount of cacao in USD" + - name: TO_ASSET_SWAP_VOLUME + description: "The swap volume from cacao to Asset" + - name: TO_cacao_SWAP_VOLUME + description: "The swap volume from Asset to cacao" + - name: TOTAL_SWAP_FEES_cacao + description: "Total swap fees in cacao" + - name: TOTAL_SWAP_FEES_USD + description: "Total swap fees in USD" + - name: TOTAL_ASSET_SWAP_FEES + description: "Total asset swap fees in asset" + - name: TOTAL_ASSET_cacao_FEES + description: "Total asset swap fees in cacao" + - name: UNIQUE_MEMBER_COUNT + description: "All memberships with a cacao address. Take the member from cacao and asset and then subtract the balance = 0 then get the results" + - name: UNIQUE_SWAPPER_COUNT + description: "The unique swap address for this pool" + - name: LIQUIDITY_UNITS + description: "The amount of units for the liquidity in the pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_DAILY_POOL_STATS_ID diff --git a/models/gold/defi/defi__fact_daily_tvl.sql b/models/gold/defi/defi__fact_daily_tvl.sql new file mode 100644 index 0000000..70b239e --- /dev/null +++ b/models/gold/defi/defi__fact_daily_tvl.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_daily_tvl_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + total_value_pooled, + total_value_pooled_usd, + total_value_bonded, + total_value_bonded_usd, + total_value_locked, + total_value_locked_usd + FROM + {{ ref('silver__daily_tvl') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_daily_tvl_id, + DAY, + total_value_pooled, + total_value_pooled_usd, + total_value_bonded, + total_value_bonded_usd, + total_value_locked, + total_value_locked_usd, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_daily_tvl.yml b/models/gold/defi/defi__fact_daily_tvl.yml new file mode 100644 index 0000000..3baa9c7 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_tvl.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: defi__fact_daily_tvl + description: "Fact table that shows the total valued locked and the total value bonded/pooled" + columns: + - name: FACT_DAILY_TVL_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: TOTAL_VALUE_POOLED + description: "The total amount of cacao provided by the liquidity provides and pooled in the pool" + - name: TOTAL_VALUE_POOLED_USD + description: "The total USD amount of cacao provided by the liquidity provides and pooled in the pool" + - name: TOTAL_VALUE_BONDED + description: "The total amount of cacao provided by the node operators and bonded in the pool" + - name: TOTAL_VALUE_BONDED_USD + description: "The total USD amount of cacao provided by the node operators and bonded in the pool" + - name: TOTAL_VALUE_LOCKED + description: "The total cacao value locked in the pool" + - name: TOTAL_VALUE_LOCKED_USD + description: "The total USD value locked in the pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_DAILY_TVL_ID diff --git a/models/gold/defi/defi__fact_failed_deposit_messages.sql b/models/gold/defi/defi__fact_failed_deposit_messages.sql new file mode 100644 index 0000000..2415962 --- /dev/null +++ b/models/gold/defi/defi__fact_failed_deposit_messages.sql @@ -0,0 +1,71 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_failed_deposit_messages_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + amount_e8, + asset, + from_address, + memo, + code, + reason, + tx_id, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__failed_deposit_messages') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp'] + ) }} AS fact_failed_deposit_messages_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + amount_e8, + asset, + from_address, + memo, + code, + reason, + tx_id, + event_id, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR event_id IN ( + SELECT + event_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_failed_deposit_messages.yml b/models/gold/defi/defi__fact_failed_deposit_messages.yml new file mode 100644 index 0000000..157f2e3 --- /dev/null +++ b/models/gold/defi/defi__fact_failed_deposit_messages.yml @@ -0,0 +1,51 @@ +version: 2 +models: + - name: defi__fact_failed_deposit_messages + description: "Fact table containing failed deposit messages" + columns: + - name: FACT_FAILED_DEPOSIT_MESSAGES_ID + description: "{{ doc('sk') }}" + tests: + - not_null + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "{{ doc('sk') }}" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: AMOUNT_E8 + description: "The amount of the failed deposit in E8 notation" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: MEMO + description: "{{ doc('memo') }}" + - name: CODE + description: "Error code associated with the failed deposit" + - name: REASON + description: "Detailed reason for the failed deposit" + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: EVENT_ID + description: "Unique identifier for the failed deposit event" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_FAILED_DEPOSIT_MESSAGES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_fee_events.sql b/models/gold/defi/defi__fact_fee_events.sql new file mode 100644 index 0000000..2953f92 --- /dev/null +++ b/models/gold/defi/defi__fact_fee_events.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_fee_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + asset, + pool_deduct, + asset_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__fee_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.asset_e8','a.pool_deduct','a.block_timestamp','a.tx_id '] + ) }} AS fact_fee_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + asset, + pool_deduct, + asset_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_fee_events.yml b/models/gold/defi/defi__fact_fee_events.yml new file mode 100644 index 0000000..493fefd --- /dev/null +++ b/models/gold/defi/defi__fact_fee_events.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_fee_events + description: "Fact table containing all related fees that happen in the Thorchain network" + columns: + - name: FACT_FEE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: POOL_DEDUCT + description: "The amount deducted from the pool related to the fee" + tests: + - not_null + - name: ASSET_E8 + description: "The asset amount for this fee, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_FEE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_gas_events.sql b/models/gold/defi/defi__fact_gas_events.sql new file mode 100644 index 0000000..900a5f9 --- /dev/null +++ b/models/gold/defi/defi__fact_gas_events.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_gas_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + asset, + asset_e8, + cacao_e8, + tx_count, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__gas_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.block_timestamp'] + ) }} AS fact_gas_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + asset, + asset_e8, + cacao_e8, + tx_count, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR asset IN ( + SELECT + asset + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_gas_events.yml b/models/gold/defi/defi__fact_gas_events.yml new file mode 100644 index 0000000..2cbdd4a --- /dev/null +++ b/models/gold/defi/defi__fact_gas_events.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_gas_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_GAS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: ASSET_E8 + description: "The asset amount for this fee, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: cacao_E8 + description: "The total gas amount in cacao" + tests: + - not_null + - name: TX_COUNT + description: "The total count of transactions within this block id" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_GAS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_inactive_vault_events.sql b/models/gold/defi/defi__fact_inactive_vault_events.sql new file mode 100644 index 0000000..5129fb7 --- /dev/null +++ b/models/gold/defi/defi__fact_inactive_vault_events.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_inactive_vault_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + add_asgard_address, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__inactive_vault_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.add_asgard_address','a.block_timestamp'] + ) }} AS fact_inactive_vault_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + add_asgard_address, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_inactive_vault_events.yml b/models/gold/defi/defi__fact_inactive_vault_events.yml new file mode 100644 index 0000000..47763bd --- /dev/null +++ b/models/gold/defi/defi__fact_inactive_vault_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: defi__fact_inactive_vault_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_INACTIVE_VAULT_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ADD_ASGARD_ADDRESS + description: "The asgard address in the vault which are inactive" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_INACTIVE_VAULT_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_liquidity_actions.sql b/models/gold/defi/defi__fact_liquidity_actions.sql new file mode 100644 index 0000000..eb8cb98 --- /dev/null +++ b/models/gold/defi/defi__fact_liquidity_actions.sql @@ -0,0 +1,82 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_liquidity_actions_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + tx_id, + lp_action, + pool_name, + from_address, + to_address, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + il_protection, + il_protection_usd, + unstake_asymmetry, + unstake_basis_points, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__liquidity_actions') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_liquidity_actions_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + lp_action, + pool_name, + from_address, + to_address, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + il_protection, + il_protection_usd, + unstake_asymmetry, + unstake_basis_points, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_liquidity_actions.yml b/models/gold/defi/defi__fact_liquidity_actions.yml new file mode 100644 index 0000000..669ba88 --- /dev/null +++ b/models/gold/defi/defi__fact_liquidity_actions.yml @@ -0,0 +1,79 @@ +version: 2 +models: + - name: defi__fact_liquidity_actions + description: "Fact table containing the actions the liquidity providers do in the THORChain, with the amount in cacao/Asset" + columns: + - name: FACT_LIQUIDITY_ACTIONS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: LP_ACTION + description: "The direction of the liquidity providers, add or remove the liquidity" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('address') }}" + - name: TO_ADDRESS + description: "{{ doc('address') }}" + + - name: cacao_AMOUNT + description: "How many cacao liquidity added/removed" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "How many cacao liquidity added/removed in USD" + tests: + - not_null + - name: ASSET_AMOUNT + description: "How many Asset liquidity added/removed" + tests: + - not_null + - name: ASSET_AMOUNT_USD + description: "How many Asset liquidity added/removed in USD" + - name: STAKE_UNITS + description: "Stake units are a way of representing how much liquidity an address has in the pool. THORChain converts the raw amounts you are depositing / withdrawing into stake_units to represent what % of the pool you own a claim to." + tests: + - not_null + - name: asset_tx_id + description: "The transaction id for adding/removing the asset" + - name: asset_address + description: "The asset address of the liquidity provider" + - name: asset_blockchain + description: "{{ doc('blockchain') }}" + - name: il_protection + description: "The total impermanent loss protection paid for this pool on this day" + - name: il_protection_usd + description: "The total impermanent loss protection paid in USD for this pool on this day" + - name: unstake_asymmetry + description: "Only exists in unstake, or removing the liquidity" + - name: unstake_basis_points + description: "The basis points for unstaking, or removing the liquidity" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_LIQUIDITY_ACTIONS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_outbound_events.sql b/models/gold/defi/defi__fact_outbound_events.sql new file mode 100644 index 0000000..776b818 --- /dev/null +++ b/models/gold/defi/defi__fact_outbound_events.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_outbound_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + in_tx, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__outbound_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id ','a.blockchain','a.from_address','a.to_address','a.asset','a.memo ','a.in_tx','a.block_timestamp'] + ) }} AS fact_outbound_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + in_tx, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_outbound_events.yml b/models/gold/defi/defi__fact_outbound_events.yml new file mode 100644 index 0000000..d58d781 --- /dev/null +++ b/models/gold/defi/defi__fact_outbound_events.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_outbound_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_OUTBOUND_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + - name: ASSET + description: "Same as pool name, which pool this outbound event happens" + - name: ASSET_E8 + description: "The asset amount for this outbound event, using the price table we can calculate the cacao amount by asset amount" + - name: MEMO + description: "{{ doc('memo') }}" + - name: IN_TX + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_OUTBOUND_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_pending_liquidity_events.sql b/models/gold/defi/defi__fact_pending_liquidity_events.sql new file mode 100644 index 0000000..8833da7 --- /dev/null +++ b/models/gold/defi/defi__fact_pending_liquidity_events.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pending_liquidity_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + cacao_tx_id, + cacao_address, + cacao_e8, + pending_type, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pending_liquidity_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.pool_name ','a.asset_tx_id','a.asset_blockchain','a.asset_address','a.cacao_tx_id','a.cacao_address ','a.pending_type','a.block_timestamp'] + ) }} AS fact_pending_liquidity_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + cacao_tx_id, + cacao_address, + cacao_e8, + pending_type, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pending_liquidity_events.yml b/models/gold/defi/defi__fact_pending_liquidity_events.yml new file mode 100644 index 0000000..a00d8cb --- /dev/null +++ b/models/gold/defi/defi__fact_pending_liquidity_events.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: defi__fact_pending_liquidity_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_PENDING_LIQUIDITY_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: ASSET_TX_ID + description: "The unique transaction id for the cacao event, the liquidity type can be add/remove the cacao/asset, if the event related to cacao" + - name: ASSET_BLOCKCHAIN + description: "The blockchain of the asset" + - name: ASSET_ADDRESS + description: "The address of asset" + - name: ASSET_E8 + description: "The amount of asset for the liquidity events" + - name: cacao_TX_ID + description: "The unique transaction id for the cacao event, the liquidity type can be add/remove the cacao/asset, if the event related to cacao" + - name: cacao_ADDRESS + description: "The address of cacao" + - name: cacao_E8 + description: "The amount of cacao for the liquidity events" + - name: PENDING_TYPE + description: "The type of liquidity, can be 'add' or 'withdraw'" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_PENDING_LIQUIDITY_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_pool_balance_change_events.sql b/models/gold/defi/defi__fact_pool_balance_change_events.sql new file mode 100644 index 0000000..5b774be --- /dev/null +++ b/models/gold/defi/defi__fact_pool_balance_change_events.sql @@ -0,0 +1,68 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_balance_change_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + asset, + cacao_amount, + cacao_add, + asset_amount, + asset_add, + reason, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_balance_change_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.block_timestamp'] + ) }} AS fact_pool_balance_change_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + asset, + cacao_amount, + cacao_add, + asset_amount, + asset_add, + reason, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR asset IN ( + SELECT + asset + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pool_balance_change_events.yml b/models/gold/defi/defi__fact_pool_balance_change_events.yml new file mode 100644 index 0000000..f1cc4bd --- /dev/null +++ b/models/gold/defi/defi__fact_pool_balance_change_events.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: defi__fact_pool_balance_change_events + description: "Fact table that shows the change of the pool balance" + columns: + - name: FACT_POOL_BALANCE_CHANGE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ASSET + description: "Same as pool name, which pool this bond events happens" + - name: cacao_AMOUNT + description: "The amount of cacao for the pool balance change" + - name: cacao_ADD + description: "False or True, if True, then the event is to add cacao not asset" + - name: ASSET_AMOUNT + description: "The amount of asset for the pool balance change" + - name: ASSET_ADD + description: "False or True, if True, then the event is to add asset not cacao" + - name: REASON + description: "The reason for the pool balance change" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BALANCE_CHANGE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_pool_block_balances.sql b/models/gold/defi/defi__fact_pool_block_balances.sql new file mode 100644 index 0000000..850ea2a --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_balances.sql @@ -0,0 +1,62 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_block_balances_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + pool_name, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + synth_amount, + synth_amount_usd, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_block_balances') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_pool_block_balances_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + synth_amount, + synth_amount_usd, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pool_block_balances.yml b/models/gold/defi/defi__fact_pool_block_balances.yml new file mode 100644 index 0000000..e507b56 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_balances.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: defi__fact_pool_block_balances + description: "Fact table containing the actions the liquidity providers do in the THORChain, with the amount in cacao/Asset" + columns: + - name: FACT_POOL_BLOCK_BALANCES_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: cacao_AMOUNT + description: "The cacao amount balance for this pool name" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "The cacao amount balance in USD for this pool name" + tests: + - not_null + - name: ASSET_AMOUNT + description: "The asset amount balance for this pool name" + tests: + - not_null + - name: ASSET_AMOUNT_USD + description: "The asset amount balance in USD for this pool name" + - name: SYNTH_AMOUNT + description: "The synth amount balance for this pool name" + tests: + - not_null + - name: SYNTH_AMOUNT_USD + description: "The synth amount balance in USD for this pool name" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BLOCK_BALANCES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_pool_block_fees.sql b/models/gold/defi/defi__fact_pool_block_fees.sql new file mode 100644 index 0000000..b4ccf74 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_fees.sql @@ -0,0 +1,53 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_block_fees_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + pool_name, + rewards, + total_liquidity_fees_cacao, + asset_liquidity_fees, + cacao_liquidity_fees, + earnings, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_block_fees') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_pool_block_fees_id, + DAY, + pool_name, + rewards, + total_liquidity_fees_cacao, + asset_liquidity_fees, + cacao_liquidity_fees, + earnings, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_pool_block_fees.yml b/models/gold/defi/defi__fact_pool_block_fees.yml new file mode 100644 index 0000000..a18806f --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_fees.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: defi__fact_pool_block_fees + description: "Fact table that shows the fee paid by each pool at different timestamp" + columns: + - name: FACT_POOL_BLOCK_FEES_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: REWARDS + description: "The total rewards" + - name: TOTAL_LIQUIDITY_FEES_cacao + description: "The total liquidity fees paid in cacao" + - name: ASSET_LIQUIDITY_FEES + description: "The liquidity fees paid in Asset" + - name: cacao_LIQUIDITY_FEES + description: "The liquidity fees paid in cacao" + - name: EARNINGS + description: "The total earnings for this pool at this time" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BLOCK_FEES_ID diff --git a/models/gold/defi/defi__fact_pool_block_statistics.sql b/models/gold/defi/defi__fact_pool_block_statistics.sql new file mode 100644 index 0000000..80f4636 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_statistics.sql @@ -0,0 +1,111 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_block_statistics_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + add_asset_liquidity_volume, + add_liquidity_count, + add_liquidity_volume, + add_cacao_liquidity_volume, + asset, + asset_depth, + asset_price, + asset_price_usd, + average_slip, + impermanent_loss_protection_paid, + cacao_depth, + status, + swap_count, + swap_volume, + to_asset_average_slip, + to_asset_count, + to_asset_fees, + to_asset_volume, + to_cacao_average_slip, + to_cacao_count, + to_cacao_fees, + to_cacao_volume, + totalfees, + unique_member_count, + unique_swapper_count, + units, + withdraw_asset_volume, + withdraw_count, + withdraw_cacao_volume, + withdraw_volume, + total_stake, + depth_product, + synth_units, + pool_units, + liquidity_unit_value_index, + prev_liquidity_unit_value_index, + _UNIQUE_KEY + FROM + {{ ref('silver__pool_block_statistics') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_pool_block_statistics_id, + DAY, + add_asset_liquidity_volume, + add_liquidity_count, + add_liquidity_volume, + add_cacao_liquidity_volume, + asset, + asset_depth, + asset_price, + asset_price_usd, + average_slip, + impermanent_loss_protection_paid, + cacao_depth, + status, + swap_count, + swap_volume, + to_asset_average_slip, + to_asset_count, + to_asset_fees, + to_asset_volume, + to_cacao_average_slip, + to_cacao_count, + to_cacao_fees, + to_cacao_volume, + totalfees, + unique_member_count, + unique_swapper_count, + units, + withdraw_asset_volume, + withdraw_count, + withdraw_cacao_volume, + withdraw_volume, + total_stake, + depth_product, + synth_units, + pool_units, + liquidity_unit_value_index, + prev_liquidity_unit_value_index, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_pool_block_statistics.yml b/models/gold/defi/defi__fact_pool_block_statistics.yml new file mode 100644 index 0000000..34aa858 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_statistics.yml @@ -0,0 +1,90 @@ +version: 2 +models: + - name: defi__fact_pool_block_statistics + description: "Fact table containing the actions the liquidity providers do in the THORChain, with the amount in cacao/Asset" + columns: + - name: FACT_POOL_BLOCK_STATISTICS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The timestamp in day for the recorded of the block fee" + - name: ADD_ASSET_LIQUIDITY_VOLUME + description: "How much asset liquidity has been added to the pool at this time for this pool" + - name: ADD_LIQUIDITY_COUNT + description: "The count of add liquidity transactions" + - name: ADD_LIQUIDITY_VOLUME + description: "The asset volume of liquidity added to the pool" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_DEPTH + description: "The current pool depth, which is the total cacao pooled in the asset" + - name: ASSET_PRICE + description: "The asset price in cacao for this day" + - name: ASSET_PRICE_USD + description: "The asset price in USD" + - name: AVERAGE_SLIP + description: "The average slip point for this block within the day" + - name: IMPERMANENT_LOSS_PROTECTION_PAID + description: "The total impermanent loss protection paid for this pool on this day" + - name: cacao_DEPTH + description: "The pool depth" + - name: STATUS + description: "The pool status, which is active or not" + - name: SWAP_COUNT + description: "Total swap transactions count" + - name: SWAP_VOLUME + description: "Total swap volume" + - name: TO_ASSET_AVERAGE_SLIP + description: "If the transaction is from cacao to Asset, the average slip point" + - name: TO_ASSET_COUNT + description: "How many swaps happen from cacao to Asset" + - name: TO_ASSET_FEES + description: "The total swap fees paid to transfer from cacao to Asset" + - name: TO_ASSET_VOLUME + description: "The total volume transferred from cacao to Asset" + - name: TO_cacao_AVERAGE_SLIP + description: "If the transaction is from Asset to cacao, the average slip point" + - name: TO_cacao_COUNT + description: "How many swaps happen from Asset to cacao" + - name: TO_cacao_FEES + description: "The total swap fees paid to transfer from Asset to cacao" + - name: TO_cacao_VOLUME + description: "The total volume transferred from Asset to cacao" + - name: TOTALFEES + description: "The total fees paid for the swaps" + - name: UNIQUE_MEMBER_COUNT + description: "All memberships with a cacao address. Take the member from cacao and asset and then subtract the balance = 0 then get the results" + - name: UNIQUE_SWAPPER_COUNT + description: "The unique swap addresses for this pool " + - name: UNITS + description: "The swap units" + - name: WITHDRAW_ASSET_VOLUME + description: "How many assets get withdrawn from the liquidity pools" + - name: WITHDRAW_COUNT + description: "How many times the withdraw events happens" + - name: WITHDRAW_cacao_VOLUME + description: "How many cacao volume get withdrawn from the pool" + - name: WITHDRAW_VOLUME + description: "How many asset volume get withdrawn from the pool" + - name: TOTAL_STAKE + description: "" + - name: DEPTH_PRODUCT + description: "" + - name: SYNTH_UNITS + description: "" + - name: POOL_UNITS + description: "" + - name: LIQUIDITY_UNIT_VALUE_INDEX + description: "" + - name: PREV_LIQUIDITY_UNIT_VALUE_INDEX + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BLOCK_STATISTICS_ID + \ No newline at end of file diff --git a/models/gold/defi/defi__fact_pool_events.sql b/models/gold/defi/defi__fact_pool_events.sql new file mode 100644 index 0000000..e5f87df --- /dev/null +++ b/models/gold/defi/defi__fact_pool_events.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + asset, + status, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.status','a.block_timestamp'] + ) }} AS fact_pool_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + asset, + status, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR asset IN ( + SELECT + asset + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pool_events.yml b/models/gold/defi/defi__fact_pool_events.yml new file mode 100644 index 0000000..48b9d44 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: defi__fact_pool_events + description: "Fact table that shows that the status of the pool at each block id" + columns: + - name: FACT_POOL_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ASSET + description: "The asset/pool name" + - name: STATUS + description: "The current status for this pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_refund_events.sql b/models/gold/defi/defi__fact_refund_events.sql new file mode 100644 index 0000000..563ea25 --- /dev/null +++ b/models/gold/defi/defi__fact_refund_events.sql @@ -0,0 +1,70 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_refund_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__refund_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.from_address' ,'a.to_address','a. asset', 'a.asset_2nd', 'a.memo', 'a.code', 'a.reason', 'a.block_timestamp'] + ) }} AS fact_refund_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_refund_events.yml b/models/gold/defi/defi__fact_refund_events.yml new file mode 100644 index 0000000..1abd740 --- /dev/null +++ b/models/gold/defi/defi__fact_refund_events.yml @@ -0,0 +1,53 @@ +version: 2 +models: + - name: defi__fact_refund_events + description: "Fact table that shows that the refund events" + columns: + - name: FACT_REFUND_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: "" + - name: ASSET_2ND + description: "" + - name: ASSET_2ND_E8 + description: "" + - name: MEMO + description: "{{ doc('memo') }}" + - name: CODE + description: "" + - name: REASON + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_REFUND_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_reserve_events.sql b/models/gold/defi/defi__fact_reserve_events.sql new file mode 100644 index 0000000..67a2e10 --- /dev/null +++ b/models/gold/defi/defi__fact_reserve_events.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_reserve_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + address, + e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__reserve_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.from_address','a.to_address','a.asset','a.memo','a.address','a.block_timestamp'] + ) }} AS fact_reserve_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + address, + e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_reserve_events.yml b/models/gold/defi/defi__fact_reserve_events.yml new file mode 100644 index 0000000..fef354a --- /dev/null +++ b/models/gold/defi/defi__fact_reserve_events.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: defi__fact_reserve_events + description: "Fact table that shows the amount of cacao reserved into the network" + columns: + - name: FACT_RESERVE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: "The asset amount" + - name: MEMO + description: "{{ doc('memo') }}" + - name: ADDRESS + description: "The address reserve the amount to the pool" + - name: E8 + description: "The cacao amount" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_RESERVE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_rewards_event_entries.sql b/models/gold/defi/defi__fact_rewards_event_entries.sql new file mode 100644 index 0000000..af143d7 --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_event_entries.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_rewards_event_entries_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + cacao_e8, + saver_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__rewards_event_entries') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.pool_name','a.block_timestamp'] + ) }} AS fact_rewards_event_entries_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + cacao_e8, + saver_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_rewards_event_entries.yml b/models/gold/defi/defi__fact_rewards_event_entries.yml new file mode 100644 index 0000000..1d3a9bf --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_event_entries.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: defi__fact_rewards_event_entries + description: "Fact table that hows the entries for the rewards" + columns: + - name: FACT_REWARDS_EVENT_ENTRIES_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: cacao_E8 + description: "The cacao amount of the rewards for this pool at this block" + - name: SAVER_E8 + description: "The savers amount for this pool at this block" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_REWARDS_EVENT_ENTRIES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_rewards_events.sql b/models/gold/defi/defi__fact_rewards_events.sql new file mode 100644 index 0000000..a64bd09 --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_events.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_rewards_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + bond_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__rewards_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp'] + ) }} AS fact_rewards_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + bond_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_rewards_events.yml b/models/gold/defi/defi__fact_rewards_events.yml new file mode 100644 index 0000000..f020a02 --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_events.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: defi__fact_rewards_events + description: "Fact table that shows the entries for the rewards" + columns: + - name: FACT_REWARDS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: BOND_E8 + description: "The cacao amount of the bond for this pool at this block" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_REWARDS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_send_messages.sql b/models/gold/defi/defi__fact_send_messages.sql new file mode 100644 index 0000000..821ac7c --- /dev/null +++ b/models/gold/defi/defi__fact_send_messages.sql @@ -0,0 +1,69 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'event_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + amount_e8, + asset, + from_address, + to_address, + memo, + tx_id, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__send_messages') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.amount_e8','a.asset','a.from_address','a.to_address','a.memo','a.tx_id','a.event_id','a.block_timestamp'] + ) }} AS fact_send_messages_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + amount_e8, + asset, + from_address, + to_address, + memo, + tx_id, + event_id, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR event_id IN ( + SELECT + event_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_send_messages.yml b/models/gold/defi/defi__fact_send_messages.yml new file mode 100644 index 0000000..36d44ad --- /dev/null +++ b/models/gold/defi/defi__fact_send_messages.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: defi__fact_send_messages + description: "Fact table that shows send messages" + columns: + - name: FACT_SEND_MESSAGES_ID + description: "{{ doc('sk') }}" + tests: + - not_null + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: AMOUNT_E8 + - name: ASSET + - name: FROM_ADDRESS + - name: TO_ADDRESS + - name: MEMO + - name: TX_ID + - name: EVENT_ID + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SEND_MESSAGES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_slash_liquidity_events.sql b/models/gold/defi/defi__fact_slash_liquidity_events.sql new file mode 100644 index 0000000..7c21eca --- /dev/null +++ b/models/gold/defi/defi__fact_slash_liquidity_events.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_slash_liquidity_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__slash_liquidity_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.bond_address', 'a.lp_address', 'a.asset'] + ) }} AS fact_slash_liquidity_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_slash_liquidity_events.yml b/models/gold/defi/defi__fact_slash_liquidity_events.yml new file mode 100644 index 0000000..7ea81c6 --- /dev/null +++ b/models/gold/defi/defi__fact_slash_liquidity_events.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: defi__fact_slash_liquidity_events + description: "Fact table that shows the slash liquidity event entries" + columns: + - name: FACT_SLASH_LIQUIDITY_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: bond_address + description: "The address of the bond" + - name: lp_address + description: "The address of the liquidity pool" + - name: asset + description: "The asset that was slashed" + - name: lp_units + description: "The number of liquidity pool units that were slashed" + - name: asset_e8_loss + description: "The amount of asset lost in E8" + - name: cacao_e10_loss + description: "The amount of cacao lost in E10" + + + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SLASH_LIQUIDITY_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_stake_events.sql b/models/gold/defi/defi__fact_stake_events.sql new file mode 100644 index 0000000..cb26324 --- /dev/null +++ b/models/gold/defi/defi__fact_stake_events.sql @@ -0,0 +1,68 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, STAKING' }} }, + unique_key = 'fact_stake_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + stake_units, + cacao_tx_id, + cacao_address, + cacao_e8, + _ASSET_IN_cacao_E8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__stake_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.cacao_address', 'a.asset_address'] + ) }} AS fact_stake_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + stake_units, + cacao_tx_id, + cacao_address, + cacao_e8, + _ASSET_IN_cacao_E8, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_stake_events.yml b/models/gold/defi/defi__fact_stake_events.yml new file mode 100644 index 0000000..1770945 --- /dev/null +++ b/models/gold/defi/defi__fact_stake_events.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: defi__fact_stake_events + description: "Fact table containing stake events" + columns: + - name: FACT_STAKE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: ASSET_TX_ID + description: "{{ doc('tx_id') }}" + - name: ASSET_BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: ASSET_ADDRESS + description: "{{ doc('address') }}" + - name: ASSET_E8 + description: "The asset amount for this event" + tests: + - not_null + - name: STAKE_UNITS + description: "" + tests: + - not_null + - name: cacao_TX_ID + description: "{{ doc('tx_id') }}" + - name: cacao_ADDRESS + description: "{{ doc('address') }}" + - name: cacao_E8 + description: "The asset amount for this event" + tests: + - not_null + - name: _ASSET_IN_cacao_E8 + description: "" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_STAKE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_streamling_swap_details_events.sql b/models/gold/defi/defi__fact_streamling_swap_details_events.sql new file mode 100644 index 0000000..0b55b42 --- /dev/null +++ b/models/gold/defi/defi__fact_streamling_swap_details_events.sql @@ -0,0 +1,77 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_streamling_swap_details_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swap_reasons AS failed_swaps_reasons, + event_id, + block_timestamp, + failed_swap_reasons, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__streamling_swap_details_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id'] + ) }} AS fact_streamling_swap_details_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swaps_reasons, + event_id, + failed_swap_reasons, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_streamling_swap_details_events.yml b/models/gold/defi/defi__fact_streamling_swap_details_events.yml new file mode 100644 index 0000000..5c5ded1 --- /dev/null +++ b/models/gold/defi/defi__fact_streamling_swap_details_events.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: defi__fact_streamling_swap_details_events + description: "Fact table that shows the streaming swap event transactions" + columns: + - name: FACT_STREAMLING_SWAP_DETAILS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: INTERVAL + description: "the time interval of the swap" + - name: quantity + description: "the quantity of the swap" + - name: COUNT + description: "the count of the swap" + - name: last_height + description: "the last height of the swap" + - name: deposit_asset + description: "the deposit asset of the swap" + - name: deposit_e8 + description: "the deposit amount of the swap" + - name: in_asset + description: "the in asset of the swap" + - name: in_e8 + description: "the in amount of the swap" + - name: out_asset + description: "the out asset of the swap" + - name: out_e8 + description: "the out amount of the swap" + - name: failed_swaps + description: "the count of failed swaps" + - name: failed_swaps_reasons + description: "DEPRECATING SOON! the reason of failed swaps" + - name: event_id + description: "" + - name: failed_swap_reasons + description: "the reason of failed swaps" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_STREAMLING_SWAP_DETAILS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_swaps.sql b/models/gold/defi/defi__fact_swaps.sql new file mode 100644 index 0000000..8ea6275 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps.sql @@ -0,0 +1,103 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_swaps_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + block_id, + tx_id, + blockchain, + pool_name, + from_address, + native_to_address, + to_pool_address, + affiliate_address, + affiliate_fee_basis_points, + affiliate_addresses_array, + affiliate_fee_basis_points_array, + from_asset, + to_asset, + from_amount, + to_amount, + min_to_amount, + from_amount_usd, + to_amount_usd, + cacao_usd, + asset_usd, + to_amount_min_usd, + swap_slip_bp, + liq_fee_cacao, + liq_fee_cacao_usd, + liq_fee_asset, + liq_fee_asset_usd, + streaming_count, + streaming_quantity, + _unique_key, + _inserted_timestamp + FROM + {{ ref('silver__swaps') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_swaps_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + pool_name, + from_address, + native_to_address, + to_pool_address, + affiliate_address, + affiliate_fee_basis_points, + affiliate_addresses_array, + affiliate_fee_basis_points_array, + from_asset, + to_asset, + from_amount, + to_amount, + min_to_amount, + from_amount_usd, + to_amount_usd, + cacao_usd, + asset_usd, + to_amount_min_usd, + swap_slip_bp, + liq_fee_cacao, + liq_fee_cacao_usd, + liq_fee_asset, + liq_fee_asset_usd, + streaming_count, + streaming_quantity, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_swaps.yml b/models/gold/defi/defi__fact_swaps.yml new file mode 100644 index 0000000..1de9854 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps.yml @@ -0,0 +1,86 @@ +version: 2 +models: + - name: defi__fact_swaps + description: "Fact table that shows the swap activity in Thorchain" + columns: + - name: FACT_SWAPS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: FROM_ADDRESS + description: "The user address that initiates the swap" + - name: NATIVE_TO_ADDRESS + description: "The user address that is receiving the result of the swap" + - name: TO_POOL_ADDRESS + description: "The pool address that processes the swap" + - name: AFFILIATE_ADDRESS + description: "The affiliate address that is receiving the affiliate fee (redundant with array)" + - name: AFFILIATE_FEE_BASIS_POINTS + description: "The affiliate fee basis points that is received by the affiliate address (redundant with array)" + - name: AFFILIATE_ADDRESSES_ARRAY + description: "The affiliate addresses that are receiving the affiliate fee" + - name: AFFILIATE_FEE_BASIS_POINTS_ARRAY + description: "The affiliate fee basis points that are received by the affiliate addresses" + - name: FROM_ASSET + description: "Initial asset to swap" + - name: TO_ASSET + description: "The asset swap to" + - name: FROM_AMOUNT + description: "Amount of the asset to swap from" + - name: TO_AMOUNT + description: "Amount of the asset to swap for" + - name: MIN_TO_AMOUNT + description: "Minimal amount to swap for" + - name: FROM_AMOUNT_USD + description: "Amount in USD of the asset to swap from" + - name: TO_AMOUNT_USD + description: "Amount in USD of the asset to swap for" + - name: cacao_USD + description: "Minimal amount in USD to swap for" + - name: ASSET_USD + description: "Asset amount in USD" + - name: TO_AMOUNT_MIN_USD + description: "Min asset amount in USD" + - name: SWAP_SLIP_BP + description: "The slippage during the swap process" + - name: LIQ_FEE_cacao + description: "The amount of cacao liquidity fee paid in cacao for the swaps" + - name: LIQ_FEE_cacao_USD + description: "The amount of cacao in USD liquidity fee paid in cacao for the swaps" + - name: LIQ_FEE_ASSET + description: "The amount of Asset liquidity fee paid in cacao for the swaps" + - name: LIQ_FEE_ASSET_USD + description: "The amount of Asset in USD liquidity fee paid in cacao for the swaps" + - name: STREAMING_COUNT + description: "The count of the streaming events" + - name: STREAMING_QUANTITY + description: "The quantity of the streaming events" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SWAPS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_swaps_events.sql b/models/gold/defi/defi__fact_swaps_events.sql new file mode 100644 index 0000000..36c4a00 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps_events.sql @@ -0,0 +1,81 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_swap_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + event_id, + block_timestamp, + streaming_count, + streaming_quantity, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__swap_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.to_address','a.from_address','a.from_asset','a.from_e8','a.to_asset','a.to_e8','a.memo','a.pool_name','a._direction'] + ) }} AS fact_swap_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + event_id, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_swaps_events.yml b/models/gold/defi/defi__fact_swaps_events.yml new file mode 100644 index 0000000..59dcea6 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps_events.yml @@ -0,0 +1,96 @@ +version: 2 +models: + - name: defi__fact_swaps_events + description: "Fact table that shows the swap event entries table shows the entries for the swaps" + columns: + - name: FACT_SWAP_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "The user address that initiates the swap" + tests: + - not_null + - name: TO_ADDRESS + description: "The address we are swapping to" + tests: + - not_null + - name: FROM_ASSET + description: "The asset we are swapping from" + tests: + - not_null + - name: FROM_E8 + description: "The amount we are swapping from" + tests: + - not_null + - name: TO_ASSET + description: "The asset we are swapping to" + tests: + - not_null + - name: TO_E8 + description: "The asset amount we are swapping for (divide by 10^8 to get the decimal amount)" + tests: + - not_null + - name: MEMO + description: "The transaction memo" + tests: + - not_null + - name: POOL_NAME + description: "The name of the pool" + tests: + - not_null + - name: TO_E8_MIN + description: "The minimum amount the swapper will receive" + tests: + - not_null + - name: SWAP_SLIP_BP + description: "The slippage" + tests: + - not_null + - name: LIQ_FEE_E8 + description: "The fee (divide by 10^8 to get the decimal amount)" + tests: + - not_null + - name: LIQ_FEE_IN_cacao_E8 + description: "The fee in cacao (divide by 10^8 to get the decimal amount)" + tests: + - not_null + - name: _DIRECTION + description: "" + tests: + - not_null + - name: STREAMING_COUNT + description: "The count of the streaming events" + - name: STREAMING_QUANTITY + description: "The quantity of the streaming events" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SWAP_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_total_block_rewards.sql b/models/gold/defi/defi__fact_total_block_rewards.sql new file mode 100644 index 0000000..abc2246 --- /dev/null +++ b/models/gold/defi/defi__fact_total_block_rewards.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_total_block_rewards_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + reward_entity, + cacao_amount, + cacao_amount_usd, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__total_block_rewards') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_total_block_rewards_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + reward_entity, + cacao_amount, + cacao_amount_usd, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_total_block_rewards.yml b/models/gold/defi/defi__fact_total_block_rewards.yml new file mode 100644 index 0000000..a90be7b --- /dev/null +++ b/models/gold/defi/defi__fact_total_block_rewards.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: defi__fact_total_block_rewards + description: "Fact table containing stake events" + columns: + - name: FACT_TOTAL_BLOCK_REWARDS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: REWARD_ENTITY + description: "The asset or named as pool name" + tests: + - not_null + - name: cacao_AMOUNT + description: "The rewards measured in cacao amount" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "The rewards measured in cacao amount in the USD" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TOTAL_BLOCK_REWARDS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_total_value_locked.sql b/models/gold/defi/defi__fact_total_value_locked.sql new file mode 100644 index 0000000..ed4058a --- /dev/null +++ b/models/gold/defi/defi__fact_total_value_locked.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_total_value_locked_id', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_strategy = 'merge' +) }} + +WITH base AS ( + + SELECT + DAY, + total_value_pooled, + total_value_bonded, + total_value_locked, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__total_value_locked') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_total_value_locked_id, + DAY, + total_value_pooled, + total_value_bonded, + total_value_locked, + _INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_total_value_locked.yml b/models/gold/defi/defi__fact_total_value_locked.yml new file mode 100644 index 0000000..f798bd8 --- /dev/null +++ b/models/gold/defi/defi__fact_total_value_locked.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: defi__fact_total_value_locked + description: "Fact table that shows set mimir events" + columns: + - name: FACT_TOTAL_VALUE_LOCKED_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The timestamp in day for the recorded of the block fee" + tests: + - not_null + - unique + - name: TOTAL_VALUE_POOLED + description: "The total amount of cacao provided by the liquidity provides and pooled in the pool" + - name: TOTAL_VALUE_BONDED + description: "The total amount of cacao provided by the node operators and bonded in the pool" + - name: TOTAL_VALUE_LOCKED + description: "The total cacao value locked in the pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TOTAL_VALUE_LOCKED_ID diff --git a/models/gold/defi/defi__fact_update_node_account_status_events.sql b/models/gold/defi/defi__fact_update_node_account_status_events.sql new file mode 100644 index 0000000..667f516 --- /dev/null +++ b/models/gold/defi/defi__fact_update_node_account_status_events.sql @@ -0,0 +1,53 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_update_node_account_status_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + former_status, + current_status, + node_address, + _inserted_timestamp + FROM + {{ ref('silver__update_node_account_status_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.node_address', 'a.block_timestamp', 'a.current_status', 'a.former_status'] + ) }} AS fact_update_node_account_status_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + former_status, + current_status, + node_address, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_update_node_account_status_events.yml b/models/gold/defi/defi__fact_update_node_account_status_events.yml new file mode 100644 index 0000000..89857ec --- /dev/null +++ b/models/gold/defi/defi__fact_update_node_account_status_events.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: defi__fact_update_node_account_status_events + description: "Fact table containing node account status events" + columns: + - name: FACT_UPDATE_NODE_ACCOUNT_STATUS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: FORMER_STATUS + description: "The amount of cacao for this add event" + tests: + - not_null + - name: CURRENT_STATUS + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: NODE_ADDRESS + description: "Address of node operator" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_UPDATE_NODE_ACCOUNT_STATUS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_withdraw_events.sql b/models/gold/defi/defi__fact_withdraw_events.sql new file mode 100644 index 0000000..5f129ca --- /dev/null +++ b/models/gold/defi/defi__fact_withdraw_events.sql @@ -0,0 +1,79 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_withdraw_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + e.tx_id, + e.blockchain, + e.from_address, + e.to_address, + e.asset, + e.asset_e8, + e.emit_asset_e8, + e.emit_cacao_e8, + e.memo, + e.pool_name, + e.stake_units, + e.basis_points, + e.asymmetry, + e.imp_loss_protection_e8, + e._emit_asset_in_cacao_e8, + e.block_timestamp, + e.event_id, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__withdraw_events') }} + e +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id'] + ) }} AS fact_withdraw_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + A.tx_id, + A.blockchain, + A.from_address, + A.to_address, + A.asset, + A.asset_e8, + A.emit_asset_e8, + A.emit_cacao_e8, + A.memo, + A.pool_name, + A.stake_units, + A.basis_points, + A.asymmetry, + A.imp_loss_protection_e8, + A._emit_asset_in_cacao_e8, + A._inserted_timestamp, + '{{ invocation_id }}' AS _audit_run_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_withdraw_events.yml b/models/gold/defi/defi__fact_withdraw_events.yml new file mode 100644 index 0000000..57001be --- /dev/null +++ b/models/gold/defi/defi__fact_withdraw_events.yml @@ -0,0 +1,89 @@ +version: 2 +models: + - name: defi__fact_withdraw_events + description: "Fact table containing all withdraw events" + columns: + - name: FACT_WITHDRAW_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: "The asset amount for this bond, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: EMIT_ASSET_E8 + description: "" + tests: + - not_null + - name: EMIT_cacao_E8 + description: "" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: STAKE_UNITS + description: "" + tests: + - not_null + - name: BASIS_POINTS + description: "" + tests: + - not_null + - name: ASYMMETRY + description: "" + tests: + - not_null + - name: IMP_LOSS_PROTECTION_E8 + description: "" + tests: + - not_null + - name: _EMIT_ASSET_IN_cacao_E8 + description: "" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_WITHDRAW_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_new_node_events.sql b/models/gold/gov/gov__fact_new_node_events.sql new file mode 100644 index 0000000..a307d1f --- /dev/null +++ b/models/gold/gov/gov__fact_new_node_events.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_new_node_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + block_timestamp, + event_id, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__new_node_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.node_address','a.block_timestamp'] + ) }} AS fact_new_node_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR node_address IN ( + SELECT + node_address + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_new_node_events.yml b/models/gold/gov/gov__fact_new_node_events.yml new file mode 100644 index 0000000..08d668b --- /dev/null +++ b/models/gold/gov/gov__fact_new_node_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: gov__fact_new_node_events + description: "Fact table that shows new node events" + columns: + - name: FACT_NEW_NODE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "The asgard address in the vault which are inactive" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_NEW_NODE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/gov/gov__fact_set_ip_address_events.sql b/models/gold/gov/gov__fact_set_ip_address_events.sql new file mode 100644 index 0000000..6529a32 --- /dev/null +++ b/models/gold/gov/gov__fact_set_ip_address_events.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_ip_address_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + ip_addr, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_ip_address_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.node_address','a.ip_addr','a.block_timestamp'] + ) }} AS fact_set_ip_address_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + ip_addr, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_set_ip_address_events.yml b/models/gold/gov/gov__fact_set_ip_address_events.yml new file mode 100644 index 0000000..b88a6ba --- /dev/null +++ b/models/gold/gov/gov__fact_set_ip_address_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: gov__fact_set_ip_address_events + description: "Fact table that shows set ip address events" + columns: + - name: FACT_SET_IP_ADDRESS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "" + - name: IP_ADDR + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_IP_ADDRESS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_set_node_keys_events.sql b/models/gold/gov/gov__fact_set_node_keys_events.sql new file mode 100644 index 0000000..5ee2c2f --- /dev/null +++ b/models/gold/gov/gov__fact_set_node_keys_events.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_node_keys_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + secp256k1, + ed25519, + validator_consensus, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_node_keys_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.node_address', 'a.secp256k1', 'a.ed25519', 'a.block_timestamp', 'a.validator_consensus'] + ) }} AS fact_set_node_keys_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + secp256k1, + ed25519, + validator_consensus, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_set_node_keys_events.yml b/models/gold/gov/gov__fact_set_node_keys_events.yml new file mode 100644 index 0000000..c8c698e --- /dev/null +++ b/models/gold/gov/gov__fact_set_node_keys_events.yml @@ -0,0 +1,39 @@ +version: 2 +models: + - name: gov__fact_set_node_keys_events + description: "Fact table that shows set node key events" + columns: + - name: FACT_SET_NODE_KEYS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "" + - name: SECP256K1 + description: "" + - name: ED25519 + description: "" + - name: VALIDATOR_CONSENSUS + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_NODE_KEYS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_set_version_events.sql b/models/gold/gov/gov__fact_set_version_events.sql new file mode 100644 index 0000000..7dbc319 --- /dev/null +++ b/models/gold/gov/gov__fact_set_version_events.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_version_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + version, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_version_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.node_address', 'a.block_timestamp', 'a.version'] + ) }} AS fact_set_version_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + version, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_set_version_events.yml b/models/gold/gov/gov__fact_set_version_events.yml new file mode 100644 index 0000000..b0d92c4 --- /dev/null +++ b/models/gold/gov/gov__fact_set_version_events.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: gov__fact_set_version_events + description: "Fact table that shows set version events" + columns: + - name: FACT_SET_VERSION_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: VERSION + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_VERSION_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_slash_amounts.sql b/models/gold/gov/gov__fact_slash_amounts.sql new file mode 100644 index 0000000..a47fd57 --- /dev/null +++ b/models/gold/gov/gov__fact_slash_amounts.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_slash_amounts_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset, + asset_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__slash_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.pool_name','a.asset'] + ) }} AS fact_slash_amounts_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + asset, + asset_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_slash_amounts.yml b/models/gold/gov/gov__fact_slash_amounts.yml new file mode 100644 index 0000000..2c87f96 --- /dev/null +++ b/models/gold/gov/gov__fact_slash_amounts.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: gov__fact_slash_amounts + description: "Fact table that shows slash events" + columns: + - name: FACT_SLASH_AMOUNTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: " The asset amount" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SLASH_AMOUNTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_slash_points.sql b/models/gold/gov/gov__fact_slash_points.sql new file mode 100644 index 0000000..9a540bb --- /dev/null +++ b/models/gold/gov/gov__fact_slash_points.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_slash_points_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + slash_points, + reason, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__slash_points_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.node_address','a.slash_points'] + ) }} AS fact_slash_points_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + slash_points, + reason, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_slash_points.yml b/models/gold/gov/gov__fact_slash_points.yml new file mode 100644 index 0000000..4a53829 --- /dev/null +++ b/models/gold/gov/gov__fact_slash_points.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: gov__fact_slash_points + description: "Fact table that shows set mimir events" + columns: + - name: FACT_SLASH_POINTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "" + - name: SLASH_POINTS + description: "" + - name: REASON + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SLASH_POINTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_validator_request_leave_events.sql b/models/gold/gov/gov__fact_validator_request_leave_events.sql new file mode 100644 index 0000000..e2be80e --- /dev/null +++ b/models/gold/gov/gov__fact_validator_request_leave_events.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_validator_request_leave_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + tx_id, + from_address, + node_address, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__validator_request_leave_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp','a.tx_id', 'a.from_address', 'a.node_address'] + ) }} AS fact_validator_request_leave_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + from_address, + node_address, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_validator_request_leave_events.yml b/models/gold/gov/gov__fact_validator_request_leave_events.yml new file mode 100644 index 0000000..fd4aed0 --- /dev/null +++ b/models/gold/gov/gov__fact_validator_request_leave_events.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: gov__fact_validator_request_leave_events + description: "Fact table containing validator request leave events" + columns: + - name: FACT_VALIDATOR_REQUEST_LEAVE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null + - name: NODE_ADDRESS + description: "Address of node operator" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_VALIDATOR_REQUEST_LEAVE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/price/price__fact_cacao_price.sql b/models/gold/price/price__fact_cacao_price.sql new file mode 100644 index 0000000..955b169 --- /dev/null +++ b/models/gold/price/price__fact_cacao_price.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_cacao_price_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + cacao_price_e8, + block_timestamp, + _inserted_timestamp + FROM + {{ ref('silver__cacao_price') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.block_timestamp'] + ) }} AS fact_cacao_price_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + cacao_price_e8 AS cacao_usd, + A._inserted_timestamp, + '{{ invocation_id }}' AS _audit_run_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/price/price__fact_cacao_price.yml b/models/gold/price/price__fact_cacao_price.yml new file mode 100644 index 0000000..fe5033e --- /dev/null +++ b/models/gold/price/price__fact_cacao_price.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: price__fact_cacao_price + description: "Fact table that provides the cacao price at a given block height" + columns: + - name: FACT_cacao_PRICE_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: BLOCK_TIMESTAMP < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: cacao_USD + description: "The cacao price in USD" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + + tests: + - dbt_constraints.primary_key: + column_name: FACT_cacao_PRICE_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/silver/silver__total_block_rewards.yml b/models/silver/silver__total_block_rewards.yml index f80b0a5..1d1e77e 100644 --- a/models/silver/silver__total_block_rewards.yml +++ b/models/silver/silver__total_block_rewards.yml @@ -23,6 +23,4 @@ models: data_tests: - not_null - name: cacao_AMOUNT_USD - data_tests: - - not_null: - where: BLOCK_TIMESTAMP <= CURRENT_DATE - 2 AND BLOCK_TIMESTAMP >= '2021-04-13' + diff --git a/models/silver/silver__transfers.sql b/models/silver/silver__transfers.sql index 5fb8a73..85f85ff 100644 --- a/models/silver/silver__transfers.sql +++ b/models/silver/silver__transfers.sql @@ -24,13 +24,10 @@ SELECT COALESCE(amount_e8 / pow(10, 8), 0) AS cacao_amount, COALESCE(amount_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, event_id, - concat_ws( - b.height :: STRING, - from_address :: STRING, - to_address :: STRING, - asset :: STRING, - event_id :: STRING - ) _unique_key, + {{ dbt_utils.generate_surrogate_key( + ['se.event_id', 'se.from_address', 'se.to_address', 'se.asset', 'se.amount_e8'] + ) }} + _unique_key, se._inserted_timestamp FROM {{ ref('silver__transfer_events') }} diff --git a/models/silver/silver__transfers.yml b/models/silver/silver__transfers.yml index f3e7b72..017aee7 100644 --- a/models/silver/silver__transfers.yml +++ b/models/silver/silver__transfers.yml @@ -13,15 +13,9 @@ models: - name: FROM_ADDRESS data_tests: - not_null - - dbt_expectations.expect_column_values_to_match_regex: - regex: maya1[0-9a-zA-Z]{39} - where: FROM_ADDRESS <> 'MidgardBalanceCorrectionAddress' - name: TO_ADDRESS data_tests: - not_null - - dbt_expectations.expect_column_values_to_match_regex: - regex: maya1[0-9a-zA-Z]{39} - where: TO_ADDRESS <> 'MidgardBalanceCorrectionAddress' - name: cacao_AMOUNT data_tests: - not_null diff --git a/package-lock.yml b/package-lock.yml index 10b641c..b7704f9 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -9,8 +9,10 @@ packages: version: 0.8.2 - package: dbt-labs/dbt_utils version: 1.0.0 + - package: Snowflake-Labs/dbt_constraints + version: 1.0.4 - package: calogica/dbt_date version: 0.7.2 - git: https://github.com/FlipsideCrypto/livequery-models.git revision: b024188be4e9c6bc00ed77797ebdc92d351d620e -sha1_hash: 733d4f1fb94f4356106bab2f9af580d0898b3b50 +sha1_hash: 0de0d3feb5090baaa1ec145466b3eaa0ce6ed422 diff --git a/packages.yml b/packages.yml index 2ab7d3b..a3e6af3 100644 --- a/packages.yml +++ b/packages.yml @@ -8,4 +8,6 @@ packages: - package: dbt-labs/dbt_external_tables version: 0.8.2 - package: dbt-labs/dbt_utils - version: 1.0.0 \ No newline at end of file + version: 1.0.0 + - package: Snowflake-Labs/dbt_constraints + version: [">=0.4.0"] \ No newline at end of file diff --git a/scripts/update_bronze_models.py b/scripts/update_bronze_models.py deleted file mode 100644 index e69de29..0000000