diff --git a/models/bronze/prices/bronze__complete_native_asset_metadata.sql b/models/bronze/prices/bronze__complete_native_asset_metadata.sql new file mode 100644 index 0000000..6a9cc5e --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_asset_metadata.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_prices'] +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'complete_native_asset_metadata' + ) }} +WHERE + blockchain = 'stellar' diff --git a/models/bronze/prices/bronze__complete_native_prices.sql b/models/bronze/prices/bronze__complete_native_prices.sql new file mode 100644 index 0000000..c504c6b --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_prices.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_prices'] +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'complete_native_prices' + ) }} +WHERE + blockchain = 'stellar' diff --git a/models/bronze/prices/bronze__complete_provider_asset_metadata.sql b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..e2b2b30 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_prices'] +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'complete_provider_asset_metadata' + ) }} +WHERE + LOWER(platform) = 'stellar' + AND len(token_address) > 0 + AND token_address NOT ILIKE 'http%' diff --git a/models/bronze/prices/bronze__complete_provider_prices.sql b/models/bronze/prices/bronze__complete_provider_prices.sql new file mode 100644 index 0000000..afe1410 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_prices.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_prices'] +) }} + +SELECT + asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id, + _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'complete_provider_prices' + ) }} diff --git a/models/bronze/prices/bronze__complete_token_asset_metadata.sql b/models/bronze/prices/bronze__complete_token_asset_metadata.sql new file mode 100644 index 0000000..c4d6ff0 --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_asset_metadata.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_prices'] +) }} + +SELECT + token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'complete_token_asset_metadata' + ) }} +WHERE + blockchain = 'stellar' diff --git a/models/bronze/prices/bronze__complete_token_prices.sql b/models/bronze/prices/bronze__complete_token_prices.sql new file mode 100644 index 0000000..807a83f --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_prices.sql @@ -0,0 +1,32 @@ +{{ config ( + materialized = 'view', + tags = ['bronze_prices'] +) }} + +SELECT + HOUR, + token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ source( + 'crosschain_silver', + 'complete_token_prices' + ) }} +WHERE + blockchain = 'stellar' diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index c2cb9d5..b2794e4 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -37,6 +37,18 @@ There is more information on how to use dbt docs in the last section of this doc - [defi.fact_liquidity_pools](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.defi__fact_liquidity_pools) - [defi.fact_trades](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.defi__fact_trades) +### Price Tables (`stellar`.`PRICE`.``) + +**Dimension Tables:** +- [price.dim_asset_metadata](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.price__dim_asset_metadata) + +**Fact Tables:** +- [price.fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.price__fact_prices_ohlc_hourly) + +**Convenience Views:** +- [price.ez_prices_hourly](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.price__ez_prices_hourly) +- [price.ez_asset_metadata](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.price__ez_asset_metadata) + ## **Data Model Overview** These Stellar models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** diff --git a/models/descriptions/prices.md b/models/descriptions/prices.md new file mode 100644 index 0000000..eeb74dc --- /dev/null +++ b/models/descriptions/prices.md @@ -0,0 +1,177 @@ +{% docs prices_table_doc %} + +This table contains hourly prices for tokens on this blockchain. +The sources of this data are [CoinMarketCap](https://coinmarketcap.com/) and [CoinGecko](https://www.coingecko.com/). + +{% enddocs %} + + +{% docs prices_address %} + +The unique address for a given token on this EVM blockchain. + +{% enddocs %} + + +{% docs prices_decimals %} + +The number of decimals for the token contract on this EVM blockchain. + +{% enddocs %} + + +{% docs prices_hour %} + +The hour at which the token price was recorded on this EVM blockchain. + +{% enddocs %} + + +{% docs prices_imputed %} + +This column indicates whether or not the price has been pulled forward from the previous hour. Sometimes hourly prices are missed from our data source, and in that case, we carry forward the last recorded hourly price until we record a new price. + +{% enddocs %} + + +{% docs prices_price %} + +The token price for a given hour on this EVM blockchain. + +{% enddocs %} + + +{% docs prices_dim_asset_metadata_table_doc %} + +A comprehensive dimensional table holding asset metadata and other relevant details pertaining to each id, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this dimensional model. If you are interested in using a curated data set instead, please utilize ez_asset_metadata. + +{% enddocs %} + + +{% docs prices_ez_asset_metadata_table_doc %} + +A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address and native asset. This data set is highly curated and contains metadata for one unique asset per blockchain. + +{% enddocs %} + + +{% docs prices_fact_prices_ohlc_hourly_table_doc %} + +A comprehensive fact table holding id and provider specific open, high, low, close hourly prices, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this fact-based model. If you are interested in using a curated data set instead, please utilize ez_prices_hourly. + +{% enddocs %} + + +{% docs prices_ez_prices_hourly_table_doc %} + +A convenience table for determining token prices by address and blockchain, and native asset prices by symbol and blockchain. This data set is highly curated and contains metadata for one price per hour per unique asset and blockchain. + +{% enddocs %} + + +{% docs prices_provider %} + +The provider or source of the data. + +{% enddocs %} + + +{% docs prices_asset_id %} + +The unique identifier representing the asset. + +{% enddocs %} + + +{% docs prices_name %} + +The name of the asset. + +{% enddocs %} + + +{% docs prices_symbol %} + +The symbol of the asset. + +{% enddocs %} + + +{% docs prices_token_address %} + +The specific address representing the asset on this EVM blockchain. This will be NULL if referring to a native asset. + +{% enddocs %} + + +{% docs prices_token_address_evm %} + +The specific address representing the asset on this EVM blockchain. This will be NULL if referring to a native asset. The case (upper / lower) may or may not be specified within the `dim_asset_metadata` table, as this column is raw and not transformed, coming directly from the provider APIs. However, in the `ez_` views, it will be lowercase by default for all EVMs. + +{% enddocs %} + + +{% docs prices_blockchain %} + +The Blockchain, Network, or Platform for this asset. + +{% enddocs %} + + +{% docs prices_blockchain_id %} + +The unique identifier of the Blockchain, Network, or Platform for this asset. + +{% enddocs %} + + +{% docs prices_is_native %} + +A flag indicating assets native to this EVM blockchain. + +{% enddocs %} + + +{% docs prices_is_deprecated %} + +A flag indicating if the asset is deprecated or no longer supported by the provider. + +{% enddocs %} + + + + +{% docs prices_is_imputed %} + +A flag indicating if the price was imputed, or derived, from the last arriving record. This is generally used for tokens with low-liquidity or inconsistent reporting. + +{% enddocs %} + + +{% docs prices_open %} + +Opening price of the recorded hour in USD. + +{% enddocs %} + + +{% docs prices_high %} + +Highest price of the recorded hour in USD. + +{% enddocs %} + + +{% docs prices_low %} + +Lowest price of the recorded hour in USD. + +{% enddocs %} + + +{% docs prices_close %} + +Closing price of the recorded hour in USD. + +{% enddocs %} + diff --git a/models/gold/prices/price__dim_asset_metadata.sql b/models/gold/prices/price__dim_asset_metadata.sql new file mode 100644 index 0000000..95d43d2 --- /dev/null +++ b/models/gold/prices/price__dim_asset_metadata.sql @@ -0,0 +1,32 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'dim_asset_metadata_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_issuer, asset_code,symbol, name) ", + tags = ['gold_prices','scheduled_core'] +) }} + +SELECT + provider_asset_id, + asset_issuer, + asset_code, + NAME, + symbol, + platform AS blockchain, + platform_id AS blockchain_id, + provider, + {{ dbt_utils.generate_surrogate_key(['complete_provider_asset_metadata_id']) }} AS dim_asset_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__complete_provider_asset_metadata') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} diff --git a/models/gold/prices/price__dim_asset_metadata.yml b/models/gold/prices/price__dim_asset_metadata.yml new file mode 100644 index 0000000..3e4a3ff --- /dev/null +++ b/models/gold/prices/price__dim_asset_metadata.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: price__dim_asset_metadata + description: '{{ doc("prices_dim_asset_metadata_table_doc") }}' + + columns: + - name: PROVIDER + description: '{{ doc("prices_provider")}}' + - name: PROVIDER_ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: NAME + description: '{{ doc("prices_name") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: BLOCKCHAIN_ID + description: '{{ doc("prices_blockchain_id") }}' + - name: DIM_ASSET_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/prices/price__ez_asset_metadata.sql b/models/gold/prices/price__ez_asset_metadata.sql new file mode 100644 index 0000000..0002ca0 --- /dev/null +++ b/models/gold/prices/price__ez_asset_metadata.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'ez_asset_metadata_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_issuer, asset_code, asset_id, symbol, name)", + tags = ['gold_prices','scheduled_core'] +) }} + +SELECT + provider_asset_id, + asset_issuer, + asset_code, + asset_id, + NAME, + symbol, + decimals, + blockchain, + FALSE AS is_native, + is_deprecated, + {{ dbt_utils.generate_surrogate_key(['complete_token_asset_metadata_id']) }} AS ez_asset_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__complete_token_asset_metadata') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} + UNION ALL + SELECT + asset_id AS provider_asset_id, + NULL AS asset_issuer, + NULL AS asset_code, + -5706705804583548011 AS asset_id, + NAME, + symbol, + decimals, + blockchain, + TRUE AS is_native, + is_deprecated, + {{ dbt_utils.generate_surrogate_key(['complete_native_asset_metadata_id']) }} AS ez_asset_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp + FROM + {{ ref('silver__complete_native_asset_metadata') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} diff --git a/models/gold/prices/price__ez_asset_metadata.yml b/models/gold/prices/price__ez_asset_metadata.yml new file mode 100644 index 0000000..f663849 --- /dev/null +++ b/models/gold/prices/price__ez_asset_metadata.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: price__ez_asset_metadata + description: '{{ doc("prices_ez_asset_metadata_table_doc") }}' + + columns: + - name: PROVIDER_ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: ASSET_ISSUER + description: '{{ doc("asset_issuer") }}' + - name: ASSET_CODE + description: '{{ doc("asset_code") }}' + - name: ASSET_ID + description: '{{ doc("asset_id") }}' + - name: NAME + description: '{{ doc("prices_name") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: DECIMALS + description: '{{ doc("prices_decimals") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' + - name: EZ_ASSET_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + \ No newline at end of file diff --git a/models/gold/prices/price__ez_prices_hourly.sql b/models/gold/prices/price__ez_prices_hourly.sql new file mode 100644 index 0000000..d3381a7 --- /dev/null +++ b/models/gold/prices/price__ez_prices_hourly.sql @@ -0,0 +1,63 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'ez_prices_hourly_id', + cluster_by = ['hour::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(asset_issuer, asset_code, asset_id, symbol, name)", + tags = ['gold_prices','scheduled_core'] +) }} + +SELECT + HOUR, + asset_issuer, + asset_code, + asset_id, + price, + symbol, + decimals, + blockchain, + FALSE AS is_native, + is_imputed, + is_deprecated, + {{ dbt_utils.generate_surrogate_key(['complete_token_prices_id']) }} AS ez_prices_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__complete_token_prices') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} + UNION ALL + SELECT + HOUR, + NULL AS asset_issuer, + NULL AS asset_code, + -5706705804583548011 AS asset_id, + NAME, + symbol, + decimals, + blockchain, + TRUE AS is_native, + is_imputed, + is_deprecated, + {{ dbt_utils.generate_surrogate_key(['complete_native_prices_id']) }} AS ez_prices_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp + FROM + {{ ref('silver__complete_native_prices') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} diff --git a/models/gold/prices/price__ez_prices_hourly.yml b/models/gold/prices/price__ez_prices_hourly.yml new file mode 100644 index 0000000..fe81d51 --- /dev/null +++ b/models/gold/prices/price__ez_prices_hourly.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: price__ez_prices_hourly + description: '{{ doc("prices_ez_prices_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: ASSET_ISSUER + description: '{{ doc("asset_issuer") }}' + - name: ASSET_CODE + description: '{{ doc("asset_code") }}' + - name: ASSET_ID + description: '{{ doc("asset_id") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: DECIMALS + description: '{{ doc("prices_decimals") }}' + - name: PRICE + description: '{{ doc("prices_price") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: IS_IMPUTED + description: '{{ doc("prices_is_imputed") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' + - name: EZ_PRICES_HOURLY_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/prices/price__fact_prices_ohlc_hourly.sql b/models/gold/prices/price__fact_prices_ohlc_hourly.sql new file mode 100644 index 0000000..dc90ca4 --- /dev/null +++ b/models/gold/prices/price__fact_prices_ohlc_hourly.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'fact_prices_ohlc_hourly_id', + cluster_by = ['hour::DATE','provider'], + tags = ['gold_prices','scheduled_core'] +) }} + +SELECT + provider_asset_id, + recorded_hour AS HOUR, + OPEN, + high, + low, + CLOSE, + provider, + {{ dbt_utils.generate_surrogate_key(['complete_provider_prices_id']) }} AS fact_prices_ohlc_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__complete_provider_prices') }} + +{% if is_incremental() %} +WHERE + modified_timestamp > ( + SELECT + COALESCE(MAX(modified_timestamp), '1970-01-01' :: TIMESTAMP) AS modified_timestamp + FROM + {{ this }}) + {% endif %} diff --git a/models/gold/prices/price__fact_prices_ohlc_hourly.yml b/models/gold/prices/price__fact_prices_ohlc_hourly.yml new file mode 100644 index 0000000..5431c0e --- /dev/null +++ b/models/gold/prices/price__fact_prices_ohlc_hourly.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: price__fact_prices_ohlc_hourly + description: '{{ doc("prices_fact_prices_ohlc_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: PROVIDER_ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: OPEN + description: '{{ doc("prices_open") }}' + - name: HIGH + description: '{{ doc("prices_high") }}' + - name: LOW + description: '{{ doc("prices_low") }}' + - name: CLOSE + description: '{{ doc("prices_close") }}' + - name: PROVIDER + description: '{{ doc("prices_provider")}}' + - name: FACT_PRICES_OHLC_HOURLY_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/silver/silver__accounts.yml b/models/silver/core/silver__accounts.yml similarity index 100% rename from models/silver/silver__accounts.yml rename to models/silver/core/silver__accounts.yml diff --git a/models/silver/silver__assets.yml b/models/silver/core/silver__assets.yml similarity index 100% rename from models/silver/silver__assets.yml rename to models/silver/core/silver__assets.yml diff --git a/models/silver/silver__ledgers.yml b/models/silver/core/silver__ledgers.yml similarity index 100% rename from models/silver/silver__ledgers.yml rename to models/silver/core/silver__ledgers.yml diff --git a/models/silver/silver__liquidity_pools.yml b/models/silver/core/silver__liquidity_pools.yml similarity index 100% rename from models/silver/silver__liquidity_pools.yml rename to models/silver/core/silver__liquidity_pools.yml diff --git a/models/silver/silver__operations.yml b/models/silver/core/silver__operations.yml similarity index 100% rename from models/silver/silver__operations.yml rename to models/silver/core/silver__operations.yml diff --git a/models/silver/silver__trades.yml b/models/silver/core/silver__trades.yml similarity index 100% rename from models/silver/silver__trades.yml rename to models/silver/core/silver__trades.yml diff --git a/models/silver/silver__transactions.yml b/models/silver/core/silver__transactions.yml similarity index 100% rename from models/silver/silver__transactions.yml rename to models/silver/core/silver__transactions.yml diff --git a/models/silver/prices/silver__complete_native_asset_metadata.sql b/models/silver/prices/silver__complete_native_asset_metadata.sql new file mode 100644 index 0000000..2e60573 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.sql @@ -0,0 +1,38 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'complete_native_asset_metadata_id', + tags = ['silver_prices','scheduled_core'] +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key(['complete_native_asset_metadata_id']) }} AS complete_native_asset_metadata_id, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze__complete_native_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_asset_metadata.yml b/models/silver/prices/silver__complete_native_asset_metadata.yml new file mode 100644 index 0000000..1c1fdf1 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__complete_native_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - SYMBOL + + columns: + - name: PROVIDER + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_native_prices.sql b/models/silver/prices/silver__complete_native_prices.sql new file mode 100644 index 0000000..59fcf50 --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'complete_native_prices_id', + cluster_by = ['hour::DATE'], + tags = ['silver_prices','scheduled_core'] +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key(['complete_native_prices_id']) }} AS complete_native_prices_id, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze__complete_native_prices' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_prices.yml b/models/silver/prices/silver__complete_native_prices.yml new file mode 100644 index 0000000..bb72433 --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__complete_native_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - SYMBOL + + columns: + - name: HOUR + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.sql b/models/silver/prices/silver__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..dcc8416 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'complete_provider_asset_metadata_id', + tags = ['silver_prices','scheduled_core'] +) }} + +SELECT + asset_id AS provider_asset_id, + UPPER( + CASE + WHEN len(TRIM(token_address)) = 56 THEN token_address + ELSE SPLIT_PART( + token_address, + '-', + 2 + ) + END + ) AS asset_issuer, + UPPER( + CASE + WHEN token_address LIKE '%-%' THEN SPLIT_PART( + token_address, + '-', + 1 + ) + ELSE symbol + END + ) AS asset_code, + NAME, + UPPER(symbol) AS symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key(['complete_provider_asset_metadata_id']) }} AS complete_provider_asset_metadata_id, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze__complete_provider_asset_metadata' + ) }} +WHERE + len(asset_issuer) > 0 + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} +) +{% endif %} diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.yml b/models/silver/prices/silver__complete_provider_asset_metadata.yml new file mode 100644 index 0000000..0b0ee59 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: silver__complete_provider_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - PROVIDER_ASSET_ID + - ASSET_ISSUER + - ASSET_CODE + - NAME + - SYMBOL + - PLATFORM + - PLATFORM_ID + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: PROVIDER_ASSET_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_prices.sql b/models/silver/prices/silver__complete_provider_prices.sql new file mode 100644 index 0000000..c4b78c2 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.sql @@ -0,0 +1,47 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'complete_provider_prices_id', + cluster_by = ['recorded_hour::DATE','provider'], + tags = ['silver_prices','scheduled_core'] +) }} + +SELECT + p.asset_id AS provider_asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + p.provider, + p.source, + p._inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key(['p.complete_provider_prices_id']) }} AS complete_provider_prices_id, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze__complete_provider_prices' + ) }} + p + INNER JOIN {{ ref('bronze__complete_provider_asset_metadata') }} + m + ON p.asset_id = m.asset_id + +{% if is_incremental() %} +WHERE + p.modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY p.asset_id, recorded_hour, p.provider +ORDER BY + p.modified_timestamp DESC)) = 1 diff --git a/models/silver/prices/silver__complete_provider_prices.yml b/models/silver/prices/silver__complete_provider_prices.yml new file mode 100644 index 0000000..4608e8e --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__complete_provider_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - PROVIDER_ASSET_ID + - RECORDED_HOUR + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: PROVIDER_ASSET_ID + tests: + - not_null + - name: RECORDED_HOUR + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_asset_metadata.sql b/models/silver/prices/silver__complete_token_asset_metadata.sql new file mode 100644 index 0000000..9da28ba --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.sql @@ -0,0 +1,96 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'complete_token_asset_metadata_id', + tags = ['silver_prices','scheduled_core'] +) }} + +WITH providers AS ( + + SELECT + asset_id AS provider_asset_id, + UPPER( + CASE + WHEN len(TRIM(token_address)) = 56 THEN token_address + ELSE SPLIT_PART( + token_address, + '-', + 2 + ) + END + ) AS asset_issuer, + UPPER( + CASE + WHEN token_address LIKE '%-%' THEN SPLIT_PART( + token_address, + '-', + 1 + ) + ELSE symbol + END + ) AS asset_code, + NAME, + UPPER(symbol) AS symbol, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp + FROM + {{ ref( + 'bronze__complete_token_asset_metadata' + ) }} A + WHERE + len(asset_issuer) > 0 + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} +) +{% endif %} + +qualify (ROW_NUMBER() over (PARTITION BY asset_issuer, asset_code +ORDER BY + provider, _inserted_timestamp DESC)) = 1 +) +SELECT + A.provider_asset_id, + A.asset_issuer, + A.asset_code, + COALESCE( + b.asset_id, + b2.asset_id + ) AS asset_id, + A.name, + A.symbol, + A.decimals, + A.blockchain, + A.blockchain_name, + A.blockchain_id, + A.is_deprecated, + A.provider, + A.source, + A._inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key(['a.asset_issuer','a.asset_code']) }} AS complete_token_asset_metadata_id, + '{{ invocation_id }}' AS _invocation_id +FROM + providers A + LEFT JOIN {{ ref('silver__assets') }} + b + ON A.asset_issuer = b.asset_issuer + AND A.asset_code = b.asset_code + LEFT JOIN {{ ref('silver__assets') }} + b2 + ON A.asset_issuer = b2.asset_issuer + AND A.symbol = b2.asset_code diff --git a/models/silver/prices/silver__complete_token_asset_metadata.yml b/models/silver/prices/silver__complete_token_asset_metadata.yml new file mode 100644 index 0000000..558e7dc --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: silver__complete_token_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ISSUER + - ASSET_CODE + - BLOCKCHAIN + + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ISSUER + tests: + - not_null + - name: ASSET_CODE + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_prices.sql b/models/silver/prices/silver__complete_token_prices.sql new file mode 100644 index 0000000..cb8411d --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.sql @@ -0,0 +1,105 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'complete_token_prices_id', + cluster_by = ['hour::DATE'], + tags = ['silver_prices','scheduled_core'] +) }} + +WITH providers AS ( + + SELECT + HOUR, + asset_id AS provider_asset_id, + UPPER( + CASE + WHEN len(TRIM(token_address)) = 56 THEN token_address + ELSE SPLIT_PART( + token_address, + '-', + 2 + ) + END + ) AS asset_issuer, + UPPER( + CASE + WHEN token_address LIKE '%-%' THEN SPLIT_PART( + token_address, + '-', + 1 + ) + ELSE symbol + END + ) AS asset_code, + NAME, + UPPER(symbol) AS symbol, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp + FROM + {{ ref( + 'bronze__complete_token_prices' + ) }} + p + WHERE + len(asset_issuer) > 0 + +{% if is_incremental() %} +AND modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} +) +{% endif %} + +qualify (ROW_NUMBER() over (PARTITION BY asset_issuer, asset_code, HOUR +ORDER BY + provider, _inserted_timestamp DESC)) = 1 +) +SELECT + A.hour, + A.provider_asset_id, + A.asset_issuer, + A.asset_code, + COALESCE( + b.asset_id, + b2.asset_id + ) AS asset_id, + A.name, + A.symbol, + A.decimals, + A.price, + A.blockchain, + A.blockchain_name, + A.blockchain_id, + A.is_imputed, + A.is_deprecated, + A.provider, + A.source, + A._inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + {{ dbt_utils.generate_surrogate_key(['a.asset_code','a.asset_issuer','a.HOUR']) }} AS complete_token_prices_id, + '{{ invocation_id }}' AS _invocation_id +FROM + providers A + LEFT JOIN {{ ref('silver__assets') }} + b + ON A.asset_issuer = b.asset_issuer + AND A.asset_code = b.asset_code + LEFT JOIN {{ ref('silver__assets') }} + b2 + ON A.asset_issuer = b2.asset_issuer + AND A.symbol = b.asset_code + AND b.asset_id IS NULL diff --git a/models/silver/prices/silver__complete_token_prices.yml b/models/silver/prices/silver__complete_token_prices.yml new file mode 100644 index 0000000..9d0058f --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: silver__complete_token_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - ASSET_ISSUER + - ASSET_CODE + + columns: + - name: HOUR + tests: + - not_null + - name: ASSET_ISSUER + tests: + - not_null + - name: ASSET_CODE + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_PRICES_ID \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 532d5cd..d231cc8 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -26,6 +26,12 @@ sources: tables: - name: number_sequence - name: labels_combined + - name: complete_native_asset_metadata + - name: complete_native_prices + - name: complete_provider_asset_metadata + - name: complete_provider_prices + - name: complete_token_asset_metadata + - name: complete_token_prices - name: github_actions database: STELLAR schema: github_actions