diff --git a/models/bronze/bronze__prices_v2.sql b/models/bronze/_legacy/bronze__prices_v2.sql similarity index 100% rename from models/bronze/bronze__prices_v2.sql rename to models/bronze/_legacy/bronze__prices_v2.sql diff --git a/models/bronze/bronze__labels.sql b/models/bronze/bronze__labels.sql index db0e93a..6cb414f 100644 --- a/models/bronze/bronze__labels.sql +++ b/models/bronze/bronze__labels.sql @@ -19,7 +19,7 @@ WITH labels AS ( labels_combined_id FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'labels_combined' ) }} WHERE diff --git a/models/bronze/prices/bronze__complete_native_asset_metadata.sql b/models/bronze/prices/bronze__complete_native_asset_metadata.sql new file mode 100644 index 0000000..835d103 --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_asset_metadata.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_asset_metadata' + ) }} +WHERE + blockchain = 'flow' + AND symbol = 'FLOW' diff --git a/models/bronze/prices/bronze__complete_native_prices.sql b/models/bronze/prices/bronze__complete_native_prices.sql new file mode 100644 index 0000000..c330bc1 --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_prices.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_prices' + ) }} +WHERE + blockchain = 'flow' + AND symbol = 'FLOW' diff --git a/models/bronze/prices/bronze__complete_provider_asset_metadata.sql b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..1b4035e --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql @@ -0,0 +1,25 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_asset_metadata' + ) }} +WHERE + platform = 'flow' \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_provider_prices.sql b/models/bronze/prices/bronze__complete_provider_prices.sql new file mode 100644 index 0000000..82d0627 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_prices.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_prices' + ) }} +-- prices for all ids diff --git a/models/bronze/prices/bronze__complete_token_asset_metadata.sql b/models/bronze/prices/bronze__complete_token_asset_metadata.sql new file mode 100644 index 0000000..14b1aba --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_asset_metadata.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_asset_metadata' + ) }} +WHERE + blockchain = 'flow' diff --git a/models/bronze/prices/bronze__complete_token_prices.sql b/models/bronze/prices/bronze__complete_token_prices.sql new file mode 100644 index 0000000..a3262c6 --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_prices.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_prices' + ) }} +WHERE + blockchain = 'flow' diff --git a/models/descriptions/prices/prices.md b/models/descriptions/prices/prices.md new file mode 100644 index 0000000..7c7f6a1 --- /dev/null +++ b/models/descriptions/prices/prices.md @@ -0,0 +1,137 @@ +{% docs prices_dim_asset_metadata_table_doc %} + +A comprehensive dimensional table holding asset metadata and other relevant details pertaining to each id, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this dimensional model. If you are interested in using a curated data set instead, please utilize ez_asset_metadata. + +{% enddocs %} + +{% docs prices_ez_asset_metadata_table_doc %} + +A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address and native asset. This data set is highly curated and contains metadata for one unique asset per blockchain. + +{% enddocs %} + +{% docs prices_fact_prices_ohlc_hourly_table_doc %} + +A comprehensive fact table holding id and provider specific open, high, low, close hourly prices, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this fact based model. If you are interested in using a curated data set instead, please utilize ez_prices_hourly. + +{% enddocs %} + +{% docs prices_ez_prices_hourly_table_doc %} + +A convenience table for determining token prices by address and blockchain, and native asset prices by symbol and blockchain. This data set is highly curated and contains metadata for one price per hour per unique asset and blockchain. + +{% enddocs %} + +{% docs prices_provider %} + +The provider or source of the data. + +{% enddocs %} + +{% docs prices_asset_id %} + +The unique identifier representing the asset. + +{% enddocs %} + +{% docs prices_name %} + +The name of asset. + +{% enddocs %} + +{% docs prices_symbol %} + +The symbol of asset. + +{% enddocs %} + +{% docs prices_token_address %} + +The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. + +{% enddocs %} + +{% docs prices_blockchain %} + +The Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_blockchain_id %} + +The unique identifier of the Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_decimals %} + +The number of decimals for the asset. May be NULL. + +{% enddocs %} + +{% docs prices_is_native %} + +A flag indicating assets native to the respective blockchain. + +{% enddocs %} + +{% docs prices_is_deprecated %} + +A flag indicating if the asset is deprecated or no longer supported by the provider. + +{% enddocs %} + +{% docs prices_id_deprecation %} + +Deprecating soon! Please use the `asset_id` column instead. + +{% enddocs %} + +{% docs prices_decimals_deprecation %} + +Deprecating soon! Please use the decimals column in `ez_asset_metadata` or join in `dim_contracts` instead. + +{% enddocs %} + +{% docs prices_hour %} + +Hour that the price was recorded at. + +{% enddocs %} + +{% docs prices_price %} + +Closing price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_is_imputed %} + +A flag indicating if the price was imputed, or derived, from the last arriving record. This is generally used for tokens with low-liquidity or inconsistent reporting. + +{% enddocs %} + +{% docs prices_open %} + +Opening price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_high %} + +Highest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_low %} + +Lowest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_close %} + +Closing price of the recorded hour in USD + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql new file mode 100644 index 0000000..16b02e0 --- /dev/null +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + token_address, + asset_id, + symbol, + name, + platform AS blockchain, + platform_id AS blockchain_id, + provider, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id AS dim_asset_metadata_id +FROM + {{ ref('silver__complete_provider_asset_metadata') }} diff --git a/models/gold/price/price__dim_asset_metadata.yml b/models/gold/price/price__dim_asset_metadata.yml new file mode 100644 index 0000000..46234b2 --- /dev/null +++ b/models/gold/price/price__dim_asset_metadata.yml @@ -0,0 +1,23 @@ +version: 2 +models: + - name: price__dim_asset_metadata + description: '{{ doc("prices_dim_asset_metadata_table_doc") }}' + + columns: + - name: PROVIDER + description: '{{ doc("prices_provider")}}' + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: NAME + description: '{{ doc("prices_name") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: BLOCKCHAIN_ID + description: '{{ doc("prices_blockchain_id") }}' + - name: DIM_ASSET_METADATA_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql new file mode 100644 index 0000000..bc1ec5c --- /dev/null +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + FALSE AS is_native, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id AS ez_asset_metadata_id +FROM + {{ ref('silver__complete_token_asset_metadata') }} +UNION ALL +SELECT + NULL AS token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + TRUE AS is_native, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id AS ez_asset_metadata_id +FROM + {{ ref('silver__complete_native_asset_metadata') }} diff --git a/models/gold/price/price__ez_asset_metadata.yml b/models/gold/price/price__ez_asset_metadata.yml new file mode 100644 index 0000000..ceb646c --- /dev/null +++ b/models/gold/price/price__ez_asset_metadata.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: price__ez_asset_metadata + description: '{{ doc("prices_ez_asset_metadata_table_doc") }}' + + columns: + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: NAME + description: '{{ doc("prices_name") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: DECIMALS + description: '{{ doc("prices_decimals") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: EZ_ASSET_METADATA_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/price/price__ez_prices_hourly.sql b/models/gold/price/price__ez_prices_hourly.sql new file mode 100644 index 0000000..dfda724 --- /dev/null +++ b/models/gold/price/price__ez_prices_hourly.sql @@ -0,0 +1,39 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + HOUR, + token_address, + symbol, + NAME, + decimals, + price, + blockchain, + FALSE AS is_native, + is_imputed, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id AS ez_prices_hourly_id +FROM + {{ ref('silver__complete_token_prices') }} +UNION ALL +SELECT + HOUR, + NULL AS token_address, + symbol, + NAME, + decimals, + price, + blockchain, + TRUE AS is_native, + is_imputed, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id AS ez_prices_hourly_id +FROM + {{ ref('silver__complete_native_prices') }} diff --git a/models/gold/price/price__ez_prices_hourly.yml b/models/gold/price/price__ez_prices_hourly.yml new file mode 100644 index 0000000..cc65a9b --- /dev/null +++ b/models/gold/price/price__ez_prices_hourly.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: price__ez_prices_hourly + description: '{{ doc("prices_ez_prices_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: DECIMALS + description: '{{ doc("prices_decimals") }}' + - name: PRICE + description: '{{ doc("prices_price") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: IS_IMPUTED + description: '{{ doc("prices_is_imputed") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' + - name: EZ_PRICES_HOURLY_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/price/price__fact_hourly_prices.yml b/models/gold/price/price__fact_hourly_prices.yml index c9fd88f..819b4df 100644 --- a/models/gold/price/price__fact_hourly_prices.yml +++ b/models/gold/price/price__fact_hourly_prices.yml @@ -3,6 +3,7 @@ version: 2 models: - name: price__fact_hourly_prices description: |- + Deprecating soon! Use `ez_prices_hourlt or price__fact_prices_ohlc_hourly` instead. This table provides hourly token price data for FLOW tokens from CoinGecko and CoinMarketCap. columns: diff --git a/models/gold/price/price__fact_prices.yml b/models/gold/price/price__fact_prices.yml index 52eee8d..d117e13 100644 --- a/models/gold/price/price__fact_prices.yml +++ b/models/gold/price/price__fact_prices.yml @@ -3,6 +3,7 @@ version: 2 models: - name: price__fact_prices description: |- + Deprecating soon! Use `ez_prices or price__fact_prices_ohlc_hourly` instead. This table reports prices derived from various on-chain sources. CoinGecko and CoinMarketCap price feeds can be found in the hourly price table. Note that prices from swaps may be volatile and are an approximation of price in USD terms, at best. A tx_hash is included for prices from swaps to provide a source. Low cap coins may have questionable prices due to low liquidity. diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.sql b/models/gold/price/price__fact_prices_ohlc_hourly.sql new file mode 100644 index 0000000..2be7d75 --- /dev/null +++ b/models/gold/price/price__fact_prices_ohlc_hourly.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + asset_id, + recorded_hour AS HOUR, + OPEN, + high, + low, + CLOSE, + provider, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id AS fact_prices_ohlc_hourly_id +FROM + {{ ref('silver__complete_provider_prices') }} diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.yml b/models/gold/price/price__fact_prices_ohlc_hourly.yml new file mode 100644 index 0000000..7dc31b5 --- /dev/null +++ b/models/gold/price/price__fact_prices_ohlc_hourly.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: price__fact_prices_ohlc_hourly + description: '{{ doc("prices_fact_prices_ohlc_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: OPEN + description: '{{ doc("prices_open") }}' + - name: HIGH + description: '{{ doc("prices_high") }}' + - name: LOW + description: '{{ doc("prices_low") }}' + - name: CLOSE + description: '{{ doc("prices_close") }}' + - name: FACT_PRICES_OHLC_HOURLY_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index dd20bed..056e948 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -14,7 +14,7 @@ WITH txs AS ( unique_from_count, total_fees AS total_fees_native, LAST_VALUE( - p.close ignore nulls + p.price ignore nulls ) over ( ORDER BY block_timestamp_hour rows unbounded preceding @@ -25,10 +25,10 @@ WITH txs AS ( FROM {{ ref('silver_stats__core_metrics_hourly') }} s - LEFT JOIN {{ ref('silver__prices_hourly') }} + LEFT JOIN {{ ref('silver__complete_native_prices') }} p - ON s.block_timestamp_hour = p.recorded_hour - AND p.id = 'Flow' + ON s.block_timestamp_hour = p.hour + AND p.asset_id = 'flow' ) SELECT A.block_timestamp_hour, @@ -41,7 +41,7 @@ SELECT b.unique_from_count, b.total_fees_native, ROUND( - b.total_fees_native * b.imputed_close, + b.total_fees_native * ZEROIFNULL(b.imputed_close), 2 ) AS total_fees_usd, A.core_metrics_block_hourly_id AS ez_core_metrics_hourly_id, diff --git a/models/silver/price/silver__prices.sql b/models/silver/_legacy/silver__prices.sql similarity index 100% rename from models/silver/price/silver__prices.sql rename to models/silver/_legacy/silver__prices.sql diff --git a/models/silver/price/silver__prices.yml b/models/silver/_legacy/silver__prices.yml similarity index 100% rename from models/silver/price/silver__prices.yml rename to models/silver/_legacy/silver__prices.yml diff --git a/models/silver/price/silver__prices_hourly.sql b/models/silver/_legacy/silver__prices_hourly.sql similarity index 96% rename from models/silver/price/silver__prices_hourly.sql rename to models/silver/_legacy/silver__prices_hourly.sql index ea0e593..00c2d86 100644 --- a/models/silver/price/silver__prices_hourly.sql +++ b/models/silver/_legacy/silver__prices_hourly.sql @@ -12,7 +12,7 @@ WITH token_prices AS ( SELECT * FROM - {{ ref('bronze__prices_v2') }} + {{ ref('silver__complete_provider_prices') }} {% if is_incremental() %} WHERE @@ -27,7 +27,7 @@ WHERE prices AS ( SELECT recorded_hour, - id AS asset_id, + asset_id, INITCAP(SPLIT(asset_id, '-') [0]) AS token, OPEN, high, diff --git a/models/silver/price/silver__prices_hourly.yml b/models/silver/_legacy/silver__prices_hourly.yml similarity index 100% rename from models/silver/price/silver__prices_hourly.yml rename to models/silver/_legacy/silver__prices_hourly.yml diff --git a/models/silver/price/silver__prices_swaps_hourly_s.sql b/models/silver/_legacy/silver__prices_swaps_hourly_s.sql similarity index 100% rename from models/silver/price/silver__prices_swaps_hourly_s.sql rename to models/silver/_legacy/silver__prices_swaps_hourly_s.sql diff --git a/models/silver/price/silver__prices_swaps_hourly_s.yml b/models/silver/_legacy/silver__prices_swaps_hourly_s.yml similarity index 100% rename from models/silver/price/silver__prices_swaps_hourly_s.yml rename to models/silver/_legacy/silver__prices_swaps_hourly_s.yml diff --git a/models/silver/price/silver__prices_swaps_s.sql b/models/silver/_legacy/silver__prices_swaps_s.sql similarity index 100% rename from models/silver/price/silver__prices_swaps_s.sql rename to models/silver/_legacy/silver__prices_swaps_s.sql diff --git a/models/silver/price/silver__prices_swaps_s.yml b/models/silver/_legacy/silver__prices_swaps_s.yml similarity index 100% rename from models/silver/price/silver__prices_swaps_s.yml rename to models/silver/_legacy/silver__prices_swaps_s.yml diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 5a77c36..b9f7be7 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -66,7 +66,7 @@ block_range AS ( _id AS block_height FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'number_sequence' ) }} WHERE @@ -111,7 +111,7 @@ block_gen AS ( _id AS block_height FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'number_sequence' ) }} WHERE diff --git a/models/silver/_observability/silver_observability__txs_completeness.sql b/models/silver/_observability/silver_observability__txs_completeness.sql index e1b6eb6..bf2f4ef 100644 --- a/models/silver/_observability/silver_observability__txs_completeness.sql +++ b/models/silver/_observability/silver_observability__txs_completeness.sql @@ -66,7 +66,7 @@ block_range AS ( _id AS block_height FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'number_sequence' ) }} WHERE diff --git a/models/silver/prices/silver__complete_native_asset_metadata.sql b/models/silver/prices/silver__complete_native_asset_metadata.sql new file mode 100644 index 0000000..98055f7 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_asset_metadata_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_asset_metadata.yml b/models/silver/prices/silver__complete_native_asset_metadata.yml new file mode 100644 index 0000000..1c1fdf1 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__complete_native_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - SYMBOL + + columns: + - name: PROVIDER + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_native_prices.sql b/models/silver/prices/silver__complete_native_prices.sql new file mode 100644 index 0000000..a88ed8d --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_prices_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_prices' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_prices.yml b/models/silver/prices/silver__complete_native_prices.yml new file mode 100644 index 0000000..bb72433 --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__complete_native_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - SYMBOL + + columns: + - name: HOUR + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.sql b/models/silver/prices/silver__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..b9132ed --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_asset_metadata_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_provider_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.yml b/models/silver/prices/silver__complete_provider_asset_metadata.yml new file mode 100644 index 0000000..8fa56fd --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__complete_provider_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - TOKEN_ADDRESS + - NAME + - SYMBOL + - PLATFORM + - PLATFORM_ID + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_prices.sql b/models/silver/prices/silver__complete_provider_prices.sql new file mode 100644 index 0000000..0fd6576 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_prices_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + p.asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + p.provider, + p.source, + p._inserted_timestamp, + p.inserted_timestamp, + p.modified_timestamp, + p.complete_provider_prices_id, + p._invocation_id +FROM + {{ ref( + 'bronze__complete_provider_prices' + ) }} + p + INNER JOIN {{ ref('bronze__complete_provider_asset_metadata') }} + m + ON p.asset_id = m.asset_id + +{% if is_incremental() %} +WHERE + p.modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY p.asset_id, recorded_hour, p.provider +ORDER BY + p.modified_timestamp DESC)) = 1 diff --git a/models/silver/prices/silver__complete_provider_prices.yml b/models/silver/prices/silver__complete_provider_prices.yml new file mode 100644 index 0000000..8db23c7 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__complete_provider_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - RECORDED_HOUR + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: RECORDED_HOUR + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_asset_metadata.sql b/models/silver/prices/silver__complete_token_asset_metadata.sql new file mode 100644 index 0000000..88dbead --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_asset_metadata_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + LOWER( + A.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_asset_metadata' + ) }} A + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_asset_metadata.yml b/models/silver/prices/silver__complete_token_asset_metadata.yml new file mode 100644 index 0000000..3eea590 --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__complete_token_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: PROVIDER + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_prices.sql b/models/silver/prices/silver__complete_token_prices.sql new file mode 100644 index 0000000..ac0d472 --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_prices_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + HOUR, + LOWER( + p.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_prices' + ) }} + p + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_prices.yml b/models/silver/prices/silver__complete_token_prices.yml new file mode 100644 index 0000000..d2ac57f --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.yml @@ -0,0 +1,41 @@ +version: 2 +models: + - name: silver__complete_token_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: HOUR + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 08f2cfa..6af53c0 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -8,12 +8,6 @@ sources: - name: flow_blocks - name: flow_txs - - name: crosschain_silver - database: crosschain - schema: silver - tables: - - name: labels_combined - - name: bronze_streamline database: streamline schema: | @@ -129,9 +123,22 @@ sources: database: crosschain schema: silver tables: + - name: labels_combined - name: hourly_prices_coin_gecko - name: hourly_prices_coin_market_cap + + - name: silver_crosschain + database: "{{ 'crosschain' if target.database == 'FLOW' else 'crosschain_dev' }}" + schema: silver + tables: + - name: complete_token_asset_metadata + - name: complete_token_prices + - name: complete_provider_asset_metadata + - name: complete_provider_prices + - name: complete_native_asset_metadata + - name: complete_native_prices - name: number_sequence + - name: labels_combined - name: bronze_api database: |