From b4eef49b022e052c39858ffc5ecac2052675daa2 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Thu, 30 Nov 2023 13:08:48 -0500 Subject: [PATCH] prices --- dbt_project.yml | 4 ++ .../bronze__asset_metadata_all_providers.sql | 19 +++++++ .../bronze__asset_metadata_priority.sql | 17 +++++++ .../bronze__hourly_prices_all_providers.sql | 20 ++++++++ .../prices/bronze__hourly_prices_priority.sql | 17 +++++++ models/descriptions/__overview__.md | 13 +++-- .../gold/price/price__dim_asset_metadata.sql | 15 ++++++ .../gold/price/price__dim_asset_metadata.yml | 25 ++++++++++ .../gold/price/price__ez_asset_metadata.sql | 14 ++++++ .../gold/price/price__ez_asset_metadata.yml | 20 ++++++++ .../price/price__ez_hourly_token_prices.sql | 15 ++++++ .../price/price__ez_hourly_token_prices.yml | 23 +++++++++ .../price/price__fact_hourly_token_prices.sql | 14 ++++++ .../price/price__fact_hourly_token_prices.yml | 16 ++++++ .../silver__asset_metadata_all_providers.sql | 46 +++++++++++++++++ .../silver__asset_metadata_all_providers.yml | 18 +++++++ .../price/silver__asset_metadata_priority.sql | 50 +++++++++++++++++++ .../price/silver__asset_metadata_priority.yml | 12 +++++ .../silver__hourly_prices_all_providers.sql | 35 +++++++++++++ .../silver__hourly_prices_all_providers.yml | 17 +++++++ .../price/silver__hourly_prices_priority.sql | 45 +++++++++++++++++ .../price/silver__hourly_prices_priority.yml | 16 ++++++ models/sources.yml | 7 ++- 23 files changed, 473 insertions(+), 5 deletions(-) create mode 100644 models/bronze/prices/bronze__asset_metadata_all_providers.sql create mode 100644 models/bronze/prices/bronze__asset_metadata_priority.sql create mode 100644 models/bronze/prices/bronze__hourly_prices_all_providers.sql create mode 100644 models/bronze/prices/bronze__hourly_prices_priority.sql create mode 100644 models/gold/price/price__dim_asset_metadata.sql create mode 100644 models/gold/price/price__dim_asset_metadata.yml create mode 100644 models/gold/price/price__ez_asset_metadata.sql create mode 100644 models/gold/price/price__ez_asset_metadata.yml create mode 100644 models/gold/price/price__ez_hourly_token_prices.sql create mode 100644 models/gold/price/price__ez_hourly_token_prices.yml create mode 100644 models/gold/price/price__fact_hourly_token_prices.sql create mode 100644 models/gold/price/price__fact_hourly_token_prices.yml create mode 100644 models/silver/price/silver__asset_metadata_all_providers.sql create mode 100644 models/silver/price/silver__asset_metadata_all_providers.yml create mode 100644 models/silver/price/silver__asset_metadata_priority.sql create mode 100644 models/silver/price/silver__asset_metadata_priority.yml create mode 100644 models/silver/price/silver__hourly_prices_all_providers.sql create mode 100644 models/silver/price/silver__hourly_prices_all_providers.yml create mode 100644 models/silver/price/silver__hourly_prices_priority.sql create mode 100644 models/silver/price/silver__hourly_prices_priority.yml diff --git a/dbt_project.yml b/dbt_project.yml index 8329ee4..09bcb4b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -47,6 +47,10 @@ vars: STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False UPDATE_UDFS_AND_SPS: False START_GHA_TASKS: False + API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' + EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' + ROLES: | + ["INTERNAL_DEV"] tests: +store_failures: true # all tests diff --git a/models/bronze/prices/bronze__asset_metadata_all_providers.sql b/models/bronze/prices/bronze__asset_metadata_all_providers.sql new file mode 100644 index 0000000..9fef02e --- /dev/null +++ b/models/bronze/prices/bronze__asset_metadata_all_providers.sql @@ -0,0 +1,19 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + token_address, + id, + symbol, + blockchain, + provider, + _unique_key, + _inserted_timestamp +FROM + {{ source( + 'crosschain_silver', + 'asset_metadata_all_providers' + ) }} +WHERE + blockchain = 'aptos' diff --git a/models/bronze/prices/bronze__asset_metadata_priority.sql b/models/bronze/prices/bronze__asset_metadata_priority.sql new file mode 100644 index 0000000..9ae119b --- /dev/null +++ b/models/bronze/prices/bronze__asset_metadata_priority.sql @@ -0,0 +1,17 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + token_address, + symbol, + provider, + id, + _inserted_timestamp +FROM + {{ source( + 'crosschain_silver', + 'asset_metadata_priority' + ) }} +WHERE + blockchain = 'aptos' diff --git a/models/bronze/prices/bronze__hourly_prices_all_providers.sql b/models/bronze/prices/bronze__hourly_prices_all_providers.sql new file mode 100644 index 0000000..a3139a1 --- /dev/null +++ b/models/bronze/prices/bronze__hourly_prices_all_providers.sql @@ -0,0 +1,20 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + token_address, + blockchain, + provider, + price, + is_imputed, + _inserted_timestamp, + _unique_key +FROM + {{ source( + 'crosschain_silver', + 'token_prices_all_providers_hourly' + ) }} +WHERE + blockchain = 'aptos' diff --git a/models/bronze/prices/bronze__hourly_prices_priority.sql b/models/bronze/prices/bronze__hourly_prices_priority.sql new file mode 100644 index 0000000..563f5ec --- /dev/null +++ b/models/bronze/prices/bronze__hourly_prices_priority.sql @@ -0,0 +1,17 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + token_address, + price, + is_imputed, + _inserted_timestamp +FROM + {{ source( + 'crosschain_silver', + 'token_prices_priority_hourly' + ) }} +WHERE + blockchain = 'aptos' diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index fe851c3..6564e9e 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -19,13 +19,13 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Tables (`aptos`.`CORE`.``) -**Dimension Tables:** +**Core Dimension Tables:** - [dim_aptos_names](#!/model/model.aptos.core__dim_aptos_names) - [dim_labels](#!/model/model.aptos.core__dim_labels) - [dim_tokens](#!/model/model.aptos.core__dim_tokens) -**Fact Tables:** +**Core Fact Tables:** - [fact_blocks](#!/model/model.aptos.core__fact_blocks) - [fact_events](#!/model/model.aptos.core__fact_events) - [fact_changes](#!/model/model.aptos.core__fact_changes) @@ -34,10 +34,15 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_transactions_state_checkpoint](#!/model/model.aptos.core__fact_transactions_state_checkpoint) - [fact_transfers](#!/model/model.aptos.core__fact_transfers) +**Core Dimension Tables:** +- [dim_asset_metadata](#!/model/model.aptos.price__dim_asset_metadata) +**Core Fact Tables:** +- [fact_hourly_token_prices](#!/model/model.aptos.price__fact_hourly_token_prices) -**Convenience Tables:** - +**Price Convenience Tables:** +- [ez_asset_metadata](#!/model/model.aptos.price__ez_asset_metadata) +- [ez_hourly_token_prices](#!/model/model.aptos.price__ez_hourly_token_prices) diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql new file mode 100644 index 0000000..8b6c904 --- /dev/null +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -0,0 +1,15 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + token_address, + id, + symbol, + NAME, + decimals, + provider +FROM + {{ ref('silver__asset_metadata_all_providers') }} \ No newline at end of file diff --git a/models/gold/price/price__dim_asset_metadata.yml b/models/gold/price/price__dim_asset_metadata.yml new file mode 100644 index 0000000..d5a030c --- /dev/null +++ b/models/gold/price/price__dim_asset_metadata.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: price__dim_asset_metadata + description: A comprehensive dimensional table holding all provider asset metadata and other relevant details pertaining to each token_address. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - PROVIDER + - ID + - TOKEN_ADDRESS + - SYMBOL + + columns: + - name: PROVIDER + description: The provider or source of the data. + - name: ID + description: The unique identifier representing the asset. + - name: NAME + description: The name of asset. + - name: SYMBOL + description: The symbol of asset. + - name: TOKEN_ADDRESS + description: The specific address representing the asset in a specific platform. + - name: DECIMALS + description: The number of decimal places the token needs adjusted where token values exist. diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql new file mode 100644 index 0000000..aa4a3df --- /dev/null +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -0,0 +1,14 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + token_address, + id, + symbol, + NAME, + decimals +FROM + {{ ref('silver__asset_metadata_priority') }} \ No newline at end of file diff --git a/models/gold/price/price__ez_asset_metadata.yml b/models/gold/price/price__ez_asset_metadata.yml new file mode 100644 index 0000000..907253b --- /dev/null +++ b/models/gold/price/price__ez_asset_metadata.yml @@ -0,0 +1,20 @@ +version: 2 +models: + - name: price__ez_asset_metadata + description: A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + + columns: + - name: ID + description: The unique identifier representing the asset. + - name: NAME + description: The name of asset. + - name: SYMBOL + description: The symbol of asset. + - name: TOKEN_ADDRESS + description: The specific address representing the asset in a specific platform. + - name: DECIMALS + description: The number of decimal places the token needs adjusted where token values exist. diff --git a/models/gold/price/price__ez_hourly_token_prices.sql b/models/gold/price/price__ez_hourly_token_prices.sql new file mode 100644 index 0000000..a584b9e --- /dev/null +++ b/models/gold/price/price__ez_hourly_token_prices.sql @@ -0,0 +1,15 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + HOUR, + token_address, + symbol, + decimals, + price, + is_imputed +FROM + {{ ref('silver__hourly_prices_priority') }} diff --git a/models/gold/price/price__ez_hourly_token_prices.yml b/models/gold/price/price__ez_hourly_token_prices.yml new file mode 100644 index 0000000..197910a --- /dev/null +++ b/models/gold/price/price__ez_hourly_token_prices.yml @@ -0,0 +1,23 @@ +version: 2 +models: + - name: price__ez_hourly_token_prices + description: A convenience table for determining token prices by address. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - TOKEN_ADDRESS + + columns: + - name: HOUR + description: Hour that the price was recorded at + - name: TOKEN_ADDRESS + description: Address of the token + - name: SYMBOL + description: Symbol of the token + - name: DECIMALS + description: The number of decimals for token contract + - name: PRICE + description: Closing price of the recorded hour in USD + - name: IS_IMPUTED + description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) \ No newline at end of file diff --git a/models/gold/price/price__fact_hourly_token_prices.sql b/models/gold/price/price__fact_hourly_token_prices.sql new file mode 100644 index 0000000..1894e4a --- /dev/null +++ b/models/gold/price/price__fact_hourly_token_prices.sql @@ -0,0 +1,14 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + HOUR, + token_address, + price, + is_imputed, + provider +FROM + {{ ref('silver__hourly_prices_all_providers') }} diff --git a/models/gold/price/price__fact_hourly_token_prices.yml b/models/gold/price/price__fact_hourly_token_prices.yml new file mode 100644 index 0000000..9337e7b --- /dev/null +++ b/models/gold/price/price__fact_hourly_token_prices.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: price__fact_hourly_token_prices + description: A comprehensive fact table holding provider specific hourly token prices. + + columns: + - name: HOUR + description: Hour that the price was recorded at + - name: TOKEN_ADDRESS + description: Address of the token + - name: PROVIDER + description: Source of the token price. + - name: PRICE + description: Closing price of the recorded hour in USD + - name: IS_IMPUTED + description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens) \ No newline at end of file diff --git a/models/silver/price/silver__asset_metadata_all_providers.sql b/models/silver/price/silver__asset_metadata_all_providers.sql new file mode 100644 index 0000000..9ce62e1 --- /dev/null +++ b/models/silver/price/silver__asset_metadata_all_providers.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'incremental', + unique_key = ['asset_metadata_all_providers_id'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + tags = ['core'] +) }} + +SELECT + token_address, + id, + COALESCE( + C.symbol, + p.symbol + ) AS symbol, + NAME, + decimals, + provider, + {{ dbt_utils.generate_surrogate_key( + ['token_address','provider','COALESCE( C.symbol, p.symbol ) ','id'] + ) }} AS asset_metadata_all_providers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + p._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__asset_metadata_all_providers') }} + p + LEFT JOIN {{ ref('silver__coin_info') }} C + ON C.coin_type = p.token_address + +{% if is_incremental() %} +WHERE + p._inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY token_address, id, COALESCE(C.symbol, p.symbol), provider +ORDER BY + p._inserted_timestamp DESC)) = 1 diff --git a/models/silver/price/silver__asset_metadata_all_providers.yml b/models/silver/price/silver__asset_metadata_all_providers.yml new file mode 100644 index 0000000..8eaedb8 --- /dev/null +++ b/models/silver/price/silver__asset_metadata_all_providers.yml @@ -0,0 +1,18 @@ +version: 2 +models: + - name: silver__asset_metadata_all_providers + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - SYMBOL + - PROVIDER + - ID + + columns: + - name: TOKEN_ADDRESS + tests: + - not_null + - name: PROVIDER + tests: + - not_null \ No newline at end of file diff --git a/models/silver/price/silver__asset_metadata_priority.sql b/models/silver/price/silver__asset_metadata_priority.sql new file mode 100644 index 0000000..813f9ac --- /dev/null +++ b/models/silver/price/silver__asset_metadata_priority.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + unique_key = ['token_address'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + tags = ['core'] +) }} + +SELECT + p.token_address, + p.id, + COALESCE( + C.symbol, + p.symbol + ) AS symbol, + C.name, + C.decimals, + p.provider, + CASE + WHEN p.provider = 'coingecko' THEN 1 + WHEN p.provider = 'coinmarketcap' THEN 2 + END AS priority, + {{ dbt_utils.generate_surrogate_key( + ['token_address'] + ) }} AS asset_metadata_priority_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + p._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__asset_metadata_priority') }} + p + LEFT JOIN {{ ref('silver__coin_info') }} C + ON C.coin_type = p.token_address + +{% if is_incremental() %} +WHERE + p._inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY token_address +ORDER BY + priority ASC)) = 1 diff --git a/models/silver/price/silver__asset_metadata_priority.yml b/models/silver/price/silver__asset_metadata_priority.yml new file mode 100644 index 0000000..9ccd956 --- /dev/null +++ b/models/silver/price/silver__asset_metadata_priority.yml @@ -0,0 +1,12 @@ +version: 2 +models: + - name: silver__asset_metadata_priority + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + + columns: + - name: TOKEN_ADDRESS + tests: + - not_null diff --git a/models/silver/price/silver__hourly_prices_all_providers.sql b/models/silver/price/silver__hourly_prices_all_providers.sql new file mode 100644 index 0000000..1f1d53f --- /dev/null +++ b/models/silver/price/silver__hourly_prices_all_providers.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'incremental', + unique_key = ['token_address', 'hour', 'provider'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + tags = ['core'] +) }} + +SELECT + HOUR, + token_address, + provider, + price, + is_imputed, + {{ dbt_utils.generate_surrogate_key( + ['token_address','hour','provider'] + ) }} AS hourly_prices_all_providers_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__hourly_prices_all_providers') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/price/silver__hourly_prices_all_providers.yml b/models/silver/price/silver__hourly_prices_all_providers.yml new file mode 100644 index 0000000..faee005 --- /dev/null +++ b/models/silver/price/silver__hourly_prices_all_providers.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: silver__hourly_prices_all_providers + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - HOUR + - PROVIDER + + columns: + - name: HOUR + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 \ No newline at end of file diff --git a/models/silver/price/silver__hourly_prices_priority.sql b/models/silver/price/silver__hourly_prices_priority.sql new file mode 100644 index 0000000..b9673f5 --- /dev/null +++ b/models/silver/price/silver__hourly_prices_priority.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + unique_key = ['token_address', 'hour'], + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + tags = ['core'] +) }} + +SELECT + p.hour, + p.token_address, + p.price, + p.is_imputed, + COALESCE( + C.symbol, + m.symbol + ) AS symbol, + C.decimals AS decimals, + {{ dbt_utils.generate_surrogate_key( + ['p.token_address','hour'] + ) }} AS hourly_prices_priority_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + p._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__hourly_prices_priority') }} + p + LEFT JOIN {{ ref('silver__asset_metadata_priority') }} + m + ON p.token_address = m.token_address + LEFT JOIN {{ ref('silver__coin_info') }} C + ON C.coin_type = p.token_address + +{% if is_incremental() %} +WHERE + p._inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) - INTERVAL '24 hours' + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/price/silver__hourly_prices_priority.yml b/models/silver/price/silver__hourly_prices_priority.yml new file mode 100644 index 0000000..3bfb2b7 --- /dev/null +++ b/models/silver/price/silver__hourly_prices_priority.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: silver__hourly_prices_priority + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - HOUR + + columns: + - name: HOUR + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 0c05e78..295c3cb 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -13,6 +13,10 @@ sources: schema: silver tables: - name: number_sequence + - name: token_prices_priority_hourly + - name: token_prices_all_providers_hourly + - name: asset_metadata_priority + - name: asset_metadata_all_providers - name: aptos_bronze database: aptos schema: bronze @@ -29,4 +33,5 @@ sources: {{ "APTOS_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "APTOS" }} tables: - name: blocks_tx - - name: transactions \ No newline at end of file + - name: transactions + \ No newline at end of file