From 19841544e84c90c8e637aef8394f60607aa9974d Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 2 Jun 2022 12:17:34 -0600 Subject: [PATCH] AN-1344/prices (#24) * cmc prices * gecko prices * adjust $ * core view, no description yet pending swap prices * prices * price to price_usd * address nulls and del mcap * tests * test fix --- models/bronze/bronze__prices.sql | 34 ++++++++++++++++ models/core/core__fact_prices.sql | 20 ++++++++++ models/core/core__fact_prices.yml | 63 ++++++++++++++++++++++++++++++ models/descriptions/asset_id.md | 5 +++ models/descriptions/market_cap.md | 5 +++ models/descriptions/price_usd.md | 5 +++ models/descriptions/recorded_at.md | 5 +++ models/descriptions/source.md | 5 +++ models/descriptions/symbol.md | 5 +++ models/descriptions/timestamp.md | 5 +++ models/descriptions/token.md | 5 +++ models/silver/silver__prices.sql | 49 +++++++++++++++++++++++ models/silver/silver__prices.yml | 63 ++++++++++++++++++++++++++++++ models/sources.yml | 9 ++++- 14 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 models/bronze/bronze__prices.sql create mode 100644 models/core/core__fact_prices.sql create mode 100644 models/core/core__fact_prices.yml create mode 100644 models/descriptions/asset_id.md create mode 100644 models/descriptions/market_cap.md create mode 100644 models/descriptions/price_usd.md create mode 100644 models/descriptions/recorded_at.md create mode 100644 models/descriptions/source.md create mode 100644 models/descriptions/symbol.md create mode 100644 models/descriptions/timestamp.md create mode 100644 models/descriptions/token.md create mode 100644 models/silver/silver__prices.sql create mode 100644 models/silver/silver__prices.yml diff --git a/models/bronze/bronze__prices.sql b/models/bronze/bronze__prices.sql new file mode 100644 index 0000000..20959a4 --- /dev/null +++ b/models/bronze/bronze__prices.sql @@ -0,0 +1,34 @@ +{{ config( + materialized = 'view' +) }} + +WITH token_prices AS ( + + SELECT + * + FROM + {{ source( + 'silver', + 'prices_v2' + ) }} + WHERE + asset_id IN ( + '4558', + -- Flow + '6993', + -- Revv + '12182', + -- Blocto Token + '15139', + -- Starly + 'flow', + 'revv', + 'starly', + 'blocto-token' + ) + AND provider IS NOT NULL +) +SELECT + * +FROM + token_prices diff --git a/models/core/core__fact_prices.sql b/models/core/core__fact_prices.sql new file mode 100644 index 0000000..a8bccd8 --- /dev/null +++ b/models/core/core__fact_prices.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +WITH prices AS ( + + SELECT + * + FROM + {{ ref('silver__prices') }} +) +SELECT + recorded_at AS TIMESTAMP, + asset_id, + token, + symbol, + price_usd, + source +FROM + prices diff --git a/models/core/core__fact_prices.yml b/models/core/core__fact_prices.yml new file mode 100644 index 0000000..24ef0a1 --- /dev/null +++ b/models/core/core__fact_prices.yml @@ -0,0 +1,63 @@ + +version: 2 + +models: + - name: core__fact_prices + description: |- + This table provides token price data for FLOW tokens. + + columns: + - name: timestamp + description: "{{ doc('timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: asset_id + description: "{{ doc('asset_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - NUMBER + - VARCHAR + + - name: token + description: "{{ doc('token') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: symbol + description: "{{ doc('symbol') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: price_usd + description: "{{ doc('price_usd') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - DOUBLE + - FLOAT + + - name: source + description: "{{ doc('source') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR diff --git a/models/descriptions/asset_id.md b/models/descriptions/asset_id.md new file mode 100644 index 0000000..0389a71 --- /dev/null +++ b/models/descriptions/asset_id.md @@ -0,0 +1,5 @@ +{% docs asset_id %} + +Identifier used by the CoinMarketCap or CoinGecko APIs. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/market_cap.md b/models/descriptions/market_cap.md new file mode 100644 index 0000000..d58e498 --- /dev/null +++ b/models/descriptions/market_cap.md @@ -0,0 +1,5 @@ +{% docs market_cap %} + +Total market cap of the token, where available, in USD. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/price_usd.md b/models/descriptions/price_usd.md new file mode 100644 index 0000000..3bfe5b4 --- /dev/null +++ b/models/descriptions/price_usd.md @@ -0,0 +1,5 @@ +{% docs price_usd %} + +Asset price, in USD. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/recorded_at.md b/models/descriptions/recorded_at.md new file mode 100644 index 0000000..0cda5f8 --- /dev/null +++ b/models/descriptions/recorded_at.md @@ -0,0 +1,5 @@ +{% docs recorded_at %} + +Timestamp of when the data was recorded from the API or from on-chain data, depending on the source. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/source.md b/models/descriptions/source.md new file mode 100644 index 0000000..7e6b8f6 --- /dev/null +++ b/models/descriptions/source.md @@ -0,0 +1,5 @@ +{% docs source %} + +Data source for the record in question. E.g. for the prices table, this may be an API. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/symbol.md b/models/descriptions/symbol.md new file mode 100644 index 0000000..37c92ee --- /dev/null +++ b/models/descriptions/symbol.md @@ -0,0 +1,5 @@ +{% docs symbol %} + +Short-name symbol for the crypto token, e.g. BLT. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/timestamp.md b/models/descriptions/timestamp.md new file mode 100644 index 0000000..35661bb --- /dev/null +++ b/models/descriptions/timestamp.md @@ -0,0 +1,5 @@ +{% docs timestamp %} + +Timestamp of when the data was recorded from the API or from on-chain data, depending on the source. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/token.md b/models/descriptions/token.md new file mode 100644 index 0000000..5b0208e --- /dev/null +++ b/models/descriptions/token.md @@ -0,0 +1,5 @@ +{% docs token %} + +Friendly name of the crypto token, e.g. Blocto Token. + +{% enddocs %} \ No newline at end of file diff --git a/models/silver/silver__prices.sql b/models/silver/silver__prices.sql new file mode 100644 index 0000000..8ce16f5 --- /dev/null +++ b/models/silver/silver__prices.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + cluster_by = ['recorded_at'], + unique_key = "concat_ws( '-', recorded_at, asset_id )" +) }} + +WITH token_prices AS ( + + SELECT + * + FROM + {{ ref('bronze__prices') }} + +{% if is_incremental() %} +WHERE + recorded_at :: DATE >= CURRENT_DATE - 2 +{% endif %} +), +prices AS ( + SELECT + recorded_at, + asset_id, + NAME AS token, + SPLIT( + symbol, + '$' + ) AS symbol_split, + symbol_split [array_size(symbol_split) - 1] :: STRING AS symbol, + price, + provider AS source + FROM + token_prices +), +FINAL AS ( + SELECT + recorded_at, + asset_id, + token, + symbol, + price AS price_usd, + source + FROM + prices +) +SELECT + * +FROM + FINAL diff --git a/models/silver/silver__prices.yml b/models/silver/silver__prices.yml new file mode 100644 index 0000000..51f438e --- /dev/null +++ b/models/silver/silver__prices.yml @@ -0,0 +1,63 @@ + +version: 2 + +models: + - name: silver__prices + description: |- + This table provides token price data for FLOW tokens. + + columns: + - name: recorded_at + description: "{{ doc('recorded_at') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: asset_id + description: "{{ doc('asset_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - NUMBER + - VARCHAR + + - name: token + description: "{{ doc('token') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: symbol + description: "{{ doc('symbol') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: price_usd + description: "{{ doc('price_usd') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - DOUBLE + + - name: source + description: "{{ doc('source') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR diff --git a/models/sources.yml b/models/sources.yml index 4bfee5a..e0f0d4a 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -12,4 +12,11 @@ sources: database: flipside_prod_db schema: crosschain tables: - - name: address_labels \ No newline at end of file + - name: address_labels + + - name: silver + database: flipside_prod_db + schema: silver + tables: + - name: market_asset_metadata + - name: prices_v2