diff --git a/data/.gitkeep b/data/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/seeds__token_labels.csv b/data/seeds__token_labels.csv new file mode 100644 index 0000000..3fb6998 --- /dev/null +++ b/data/seeds__token_labels.csv @@ -0,0 +1,21 @@ +token,symbol,token_contract +Starly,STARLY,A.142fa6570b62fd97.StarlyToken +Blocto Token,BLT,A.0f9df91c9121c460.BloctoToken +Flow,FLOW,A.1654653399040a61.FlowToken +Sportium,SPRT,A.475755d2c9dccc3a.TeleportedSportiumToken +Rally,RLY,A.231cc0dbbcffc4b7.RLY +REVV,REVV,A.d01e482eb680ec9f.REVV +USDT,USDT,A.cfdd90d4a00f7b5b.TeleportedTetherToken +FUSD,FUSD,A.3c5959b568896393.FUSD +USDC,USDC,A.b19436aae4d94622.FiatToken +Celer Bridged AVAX,ceAVAX,A.231cc0dbbcffc4b7.ceAVAX +Celer Bridged BNB,ceBNB,A.231cc0dbbcffc4b7.ceBNB +Celer Bridged BUSD,ceBUSD,A.231cc0dbbcffc4b7.ceBUSD +Celer Bridged DAI,ceDAI,A.231cc0dbbcffc4b7.ceDAI +Celer Bridged FTM,ceFTM,A.231cc0dbbcffc4b7.ceFTM +Celer Bridged MATIC,cdMATIC,A.231cc0dbbcffc4b7.ceMATIC +Celer Bridged USDT,ceUSDT,A.231cc0dbbcffc4b7.ceUSDT +Celer Bridged WBTC,ceWBTC,A.231cc0dbbcffc4b7.ceWBTC +Celer Bridged WETH,ceWETH,A.231cc0dbbcffc4b7.ceWETH +Faze Utility Coin,FazeUtiltiyCoin,A.4eded0de73020ca5.FazeUtilityCoin +Silverlight Coin,SilverlightCoin,A.34dbcbc97ccf37b7.SilverlightCoin diff --git a/data/seeds__token_labels.yml b/data/seeds__token_labels.yml new file mode 100644 index 0000000..e4dd03d --- /dev/null +++ b/data/seeds__token_labels.yml @@ -0,0 +1,9 @@ +version: 2 + +seeds: + - name: seeds__token_labels + config: + column_types: + token: varchar(48) + symbol: varchar(16) + token_contract: varchar(96) diff --git a/dbt_project.yml b/dbt_project.yml index 749a1bd..699e69e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -25,9 +25,9 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_packages" on-run-start: - - '{{create_sps()}}' - - '{{create_udfs()}}' - - '{{sp_create_bulk_get_topshot_moments_minted_metadata()}}' + - "{{create_sps()}}" + - "{{create_udfs()}}" + - "{{sp_create_bulk_get_topshot_moments_minted_metadata()}}" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models diff --git a/models/core/core__fact_prices.sql b/models/core/core__fact_prices.sql index a8bccd8..2d1a617 100644 --- a/models/core/core__fact_prices.sql +++ b/models/core/core__fact_prices.sql @@ -2,19 +2,56 @@ materialized = 'view' ) }} -WITH prices AS ( +WITH token_labels AS ( SELECT * + FROM + {{ ref('seeds__token_labels') }} +), +prices AS ( + SELECT + recorded_at AS TIMESTAMP, + token, + symbol, + price_usd, + source FROM {{ ref('silver__prices') }} +), +prices_swaps AS ( + SELECT + block_timestamp AS TIMESTAMP, + token_contract, + swap_price AS price_usd, + source + FROM + {{ ref('silver__prices_swaps') }} +), +viewnion AS ( + SELECT + TIMESTAMP, + p.token, + p.symbol, + l.token_contract, + price_usd, + source + FROM + prices p + LEFT JOIN token_labels l USING (symbol) + UNION + SELECT + TIMESTAMP, + l.token, + l.symbol, + ps.token_contract, + price_usd, + source + FROM + prices_swaps ps + LEFT JOIN token_labels l USING (token_contract) ) SELECT - recorded_at AS TIMESTAMP, - asset_id, - token, - symbol, - price_usd, - source + * FROM - prices + viewnion diff --git a/models/core/core__fact_prices.yml b/models/core/core__fact_prices.yml index 24ef0a1..10b7b10 100644 --- a/models/core/core__fact_prices.yml +++ b/models/core/core__fact_prices.yml @@ -1,4 +1,3 @@ - version: 2 models: @@ -15,16 +14,6 @@ models: column_type_list: - TIMESTAMP_NTZ - - name: asset_id - description: "{{ doc('asset_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - NUMBER - - VARCHAR - - name: token description: "{{ doc('token') }}" tests: @@ -43,6 +32,15 @@ models: - STRING - VARCHAR + - name: token_contract + description: "{{ doc('token_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: price_usd description: "{{ doc('price_usd') }}" tests: diff --git a/models/descriptions/source.md b/models/descriptions/source.md index 7e6b8f6..3afba39 100644 --- a/models/descriptions/source.md +++ b/models/descriptions/source.md @@ -1,5 +1,5 @@ {% docs source %} -Data source for the record in question. E.g. for the prices table, this may be an API. +Data source for the record in question. E.g. for the prices table, this may be an API or derived from swaps. Stableswap indicates the price was calculated from a swap where one of the assets was a stablecoin, while for a flowswap the other token was Flow. {% enddocs %} \ No newline at end of file diff --git a/models/descriptions/swap_price.md b/models/descriptions/swap_price.md new file mode 100644 index 0000000..476c604 --- /dev/null +++ b/models/descriptions/swap_price.md @@ -0,0 +1,5 @@ +{% docs swap_price %} + +The asset price from a swap. + +{% enddocs %} \ No newline at end of file diff --git a/models/silver/silver__prices.sql b/models/silver/silver__prices.sql index e00126e..57022f9 100644 --- a/models/silver/silver__prices.sql +++ b/models/silver/silver__prices.sql @@ -32,6 +32,21 @@ prices AS ( FROM token_prices ), +adj_token_names AS ( + SELECT + recorded_at, + asset_id, + CASE + WHEN token LIKE 'Flow (%' THEN 'Flow' + WHEN token = 'Blocto Token' THEN 'Blocto' + ELSE token + END AS token, + symbol, + price, + source + FROM + prices +), FINAL AS ( SELECT recorded_at, @@ -41,7 +56,7 @@ FINAL AS ( price AS price_usd, source FROM - prices + adj_token_names ) SELECT * diff --git a/models/silver/silver__prices_swaps.sql b/models/silver/silver__prices_swaps.sql new file mode 100644 index 0000000..2ea1458 --- /dev/null +++ b/models/silver/silver__prices_swaps.sql @@ -0,0 +1,182 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + cluster_by = ['block_timestamp::date'], + unique_key = "CONCAT_WS('-', block_timestamp, token_contract)" +) }} + +WITH swaps AS ( + + SELECT + tx_id, + block_timestamp, + block_height, + swap_contract, + trader, + token_out_amount, + token_out_contract, + token_in_amount, + token_in_contract, + _ingested_at, + _inserted_timestamp + FROM + {{ ref('silver__swaps_single_trade') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +flow_price AS ( + SELECT + DATE_TRUNC( + 'm', + recorded_at + ) AS _timestamp, + price_usd + FROM + {{ ref('silver__prices') }} + WHERE + symbol = 'FLOW' +), +stable_out AS ( + SELECT + tx_id, + block_timestamp, + token_in_contract AS token_contract, + token_out_amount / token_in_amount AS swap_price, + _inserted_timestamp, + 'stableswap' AS source + FROM + swaps + WHERE + token_out_contract IN ( + 'A.cfdd90d4a00f7b5b.TeleportedTetherToken', + 'A.3c5959b568896393.FUSD', + 'A.b19436aae4d94622.FiatToken' + ) +), +stable_in AS ( + SELECT + tx_id, + block_timestamp, + token_out_contract AS token_contract, + token_in_amount / token_out_amount AS swap_price, + _inserted_timestamp, + 'stableswap' AS source + FROM + swaps + WHERE + token_in_contract IN ( + 'A.cfdd90d4a00f7b5b.TeleportedTetherToken', + 'A.3c5959b568896393.FUSD', + 'A.b19436aae4d94622.FiatToken' + ) +), +stbl_tbl_union AS ( + SELECT + * + FROM + stable_out + UNION + SELECT + * + FROM + stable_in +), +flow_out AS ( + SELECT + tx_id, + block_timestamp, + token_in_contract AS token_contract, + token_out_amount / token_in_amount AS swap_price_in_flow, + _inserted_timestamp + FROM + swaps + WHERE + token_out_contract = 'A.1654653399040a61.FlowToken' +), +flow_in AS ( + SELECT + tx_id, + block_timestamp, + token_in_contract AS token_contract, + token_out_amount / token_in_amount AS swap_price_in_flow, + _inserted_timestamp + FROM + swaps + WHERE + token_out_contract = 'A.1654653399040a61.FlowToken' +), +flow_tbl_union AS ( + SELECT + tx_id, + DATE_TRUNC( + 'm', + block_timestamp + ) AS _timestamp, + token_contract, + swap_price_in_flow, + _inserted_timestamp, + 'flowswap' AS source + FROM + flow_out + UNION + SELECT + tx_id, + DATE_TRUNC( + 'm', + block_timestamp + ) AS _timestamp, + token_contract, + swap_price_in_flow, + _inserted_timestamp, + 'flowswap' AS source + FROM + flow_in +), +to_usd AS ( + SELECT + tx_id, + ftu._timestamp, + token_contract, + swap_price_in_flow, + swap_price_in_flow * p.price_usd AS swap_price_usd, + _inserted_timestamp, + source + FROM + flow_tbl_union ftu + LEFT JOIN flow_price p USING (_timestamp) +), +FINAL AS ( + SELECT + tx_id, + block_timestamp, + token_contract, + swap_price, + _inserted_timestamp, + source + FROM + stbl_tbl_union + UNION + SELECT + tx_id, + _timestamp AS block_timestamp, + token_contract, + swap_price_usd AS swap_price, + _inserted_timestamp, + source + FROM + to_usd + WHERE + swap_price IS NOT NULL +) +SELECT + * +FROM + FINAL diff --git a/models/silver/silver__prices_swaps.yml b/models/silver/silver__prices_swaps.yml new file mode 100644 index 0000000..ca0c573 --- /dev/null +++ b/models/silver/silver__prices_swaps.yml @@ -0,0 +1,63 @@ +version: 2 + +models: + - name: silver__prices_swaps + description: |- + This table provides token price data for FLOW tokens. + + columns: + - name: tx_id + description: "{{ doc('tx_id') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - STRING + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: token_contract + description: "{{ doc('token_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - STRING + + - name: swap_price + description: "{{ doc('swap_price') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - FLOAT + - DOUBLE + - NUMBER + + - name: _inserted_timestamp + description: "{{ doc('_inserted_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: source + description: "{{ doc('source') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR