AN-1714/prices from swaps (#37)

* init silver model

* cluster by block_timestamp

* column fixes

* prices from swaps, need to update core to add symbol & contract

* token labels seed and updated core view

* docs

data/seeds__token_labels.csv

@@ -0,0 +1,21 @@
+token,symbol,token_contract
+Starly,STARLY,A.142fa6570b62fd97.StarlyToken
+Blocto Token,BLT,A.0f9df91c9121c460.BloctoToken
+Flow,FLOW,A.1654653399040a61.FlowToken
+Sportium,SPRT,A.475755d2c9dccc3a.TeleportedSportiumToken
+Rally,RLY,A.231cc0dbbcffc4b7.RLY
+REVV,REVV,A.d01e482eb680ec9f.REVV
+USDT,USDT,A.cfdd90d4a00f7b5b.TeleportedTetherToken
+FUSD,FUSD,A.3c5959b568896393.FUSD
+USDC,USDC,A.b19436aae4d94622.FiatToken
+Celer Bridged AVAX,ceAVAX,A.231cc0dbbcffc4b7.ceAVAX
+Celer Bridged BNB,ceBNB,A.231cc0dbbcffc4b7.ceBNB
+Celer Bridged BUSD,ceBUSD,A.231cc0dbbcffc4b7.ceBUSD
+Celer Bridged DAI,ceDAI,A.231cc0dbbcffc4b7.ceDAI
+Celer Bridged FTM,ceFTM,A.231cc0dbbcffc4b7.ceFTM
+Celer Bridged MATIC,cdMATIC,A.231cc0dbbcffc4b7.ceMATIC
+Celer Bridged USDT,ceUSDT,A.231cc0dbbcffc4b7.ceUSDT
+Celer Bridged WBTC,ceWBTC,A.231cc0dbbcffc4b7.ceWBTC
+Celer Bridged WETH,ceWETH,A.231cc0dbbcffc4b7.ceWETH
+Faze Utility Coin,FazeUtiltiyCoin,A.4eded0de73020ca5.FazeUtilityCoin
+Silverlight Coin,SilverlightCoin,A.34dbcbc97ccf37b7.SilverlightCoin

data/seeds__token_labels.yml

@@ -0,0 +1,9 @@
+version: 2
+
+seeds:
+  - name: seeds__token_labels
+    config:
+      column_types:
+        token: varchar(48)
+        symbol: varchar(16)
+        token_contract: varchar(96)

dbt_project.yml

@@ -25,9 +25,9 @@ clean-targets: # directories to be removed by `dbt clean`
   - "dbt_packages"
 
 on-run-start:
-  - '{{create_sps()}}'
-  - '{{create_udfs()}}'
-  - '{{sp_create_bulk_get_topshot_moments_minted_metadata()}}'
+  - "{{create_sps()}}"
+  - "{{create_udfs()}}"
+  - "{{sp_create_bulk_get_topshot_moments_minted_metadata()}}"
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models

models/core/core__fact_prices.sql

@@ -2,19 +2,56 @@
     materialized = 'view'
 ) }}
 
-WITH prices AS (
+WITH token_labels AS (
 
     SELECT
         *
     FROM
-        {{ ref('silver__prices') }}
-)
-SELECT
+        {{ ref('seeds__token_labels') }}
+),
+prices AS (
+    SELECT
         recorded_at AS TIMESTAMP,
-    asset_id,
         token,
         symbol,
         price_usd,
         source
+    FROM
+        {{ ref('silver__prices') }}
+),
+prices_swaps AS (
+    SELECT
+        block_timestamp AS TIMESTAMP,
+        token_contract,
+        swap_price AS price_usd,
+        source
+    FROM
+        {{ ref('silver__prices_swaps') }}
+),
+viewnion AS (
+    SELECT
+        TIMESTAMP,
+        p.token,
+        p.symbol,
+        l.token_contract,
+        price_usd,
+        source
+    FROM
+        prices p
+        LEFT JOIN token_labels l USING (symbol)
+    UNION
+    SELECT
+        TIMESTAMP,
+        l.token,
+        l.symbol,
+        ps.token_contract,
+        price_usd,
+        source
+    FROM
+        prices_swaps ps
+        LEFT JOIN token_labels l USING (token_contract)
+)
+SELECT
+    *
 FROM
-    prices
+    viewnion

models/core/core__fact_prices.yml

@@ -1,4 +1,3 @@
-
 version: 2
 
 models:
@@ -15,16 +14,6 @@ models:
               column_type_list:
                 - TIMESTAMP_NTZ
 
-      - name: asset_id
-        description: "{{ doc('asset_id') }}"
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - STRING
-                - NUMBER
-                - VARCHAR
-
       - name: token
         description: "{{ doc('token') }}"
         tests:
@@ -43,6 +32,15 @@ models:
                 - STRING
                 - VARCHAR
 
+      - name: token_contract
+        description: "{{ doc('token_contract') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+
       - name: price_usd
         description: "{{ doc('price_usd') }}"
         tests:

models/descriptions/source.md

@@ -1,5 +1,5 @@
 {% docs source %}
 
-Data source for the record in question. E.g. for the prices table, this may be an API.
+Data source for the record in question. E.g. for the prices table, this may be an API or derived from swaps. Stableswap indicates the price was calculated from a swap where one of the assets was a stablecoin, while for a flowswap the other token was Flow.
 
 {% enddocs %}

models/descriptions/swap_price.md

@@ -0,0 +1,5 @@
+{% docs swap_price %}
+
+The asset price from a swap.
+
+{% enddocs %}

models/silver/silver__prices.sql

@@ -32,6 +32,21 @@ prices AS (
     FROM
         token_prices
 ),
+adj_token_names AS (
+    SELECT
+        recorded_at,
+        asset_id,
+        CASE
+            WHEN token LIKE 'Flow (%' THEN 'Flow'
+            WHEN token = 'Blocto Token' THEN 'Blocto'
+            ELSE token
+        END AS token,
+        symbol,
+        price,
+        source
+    FROM
+        prices
+),
 FINAL AS (
     SELECT
         recorded_at,
@@ -41,7 +56,7 @@ FINAL AS (
         price AS price_usd,
         source
     FROM
-        prices
+        adj_token_names
 )
 SELECT
     *

models/silver/silver__prices_swaps.sql

@@ -0,0 +1,182 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    cluster_by = ['block_timestamp::date'],
+    unique_key = "CONCAT_WS('-', block_timestamp, token_contract)"
+) }}
+
+WITH swaps AS (
+
+    SELECT
+        tx_id,
+        block_timestamp,
+        block_height,
+        swap_contract,
+        trader,
+        token_out_amount,
+        token_out_contract,
+        token_in_amount,
+        token_in_contract,
+        _ingested_at,
+        _inserted_timestamp
+    FROM
+        {{ ref('silver__swaps_single_trade') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}
+),
+flow_price AS (
+    SELECT
+        DATE_TRUNC(
+            'm',
+            recorded_at
+        ) AS _timestamp,
+        price_usd
+    FROM
+        {{ ref('silver__prices') }}
+    WHERE
+        symbol = 'FLOW'
+),
+stable_out AS (
+    SELECT
+        tx_id,
+        block_timestamp,
+        token_in_contract AS token_contract,
+        token_out_amount / token_in_amount AS swap_price,
+        _inserted_timestamp,
+        'stableswap' AS source
+    FROM
+        swaps
+    WHERE
+        token_out_contract IN (
+            'A.cfdd90d4a00f7b5b.TeleportedTetherToken',
+            'A.3c5959b568896393.FUSD',
+            'A.b19436aae4d94622.FiatToken'
+        )
+),
+stable_in AS (
+    SELECT
+        tx_id,
+        block_timestamp,
+        token_out_contract AS token_contract,
+        token_in_amount / token_out_amount AS swap_price,
+        _inserted_timestamp,
+        'stableswap' AS source
+    FROM
+        swaps
+    WHERE
+        token_in_contract IN (
+            'A.cfdd90d4a00f7b5b.TeleportedTetherToken',
+            'A.3c5959b568896393.FUSD',
+            'A.b19436aae4d94622.FiatToken'
+        )
+),
+stbl_tbl_union AS (
+    SELECT
+        *
+    FROM
+        stable_out
+    UNION
+    SELECT
+        *
+    FROM
+        stable_in
+),
+flow_out AS (
+    SELECT
+        tx_id,
+        block_timestamp,
+        token_in_contract AS token_contract,
+        token_out_amount / token_in_amount AS swap_price_in_flow,
+        _inserted_timestamp
+    FROM
+        swaps
+    WHERE
+        token_out_contract = 'A.1654653399040a61.FlowToken'
+),
+flow_in AS (
+    SELECT
+        tx_id,
+        block_timestamp,
+        token_in_contract AS token_contract,
+        token_out_amount / token_in_amount AS swap_price_in_flow,
+        _inserted_timestamp
+    FROM
+        swaps
+    WHERE
+        token_out_contract = 'A.1654653399040a61.FlowToken'
+),
+flow_tbl_union AS (
+    SELECT
+        tx_id,
+        DATE_TRUNC(
+            'm',
+            block_timestamp
+        ) AS _timestamp,
+        token_contract,
+        swap_price_in_flow,
+        _inserted_timestamp,
+        'flowswap' AS source
+    FROM
+        flow_out
+    UNION
+    SELECT
+        tx_id,
+        DATE_TRUNC(
+            'm',
+            block_timestamp
+        ) AS _timestamp,
+        token_contract,
+        swap_price_in_flow,
+        _inserted_timestamp,
+        'flowswap' AS source
+    FROM
+        flow_in
+),
+to_usd AS (
+    SELECT
+        tx_id,
+        ftu._timestamp,
+        token_contract,
+        swap_price_in_flow,
+        swap_price_in_flow * p.price_usd AS swap_price_usd,
+        _inserted_timestamp,
+        source
+    FROM
+        flow_tbl_union ftu
+        LEFT JOIN flow_price p USING (_timestamp)
+),
+FINAL AS (
+    SELECT
+        tx_id,
+        block_timestamp,
+        token_contract,
+        swap_price,
+        _inserted_timestamp,
+        source
+    FROM
+        stbl_tbl_union
+    UNION
+    SELECT
+        tx_id,
+        _timestamp AS block_timestamp,
+        token_contract,
+        swap_price_usd AS swap_price,
+        _inserted_timestamp,
+        source
+    FROM
+        to_usd
+    WHERE
+        swap_price IS NOT NULL
+)
+SELECT
+    *
+FROM
+    FINAL

models/silver/silver__prices_swaps.yml

@@ -0,0 +1,63 @@
+version: 2
+
+models:
+  - name: silver__prices_swaps
+    description: |-
+      This table provides token price data for FLOW tokens.
+
+    columns:
+      - name: tx_id
+        description: "{{ doc('tx_id') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - VARCHAR
+                - STRING
+
+      - name: block_timestamp
+        description: "{{ doc('block_timestamp') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+
+      - name: token_contract
+        description: "{{ doc('token_contract') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - VARCHAR
+                - STRING
+
+      - name: swap_price
+        description: "{{ doc('swap_price') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - FLOAT
+                - DOUBLE
+                - NUMBER
+
+      - name: _inserted_timestamp
+        description: "{{ doc('_inserted_timestamp') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+
+      - name: source
+        description: "{{ doc('source') }}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR