prices

dbt_project.yml

@@ -47,6 +47,10 @@ vars:
   STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
   UPDATE_UDFS_AND_SPS: False
   START_GHA_TASKS: False
+  API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' 
+  EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}'
+  ROLES: |
+      ["INTERNAL_DEV"]
 
 tests:
   +store_failures: true # all tests

models/bronze/prices/bronze__asset_metadata_all_providers.sql

@@ -0,0 +1,19 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+SELECT
+    token_address,
+    id,
+    symbol,
+    blockchain,
+    provider,
+    _unique_key,
+    _inserted_timestamp
+FROM
+    {{ source(
+        'crosschain_silver',
+        'asset_metadata_all_providers'
+    ) }}
+WHERE
+    blockchain = 'aptos'

models/bronze/prices/bronze__asset_metadata_priority.sql

@@ -0,0 +1,17 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+SELECT
+    token_address,
+    symbol,
+    provider,
+    id,
+    _inserted_timestamp
+FROM
+    {{ source(
+        'crosschain_silver',
+        'asset_metadata_priority'
+    ) }}
+WHERE
+    blockchain = 'aptos'

models/bronze/prices/bronze__hourly_prices_all_providers.sql

@@ -0,0 +1,20 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+SELECT
+    HOUR,
+    token_address,
+    blockchain,
+    provider,
+    price,
+    is_imputed,
+    _inserted_timestamp,
+    _unique_key
+FROM
+    {{ source(
+        'crosschain_silver',
+        'token_prices_all_providers_hourly'
+    ) }}
+WHERE
+    blockchain = 'aptos'

models/bronze/prices/bronze__hourly_prices_priority.sql

@@ -0,0 +1,17 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+SELECT
+    HOUR,
+    token_address,
+    price,
+    is_imputed,
+    _inserted_timestamp
+FROM
+    {{ source(
+        'crosschain_silver',
+        'token_prices_priority_hourly'
+    ) }}
+WHERE
+    blockchain = 'aptos'

models/descriptions/__overview__.md

@@ -19,13 +19,13 @@ There is more information on how to use dbt docs in the last section of this doc
 
 ### Core Tables (`aptos`.`CORE`.`<table_name>`)
 
-**Dimension Tables:**
+**Core Dimension Tables:**
 - [dim_aptos_names](#!/model/model.aptos.core__dim_aptos_names)
 - [dim_labels](#!/model/model.aptos.core__dim_labels)
 - [dim_tokens](#!/model/model.aptos.core__dim_tokens)
 
 
-**Fact Tables:**
+**Core Fact Tables:**
 - [fact_blocks](#!/model/model.aptos.core__fact_blocks)
 - [fact_events](#!/model/model.aptos.core__fact_events)
 - [fact_changes](#!/model/model.aptos.core__fact_changes)
@@ -34,10 +34,15 @@ There is more information on how to use dbt docs in the last section of this doc
 - [fact_transactions_state_checkpoint](#!/model/model.aptos.core__fact_transactions_state_checkpoint)
 - [fact_transfers](#!/model/model.aptos.core__fact_transfers)
 
+**Core Dimension Tables:**
+- [dim_asset_metadata](#!/model/model.aptos.price__dim_asset_metadata)
 
+**Core Fact Tables:**
+- [fact_hourly_token_prices](#!/model/model.aptos.price__fact_hourly_token_prices)
 
-**Convenience Tables:**
-
+**Price Convenience Tables:**
+- [ez_asset_metadata](#!/model/model.aptos.price__ez_asset_metadata)
+- [ez_hourly_token_prices](#!/model/model.aptos.price__ez_hourly_token_prices)
 
 
 

models/gold/price/price__dim_asset_metadata.sql

@@ -0,0 +1,15 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    token_address,
+    id,
+    symbol,
+    NAME,
+    decimals,
+    provider
+FROM
+    {{ ref('silver__asset_metadata_all_providers') }}

models/gold/price/price__dim_asset_metadata.yml

@@ -0,0 +1,25 @@
+version: 2
+models:
+  - name: price__dim_asset_metadata
+    description: A comprehensive dimensional table holding all provider asset metadata and other relevant details pertaining to each token_address.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - PROVIDER
+            - ID
+            - TOKEN_ADDRESS
+            - SYMBOL
+            
+    columns:
+      - name: PROVIDER
+        description: The provider or source of the data.
+      - name: ID
+        description: The unique identifier representing the asset.
+      - name: NAME
+        description: The name of asset.
+      - name: SYMBOL
+        description: The symbol of asset.
+      - name: TOKEN_ADDRESS
+        description: The specific address representing the asset in a specific platform.
+      - name: DECIMALS
+        description: The number of decimal places the token needs adjusted where token values exist.

models/gold/price/price__ez_asset_metadata.sql

@@ -0,0 +1,14 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    token_address,
+    id,
+    symbol,
+    NAME,
+    decimals
+FROM
+    {{ ref('silver__asset_metadata_priority') }}

models/gold/price/price__ez_asset_metadata.yml

@@ -0,0 +1,20 @@
+version: 2
+models:
+  - name: price__ez_asset_metadata
+    description: A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TOKEN_ADDRESS
+            
+    columns:
+      - name: ID
+        description: The unique identifier representing the asset.
+      - name: NAME
+        description: The name of asset.
+      - name: SYMBOL
+        description: The symbol of asset.
+      - name: TOKEN_ADDRESS
+        description: The specific address representing the asset in a specific platform.
+      - name: DECIMALS
+        description: The number of decimal places the token needs adjusted where token values exist.

models/gold/price/price__ez_hourly_token_prices.sql

@@ -0,0 +1,15 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    HOUR,
+    token_address,
+    symbol,
+    decimals,
+    price,
+    is_imputed
+FROM
+    {{ ref('silver__hourly_prices_priority') }}

models/gold/price/price__ez_hourly_token_prices.yml

@@ -0,0 +1,23 @@
+version: 2
+models:
+  - name: price__ez_hourly_token_prices
+    description: A convenience table for determining token prices by address.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - HOUR
+            - TOKEN_ADDRESS
+            
+    columns:
+      - name: HOUR
+        description: Hour that the price was recorded at
+      - name: TOKEN_ADDRESS
+        description: Address of the token
+      - name: SYMBOL
+        description: Symbol of the token
+      - name: DECIMALS
+        description: The number of decimals for token contract
+      - name: PRICE
+        description: Closing price of the recorded hour in USD
+      - name: IS_IMPUTED
+        description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens)

models/gold/price/price__fact_hourly_token_prices.sql

@@ -0,0 +1,14 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    HOUR,
+    token_address,
+    price,
+    is_imputed,
+    provider
+FROM
+    {{ ref('silver__hourly_prices_all_providers') }}

models/gold/price/price__fact_hourly_token_prices.yml

@@ -0,0 +1,16 @@
+version: 2
+models:
+  - name: price__fact_hourly_token_prices
+    description: A comprehensive fact table holding provider specific hourly token prices.
+
+    columns:
+      - name: HOUR
+        description: Hour that the price was recorded at
+      - name: TOKEN_ADDRESS
+        description: Address of the token
+      - name: PROVIDER
+        description: Source of the token price.
+      - name: PRICE
+        description: Closing price of the recorded hour in USD
+      - name: IS_IMPUTED
+        description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens)

models/silver/price/silver__asset_metadata_all_providers.sql

@@ -0,0 +1,46 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['asset_metadata_all_providers_id'],
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['core']
+) }}
+
+SELECT
+    token_address,
+    id,
+    COALESCE(
+        C.symbol,
+        p.symbol
+    ) AS symbol,
+    NAME,
+    decimals,
+    provider,
+    {{ dbt_utils.generate_surrogate_key(
+        ['token_address','provider','COALESCE( C.symbol, p.symbol ) ','id']
+    ) }} AS asset_metadata_all_providers_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    p._inserted_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    {{ ref('bronze__asset_metadata_all_providers') }}
+    p
+    LEFT JOIN {{ ref('silver__coin_info') }} C
+    ON C.coin_type = p.token_address
+
+{% if is_incremental() %}
+WHERE
+    p._inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY token_address, id, COALESCE(C.symbol, p.symbol), provider
+ORDER BY
+    p._inserted_timestamp DESC)) = 1

models/silver/price/silver__asset_metadata_all_providers.yml

@@ -0,0 +1,18 @@
+version: 2
+models:
+  - name: silver__asset_metadata_all_providers
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TOKEN_ADDRESS
+            - SYMBOL
+            - PROVIDER
+            - ID
+
+    columns:   
+      - name: TOKEN_ADDRESS
+        tests:
+          - not_null
+      - name: PROVIDER
+        tests:
+          - not_null

models/silver/price/silver__asset_metadata_priority.sql

@@ -0,0 +1,50 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['token_address'],
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['core']
+) }}
+
+SELECT
+    p.token_address,
+    p.id,
+    COALESCE(
+        C.symbol,
+        p.symbol
+    ) AS symbol,
+    C.name,
+    C.decimals,
+    p.provider,
+    CASE
+        WHEN p.provider = 'coingecko' THEN 1
+        WHEN p.provider = 'coinmarketcap' THEN 2
+    END AS priority,
+    {{ dbt_utils.generate_surrogate_key(
+        ['token_address']
+    ) }} AS asset_metadata_priority_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    p._inserted_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    {{ ref('bronze__asset_metadata_priority') }}
+    p
+    LEFT JOIN {{ ref('silver__coin_info') }} C
+    ON C.coin_type = p.token_address
+
+{% if is_incremental() %}
+WHERE
+    p._inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY token_address
+ORDER BY
+    priority ASC)) = 1

models/silver/price/silver__asset_metadata_priority.yml

@@ -0,0 +1,12 @@
+version: 2
+models:
+  - name: silver__asset_metadata_priority
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TOKEN_ADDRESS
+
+    columns:   
+      - name: TOKEN_ADDRESS
+        tests:
+          - not_null

models/silver/price/silver__hourly_prices_all_providers.sql

@@ -0,0 +1,35 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['token_address', 'hour', 'provider'],
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['core']
+) }}
+
+SELECT
+    HOUR,
+    token_address,
+    provider,
+    price,
+    is_imputed,
+    {{ dbt_utils.generate_surrogate_key(
+        ['token_address','hour','provider']
+    ) }} AS hourly_prices_all_providers_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    _inserted_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    {{ ref('bronze__hourly_prices_all_providers') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/silver/price/silver__hourly_prices_all_providers.yml

@@ -0,0 +1,17 @@
+version: 2
+models:
+  - name: silver__hourly_prices_all_providers
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TOKEN_ADDRESS
+            - HOUR
+            - PROVIDER
+
+    columns:   
+      - name: HOUR
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1

models/silver/price/silver__hourly_prices_priority.sql

@@ -0,0 +1,45 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['token_address', 'hour'],
+    incremental_strategy = 'merge',
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['core']
+) }}
+
+SELECT
+    p.hour,
+    p.token_address,
+    p.price,
+    p.is_imputed,
+    COALESCE(
+        C.symbol,
+        m.symbol
+    ) AS symbol,
+    C.decimals AS decimals,
+    {{ dbt_utils.generate_surrogate_key(
+        ['p.token_address','hour']
+    ) }} AS hourly_prices_priority_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    p._inserted_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    {{ ref('bronze__hourly_prices_priority') }}
+    p
+    LEFT JOIN {{ ref('silver__asset_metadata_priority') }}
+    m
+    ON p.token_address = m.token_address
+    LEFT JOIN {{ ref('silver__coin_info') }} C
+    ON C.coin_type = p.token_address
+
+{% if is_incremental() %}
+WHERE
+    p._inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            ) - INTERVAL '24 hours'
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/silver/price/silver__hourly_prices_priority.yml

@@ -0,0 +1,16 @@
+version: 2
+models:
+  - name: silver__hourly_prices_priority
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TOKEN_ADDRESS
+            - HOUR
+
+    columns:   
+      - name: HOUR
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1

models/sources.yml

@@ -13,6 +13,10 @@ sources:
     schema: silver
     tables:
       - name: number_sequence
+      - name: token_prices_priority_hourly
+      - name: token_prices_all_providers_hourly
+      - name: asset_metadata_priority
+      - name: asset_metadata_all_providers
   - name: aptos_bronze
     database: aptos
     schema: bronze
@@ -29,4 +33,5 @@ sources:
       {{ "APTOS_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "APTOS" }}
     tables:
       - name: blocks_tx
-      - name: transactions
+      - name: transactions
+