AN-5901/add-usdc-and-usdt-historical-xchain-supply (#100)

data/bronze__defillama_usdt_usdc_chain_seed.csv

@@ -0,0 +1,145 @@
+CHAIN,STABLECOIN_ID,SYMBOL
+algorand,1,USDT
+algorand,2,USDC
+aptos,2,USDC
+aptos,1,USDT
+arbitrum,2,USDC
+arbitrum,1,USDT
+astar,2,USDC
+astar,1,USDT
+aurora,1,USDT
+aurora,2,USDC
+avalanche,1,USDT
+avalanche,2,USDC
+berachain,2,USDC
+berachain,1,USDT
+binance,1,USDT
+binance,2,USDC
+bittorrent,1,USDT
+boba,2,USDC
+boba,1,USDT
+canto,2,USDC
+canto,1,USDT
+celo,1,USDT
+celo,2,USDC
+crab,1,USDT
+crab,2,USDC
+defichain,2,USDC
+defichain,1,USDT
+dogechain,1,USDT
+dogechain,2,USDC
+elrond,2,USDC
+elrond,1,USDT
+eos,1,USDT
+ethereum,1,USDT
+ethereum,2,USDC
+ethereumclassic,2,USDC
+ethereumclassic,1,USDT
+everscale,1,USDT
+everscale,2,USDC
+evmos,1,USDT
+evmos,2,USDC
+fantom,1,USDT
+fantom,2,USDC
+fuse,1,USDT
+fuse,2,USDC
+harmony,1,USDT
+heco,1,USDT
+icp,1,USDT
+icp,2,USDC
+injective,1,USDT
+injective,2,USDC
+ink,2,USDC
+ink,1,USDT
+iotex,1,USDT
+iotex,2,USDC
+kardia,2,USDC
+kardia,1,USDT
+kava,1,USDT
+kava,2,USDC
+klaytn,1,USDT
+klaytn,2,USDC
+kucoin,2,USDC
+kucoin,1,USDT
+kusama,1,USDT
+linea,1,USDT
+linea,2,USDC
+manta,2,USDC
+manta,1,USDT
+mantle,2,USDC
+mantle,1,USDT
+meter,2,USDC
+meter,1,USDT
+metis,1,USDT
+metis,2,USDC
+mixin,1,USDT
+mixin,2,USDC
+mode,1,USDT
+mode,2,USDC
+moonbeam,2,USDC
+moonbeam,1,USDT
+moonriver,1,USDT
+moonriver,2,USDC
+morph,1,USDT
+morph,2,USDC
+near,1,USDT
+near,2,USDC
+neo,1,USDT
+oasis,2,USDC
+oasis,1,USDT
+okexchain,1,USDT
+okexchain,2,USDC
+ontology,1,USDT
+optimism,2,USDC
+optimism,1,USDT
+osmosis,2,USDC
+osmosis,1,USDT
+polkadot,2,USDC
+polkadot,1,USDT
+pulse,1,USDT
+pulse,2,USDC
+pulsechain,1,USDT
+pulsechain,2,USDC
+rsk,1,USDT
+rsk,2,USDC
+scroll,1,USDT
+scroll,2,USDC
+sei,1,USDT
+sei,2,USDC
+shiden,1,USDT
+solana,2,USDC
+solana,1,USDT
+stacks,1,USDT
+starknet,2,USDC
+starknet,1,USDT
+sui,2,USDC
+sui,1,USDT
+sxnetwork,2,USDC
+sxnetwork,1,USDT
+syscoin,2,USDC
+syscoin,1,USDT
+taiko,2,USDC
+taiko,1,USDT
+telos,2,USDC
+telos,1,USDT
+terra,1,USDT
+terra,2,USDC
+tezos,1,USDT
+theta,1,USDT
+theta,2,USDC
+thundercore,2,USDC
+thundercore,1,USDT
+tomochain,2,USDC
+tomochain,1,USDT
+ton,1,USDT
+tron,2,USDC
+tron,1,USDT
+wanchain,2,USDC
+wanchain,1,USDT
+waves,1,USDT
+waves,2,USDC
+xdai,1,USDT
+xdai,2,USDC
+zilliqa,1,USDT
+zircuit,2,USDC
+zircuit,1,USDT

models/defillama/bronze/bronze__defillama_usdc_usdt_supply.sql

@@ -0,0 +1,56 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['chain', 'stablecoin_id','_inserted_timestamp'],
+    tags = ['defillama']
+) }}
+
+WITH usdt_supply AS (
+
+    SELECT
+        C.chain,
+        1 AS stablecoin_id,
+        live.udf_api(
+            'GET',
+            'https://pro-api.llama.fi/{api_key}/stablecoins/stablecoincharts/' || C.chain || '?stablecoin=1',{},{},
+            --usdt
+            'Vault/prod/external/defillama'
+        ) AS READ,
+        SYSDATE() AS _inserted_timestamp
+    FROM
+        {{ ref('bronze__defillama_usdt_usdc_chain_seed') }} C
+    WHERE
+        stablecoin_id = 1
+),
+usdc_supply AS (
+    SELECT
+        C.chain,
+        2 AS stablecoin_id,
+        live.udf_api(
+            'GET',
+            'https://pro-api.llama.fi/{api_key}/stablecoins/stablecoincharts/' || C.chain || '?stablecoin=2',{},{},
+            --usdc
+            'Vault/prod/external/defillama'
+        ) AS READ,
+        SYSDATE() AS _inserted_timestamp
+    FROM
+        {{ ref('bronze__defillama_usdt_usdc_chain_seed') }} C
+    WHERE
+        stablecoin_id = 2
+)
+SELECT
+    chain,
+    stablecoin_id,
+    READ,
+    bytes,
+    _inserted_timestamp
+FROM
+    usdt_supply
+UNION ALL
+SELECT
+    chain,
+    stablecoin_id,
+    READ,
+    bytes,
+    _inserted_timestamp
+FROM
+    usdc_supply

models/defillama/bronze/bronze__defillama_usdc_usdt_supply.yml

@@ -0,0 +1,10 @@
+version: 2
+models:
+  - name: bronze__defillama_usdc_usdt_supply
+    columns:
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ

models/defillama/gold/defillama__fact_usdt_usdc_supply.sql

@@ -0,0 +1,28 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    tags = ['defillama'],
+    meta={
+        'database_tags':{
+            'table': {
+                'PROTOCOL': 'DEFILLAMA'
+            }
+        }
+    }
+) }}
+
+SELECT
+    date,
+    stablecoin_id,
+    stablecoin,
+    symbol,
+    chain,
+    total_bridged_usd,
+    total_circulating,
+    total_circulating_usd,
+    defillama_usdc_usdt_supply_id as defillama_fact_usdt_usdc_supply_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM
+    {{ ref('silver__defillama_usdt_usdc_supply') }} f

models/defillama/gold/defillama__fact_usdt_usdc_supply.yml

@@ -0,0 +1,32 @@
+version: 2
+models:
+  - name: defillama__fact_usdt_usdc_supply
+    description: Daily USDT and USDC supply metrics across different blockchains
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - DEFILLAMA_FACT_USDT_USDC_SUPPLY_ID
+    
+    columns:
+      - name: DATE
+        description: Date of the supply data point
+      - name: STABLECOIN_ID
+        description: Unique identifier for the stablecoin from Defillama
+      - name: STABLECOIN
+        description: Name of the stablecoin
+      - name: SYMBOL
+        description: Token symbol (USDC or USDT)
+      - name: CHAIN
+        description: Blockchain network where the stablecoin is deployed
+      - name: TOTAL_BRIDGED_USD
+        description: Total value of bridged stablecoins in USD
+      - name: TOTAL_CIRCULATING
+        description: Total circulating supply of the stablecoin
+      - name: TOTAL_CIRCULATING_USD
+        description: Total circulating supply in USD value
+      - name: DEFILLAMA_FACT_USDT_USDC_SUPPLY_ID
+        description: Unique identifier for this record
+      - name: INSERTED_TIMESTAMP
+        description: Timestamp when the record was inserted
+      - name: MODIFIED_TIMESTAMP
+        description: Timestamp when the record was last modified

models/defillama/silver/silver__defillama_usdt_usdc_supply.sql

@@ -0,0 +1,83 @@
+{{ config(
+    materialized = 'incremental',
+    cluster_by = 'chain',
+    unique_key = 'defillama_usdc_usdt_supply_id',
+    tags = ['defillama']
+) }}
+
+WITH base AS (
+
+    SELECT
+        chain,
+        stablecoin_id,
+        READ,
+        bytes,
+        _inserted_timestamp
+    FROM
+        {{ ref('bronze__defillama_usdc_usdt_supply') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp > (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}
+),
+flattened_supply AS (
+    SELECT
+        chain,
+        stablecoin_id,
+        TO_TIMESTAMP_NTZ(
+            VALUE :date :: INT
+        ) AS DATE,
+        VALUE :totalBridgedToUSD :peggedUSD :: FLOAT AS total_bridged_usd,
+        VALUE :totalCirculating :peggedUSD :: FLOAT AS total_circulating,
+        VALUE :totalCirculatingUSD :peggedUSD :: FLOAT AS total_circulating_usd,
+        _inserted_timestamp
+    FROM
+        base,
+        LATERAL FLATTEN(
+            input => READ :data
+        )
+)
+SELECT
+    A.date,
+    A.stablecoin_id,
+    b.stablecoin,
+    b.symbol,
+    A.chain,
+    A.total_bridged_usd,
+    A.total_circulating,
+    A.total_circulating_usd,
+    A._inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['chain','date','a.stablecoin_id']
+    ) }} AS defillama_usdc_usdt_supply_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    flattened_supply A
+    LEFT JOIN {{ ref('bronze__defillama_stablecoins') }}
+    b
+    ON A.stablecoin_id = b.stablecoin_id 
+{% if is_incremental() %}
+WHERE
+    DATE > (
+        SELECT
+            MAX(DATE) :: DATE
+        FROM
+            {{ this }}
+    )
+{% endif %}
+    qualify ROW_NUMBER() over (
+        PARTITION BY A.chain,
+        A.date,
+        A.stablecoin_id
+        ORDER BY
+            A._inserted_timestamp DESC
+    ) = 1
+

models/defillama/silver/silver__defillama_usdt_usdc_supply.yml

@@ -0,0 +1,18 @@
+version: 2
+models:
+  - name: silver__defillama_usdt_usdc_supply
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - DEFILLAMA_USDC_USDT_SUPPLY_ID
+    
+    columns:
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2