WH tokens and boba and verified (#514)

* .

* wf

* clean lower

---------

Co-authored-by: Austin <austin@flipsidecrypto.com>

.github/workflows/dbt_run_adhoc_token_verification.yml

@@ -0,0 +1,52 @@
+name: dbt_run_adhoc_token_verification
+run-name: dbt_run_adhoc_token_verification
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt seed -m silver__manual_verified_token_mapping
+          dbt run -m models/silver/core/tokens/silver__manually_verified_tokens.sql models/silver/core/tokens/silver__tokens_enhanced.sql
+
+  notify-failure:
+    needs: [run_dbt_jobs]
+    if: failure()
+    uses: ./.github/workflows/slack_notify.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

data/metrics/silver__manual_verified_token_mapping.csv

@@ -194,6 +194,17 @@ bob,0x6c851f501a3f24e29a8e39a29591cddf09369080,dai,4943,,TRUE
 bob,0xa45d4121b3d47719ff57a947a9d961539ba33204,bitcoin,,,TRUE
 bob,0xba20a5e63eeefffa6fd365e7e540628f8fc61474,sovryn,,,TRUE
 bob,0x1217bfe6c773eec6cc4a38b5dc45b92292b6e189,openusdt,,,TRUE
+bob,0x03c7054bcb39f7b2e5b2c7acb37583e32d70cfa3,wrapped-bitcoin,,,TRUE
+bob,0xe75d0fb2c24a55ca1e3f96781a2bcc7bdba058f0,bob-network-bridged-usdce-bob-network,,,TRUE
+bob,0x541fd749419ca806a8bc7da8ac23d346f2df8b77,solv-btc,,,TRUE
+bob,0x4200000000000000000000000000000000000006,weth,,,TRUE
+bob,0x05d032ac25d322df992303dca074ee7392c117b9,tether,,,TRUE
+bob,0xcc0966d8418d412c599a6421b760a847eb169a8c,solv-btc,,,TRUE
+bob,0xbba2ef945d523c4e2608c9e1214c2cc64d4fc2e2,tbtc,,,TRUE
+bob,0x78fea795cbfcc5ffd6fb5b845a4f53d25c283bdb,satoshi-stablecoin,,,TRUE
+bob,0xe53dcec07d16d88e386ae0710e86d9a400f83c31,babylon,,,TRUE
+bob,0x236f8c0a61da474db21b693fb2ea7aab0c803894,universal-btc,,,TRUE
+bob,0xb5686c4f60904ec2bda6277d6fe1f7caa8d1b41a,rocket-pool-eth,,,TRUE
 bsc,0x43ef5494ac2fc8d1a6f5e838a02cb32b1941fda6,,,,FALSE
 bsc,0xaA292A51489bb4737f279aafA97F19F833337b0B,,,,FALSE
 bsc,0x8d0d000ee44948fc98c9b98a4fa4921476f08b0d,usd1-wlfi,,,TRUE

models/silver/core/tokens/silver__tokens_enhanced.sql

@@ -270,33 +270,95 @@ WHERE
         FROM
             {{ this }}
     )
-    OR address in (select address from man_ver where modified_timestamp > (select max(modified_timestamp) from {{ this }}))
+    OR address IN (
+        SELECT
+            address
+        FROM
+            man_ver
+        WHERE
+            modified_timestamp > (
+                SELECT
+                    MAX(modified_timestamp)
+                FROM
+                    {{ this }}
+            )
+    )
 {% endif %}
 ),
+wormhole_secured AS (
+    SELECT
+        blockchain,
+        token_address,
+        LOWER(token_address) AS token_address_lower,
+        symbol,
+        coingecko_id AS cg_id
+    FROM
+        {{ ref('silver__wormhole_secured_tokens') }}
+    WHERE
+        coingecko_id IS NOT NULL
+        AND len(token_address) > 0
+        AND blockchain IN (
+            SELECT
+                blockchain
+            FROM
+                {{ ref('silver__transfers_summary') }}
+        )
+
+{% if is_incremental() %}
+AND inserted_timestamp :: DATE >= (
+    SELECT
+        MAX(modified_timestamp) :: DATE - 7
+    FROM
+        {{ this }}
+)
+{% endif %}
+),
+combo AS (
+    SELECT
+        blockchain,
+        address_lower
+    FROM
+        token_base
+    UNION
+    SELECT
+        blockchain,
+        token_address_lower
+    FROM
+        wormhole_secured
+),
 FINAL AS (
     SELECT
         A.blockchain,
-        A.address,
+        COALESCE(
+            tb.address,
+            wh.token_address
+        ) AS address,
         CASE
             WHEN mv.verified_override IS NOT NULL THEN mv.verified_override
             WHEN b.blockchain IS NOT NULL
-            OR b_ex.blockchain IS NOT NULL THEN TRUE
+            OR b_ex.blockchain IS NOT NULL
+            OR wh.blockchain IS NOT NULL THEN TRUE
             ELSE FALSE
         END AS is_verified,
         CASE
-            WHEN b_ex.blockchain IS NOT NULL THEN TRUE
+            WHEN b_ex.blockchain IS NOT NULL
+            OR wh.blockchain IS NOT NULL THEN TRUE
             ELSE FALSE
         END AS is_verified_external,
-        A.symbol,
-        A.decimals,
-        A.name,
+        COALESCE(
+            tb.symbol,
+            wh.symbol
+        ) AS symbol,
+        tb.decimals,
+        tb.name,
         COALESCE(
             man_cg.cg_id,
             cg.id,
             cg_add.id,
             cs1_cg.id,
             cs2_cg.id,
-            c_ton_cg.id
+            c_ton_cg.id,
+            wh.cg_id
         ) AS coingecko_id,
         COALESCE(
             man_cmc.cmc_id :: STRING,
@@ -314,7 +376,10 @@ FINAL AS (
             ELSE FALSE
         END has_price_mapping
     FROM
-        token_base A
+        combo A
+        LEFT JOIN token_base tb
+        ON A.blockchain = tb.blockchain
+        AND A.address_lower = tb.address_lower
         LEFT JOIN man_ver mv
         ON A.blockchain = mv.blockchain
         AND A.address_lower = mv.address_lower
@@ -343,23 +408,23 @@ FINAL AS (
         LEFT JOIN stell cs1_cg
         ON A.blockchain = 'stellar'
         AND cs1_cg.provider = 'coingecko'
-        AND A.ton_asset_issuer = cs1_cg.asset_issuer
-        AND A.ton_asset_code = cs1_cg.asset_code
+        AND tb.ton_asset_issuer = cs1_cg.asset_issuer
+        AND tb.ton_asset_code = cs1_cg.asset_code
         LEFT JOIN stell cs2_cg
         ON A.blockchain = 'stellar'
         AND cs2_cg.provider = 'coinmarketcap'
-        AND A.ton_asset_issuer = cs2_cg.asset_issuer
-        AND A.ton_asset_code = cs2_cg.symbol
+        AND tb.ton_asset_issuer = cs2_cg.asset_issuer
+        AND tb.ton_asset_code = cs2_cg.symbol
         LEFT JOIN stell cs1_cmc
         ON A.blockchain = 'stellar'
         AND cs1_cmc.provider = 'coinmarketcap'
-        AND A.ton_asset_issuer = cs1_cmc.asset_issuer
-        AND A.ton_asset_code = cs1_cmc.asset_code
+        AND tb.ton_asset_issuer = cs1_cmc.asset_issuer
+        AND tb.ton_asset_code = cs1_cmc.asset_code
         LEFT JOIN stell cs2_cmc
         ON A.blockchain = 'stellar'
         AND cs2_cmc.provider = 'coinmarketcap'
-        AND A.ton_asset_issuer = cs2_cmc.asset_issuer
-        AND A.ton_asset_code = cs2_cmc.symbol
+        AND tb.ton_asset_issuer = cs2_cmc.asset_issuer
+        AND tb.ton_asset_code = cs2_cmc.symbol
         LEFT JOIN ton c_ton_cg
         ON A.blockchain = 'ton'
         AND c_ton_cg.provider = 'coingecko'
@@ -376,6 +441,9 @@ FINAL AS (
         ON A.blockchain = man_cmc.blockchain
         AND A.address_lower = man_cmc.address_lower
         AND man_cmc.cmc_id IS NOT NULL
+        LEFT JOIN wormhole_secured wh
+        ON A.blockchain = wh.blockchain
+        AND A.address_lower = wh.token_address_lower
 )
 SELECT
     A.blockchain,

models/silver/core/tokens/silver__wormhole_secured_tokens.sql

@@ -0,0 +1,60 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = ['blockchain', 'token_address'],
+    merge_exclude_columns = ['inserted_timestamp'],
+    full_refresh = false,
+    tags = ['daily']
+) }}
+
+WITH worms AS (
+
+    SELECT
+        {{ target.database }}.live.udf_api('https://api.wormholescan.io/api/v1/wormhole/assets/secured-tokens') restp,
+        SYSDATE() AS _inserted_timestamp
+),
+flat_worms AS (
+    SELECT
+        b.value :coingecko_id :: STRING AS cg,
+        b.value :symbol :: STRING AS symbol,
+        C.key :: STRING AS blockchain,
+        C.value :: STRING AS token_address,
+        _inserted_timestamp
+    FROM
+        worms,
+        LATERAL FLATTEN (
+            restp :data
+        ) b,
+        LATERAL FLATTEN(
+            b.value :platforms
+        ) C
+)
+SELECT
+    REPLACE(
+        REPLACE(
+            REPLACE(REPLACE(blockchain, 'binance-smart-chain', 'bsc'), 'arbitrum-one', 'arbitrum'),
+            'optimistic-ethereum',
+            'optimism'
+        ),
+        'polygon-pos',
+        'polygon'
+    ) AS blockchain,
+    token_address,
+    LOWER(token_address) AS token_address_lower,
+    symbol,
+    cg AS coingecko_id,
+    {{ dbt_utils.generate_surrogate_key(['blockchain','token_address']) }} AS wormhole_secured_tokens_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    flat_worms qualify ROW_NUMBER() over (
+        PARTITION BY blockchain,
+        token_address_lower
+        ORDER BY
+            --keep the case sensitive token address first
+            CASE
+                WHEN token_address_lower = token_address THEN 1
+                ELSE 0
+            END,
+            _inserted_timestamp DESC
+    ) = 1