Migrate solscan token metadata to solscan v2 (#717)

* get solscan token list using v2 api

* bronze models to retrieve token list data

* get token list once per week

* fix workflow name

.github/workflows/dbt_run_streamline_solscan_token_list.yml

@@ -0,0 +1,46 @@
+name: dbt_run_streamline_solscan_token_list
+run-name: dbt_run_streamline_solscan_token_list
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs 01:15 every Sunday (see https://crontab.guru)
+    - cron: '15 1 * * 0'
+    
+env:
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ vars.PYTHON_VERSION }}"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -s "solana_models,streamline__solscan_token_list_by_market_cap" --vars '{"STREAMLINE_INVOKE_STREAMS": True}'
+

models/bronze/streamline/bronze__streamline_FR_solscan_token_list.sql

@@ -0,0 +1,12 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+{% set model = "solscan_token_list" %}
+{{ streamline_external_table_FR_query(
+    model,
+    partition_function = "to_date(split_part(split_part(file_name, '/', -2), '_result', 1), 'YYYY_MM_DD')",
+    partition_name = "_partition_by_created_date",
+    unique_key = "",
+    other_cols = ""
+) }}

models/bronze/streamline/bronze__streamline_solscan_token_list.sql

@@ -0,0 +1,12 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+{% set model = "solscan_token_list" %}
+{{ streamline_external_table_query(
+    model,
+    partition_function = "to_date(split_part(split_part(file_name, '/', -2), '_result', 1), 'YYYY_MM_DD')",
+    partition_name = "_partition_by_created_date",
+    unique_key = "",
+    other_cols = ""
+) }}

models/sources.yml

@@ -64,6 +64,7 @@ sources:
       - name: validator_metadata_2
       - name: validators_list_2
       - name: solscan_blocks_2
+      - name: solscan_token_list
   - name: bronze_api
     schema: bronze_api
     tables:

models/streamline/token/metadata/streamline__solscan_token_list_by_market_cap.sql

@@ -0,0 +1,38 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = 'streamline.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params ={ 
+            "external_table" :"solscan_token_list",
+            "sql_limit" :"250",
+            "producer_batch_size" :"250",
+            "worker_batch_size" :"250",
+            "sql_source" :"{{this.identifier}}",
+        }
+    )
+) }}
+
+WITH page_numbers AS (
+    SELECT
+        SEQ4()+1 AS page_number
+    FROM
+        TABLE(GENERATOR(rowcount => 250))
+)
+SELECT
+    page_number,
+    replace(current_date::string,'-','_') AS partition_key, -- Issue with streamline handling `-` in partition key so changing to `_`
+    live.udf_api(
+        'GET',
+        concat('{Service}/token/list?sort_by=market_cap&sort_order=desc&page_size=40&page=',page_number),
+        object_construct(
+            'Content-Type',
+            'application/json',
+            'token',
+            '{Authentication}'
+        ),
+        {},
+        'Vault/prod/solana/solscan/v2'
+    ) AS request
+FROM
+    page_numbers