yml docs, upd gha schedule

.github/workflows/dbt_run_livequery_weekly.yml

@@ -4,8 +4,8 @@ run-name: dbt_run_livequery
 on:
   workflow_dispatch:
   schedule:
-    # Runs hourly (see https://crontab.guru)
-    - cron: '0 * * * *'
+    # Runs weekly at 0 UTC on Mondays (see https://crontab.guru)
+    - cron: '0 0 * * 1'
 
 env:
   USE_VARS: "${{ vars.USE_VARS }}"
@@ -28,7 +28,7 @@ jobs:
     uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
     with:
       dbt_command: >
-        dbt run -s tag:livequery --vars '{"PAGODA_API_KEY": "${{ secrets.PAGODA_API_KEY }}", "SQL_LIMIT": ${{ vars.PAGODA_SQL_LIMIT }}}'
+        dbt run -s tag:nearblocks
       environment: workflow_prod
       warehouse: ${{ vars.WAREHOUSE }}
     secrets: inherit 

.github/workflows/dbt_run_nearblocks_api.yml

@@ -3,9 +3,10 @@ run-name: dbt_run_nearblocks_api
 
 on:
   workflow_dispatch:
-  schedule:
+  # Disabling schedule 
+  # schedule:
     # Runs 0100 UTC daily (see https://crontab.guru)
-    - cron: '0 1 * * *'
+    # - cron: '0 1 * * *'
 
 env:
   USE_VARS: "${{ vars.USE_VARS }}"

macros/api_udf/create_get_nearblocks_fts.sql

@@ -1,3 +1,4 @@
+{# Deprecated 9/25/2023 #}
 {% macro create_get_nearblocks_fts() %}
   {% set create_table %}
   CREATE schema if NOT EXISTS {{ target.database }}.bronze_api;

macros/api_udf/get_nearblocks_fts.sql

@@ -1,3 +1,5 @@
+{# Deprecated 9/25/2023 #}
+
 {% macro get_nearblocks_fts() %}
 {% set sql %}
 

models/descriptions/base_uri.md

@@ -0,0 +1,5 @@
+{% docs base_uri %}
+
+The URI associated with the token or NFT, set by the deployer or creator.
+
+{% enddocs %}

models/descriptions/contract_address.md

@@ -0,0 +1,5 @@
+{% docs contract_address %}
+
+The address of the deployed contract for the token, nft, or other smart contract.
+
+{% enddocs %}

models/descriptions/contract_metadata.md

@@ -0,0 +1,5 @@
+{% docs contract_metadata %}
+
+Token contract metadata from the Pagoda API. Should correspond with the metadata provided by the Nearblocks API.
+
+{% enddocs %}

models/descriptions/data.md

@@ -0,0 +1,5 @@
+{% docs data %}
+
+A JSON object representing the response from an API.
+
+{% enddocs %}

models/descriptions/icon.md

@@ -0,0 +1,5 @@
+{% docs icon %}
+
+A base64 encoded image for the token or NFT's icon.
+
+{% enddocs %}

models/descriptions/metadata_id.md

@@ -0,0 +1,5 @@
+{% docs metadata_id %}
+
+A MD5 Hash of an NFT contract and series ID, providing an unique indicator for collecting NFT Series Metadata.
+
+{% enddocs %}

models/descriptions/name.md

@@ -0,0 +1,5 @@
+{% docs name %}
+
+The name of the asset.
+
+{% enddocs %}

models/descriptions/series_id.md

@@ -0,0 +1,5 @@
+{% docs series_id %}
+
+The series, or collection, id for a NFT project. Many platforms issues NFT collections from a marketplace contract. For example, the contract `x.paras.near` is the Paras NFT Marketplace and mints NFTs from this master-contract. A token ID 1234:123 indicates that the NFT is token `123` within series `1234`.
+
+{% enddocs %}

models/descriptions/token_metadata.md

@@ -0,0 +1,5 @@
+{% docs token_metadata %}
+
+NFT token-level metadata for a NFT series / collection.
+
+{% enddocs %}

models/descriptions/tokens.md

@@ -0,0 +1,5 @@
+{% docs tokens %}
+
+Count of tokens issued by the NFT Contract.
+
+{% enddocs %}

models/gold/core/core__dim_token_labels.sql

@@ -15,6 +15,7 @@ WITH token_labels AS (
         {{ ref('silver__token_labels') }}
 ),
 nearblocks_fts_api AS (
+{# Deprecated 9/25/2023, TODO update this view to new token ingestion job #}
     SELECT
         token,
         symbol,

models/gold/core/core__fact_token_metadata.sql

@@ -9,6 +9,7 @@
     },
     tags = ['core']
 ) }}
+{# Deprecated 9/25/2023 TODO deprecate and delete in favor of new view #}
 
 WITH nearblocks_ft_api AS (
 

models/gold/core/core__fact_token_metadata.yml

@@ -3,7 +3,7 @@ version: 2
 models:
   - name: core__fact_token_metadata
     description: |-
-      Holds data ingested from the Nearblocks Fungible Token API endpoint at: https://api.nearblocks.io/api-docs/#/FTs/get_v1_fts. This data includes updating pieces, such as past 24h volume and transfers, thus differs from the dimensional token label table.
+      Deprecating soon in favor of a new dim table. Note the new table will not contain daily data. Holds data ingested from the Nearblocks Fungible Token API endpoint at: https://api.nearblocks.io/api-docs/#/FTs/get_v1_fts. This data includes updating pieces, such as past 24h volume and transfers, thus differs from the dimensional token label table.
 
     columns:
       - name: DATE

models/silver/labels/silver__api_nearblocks_fts.sql

@@ -6,6 +6,8 @@
     tags = ['api', 'labels']
 ) }}
 
+{# Deprecated 9/25/2023, TODO disable via config #}
+
 WITH nearblocks_token_api AS (
 
     SELECT

models/silver/labels/silver__ft_contract_metadata.sql

@@ -24,8 +24,6 @@ WHERE
 ),
 flatten_results AS (
     SELECT
-        page,
-        INDEX,
         VALUE :contract :: STRING AS contract_address,
         VALUE :decimals :: INT AS decimals,
         VALUE :icon :: STRING AS icon,
@@ -42,3 +40,4 @@ SELECT
     *
 FROM
     flatten_results
+qualify row_number() over (partition by contract_address order by _inserted_timestamp desc) = 1

models/silver/labels/silver__ft_contract_metadata.yml

@@ -0,0 +1,51 @@
+version: 2
+
+models:
+  - name: silver__ft_contract_metadata
+    description: |-
+      Fungible Token contract metadata provided by the Nearblocks NFT endpoint.
+    tests:
+      - dbt_utils.recency:
+          datepart: day
+          field: _inserted_timestamp
+          interval: 14
+
+    columns:
+      - name: CONTRACT_ADDRESS
+        description: "{{ doc('contract_address')}}"
+        tests:
+          - not_null
+          - unique
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: DECIMALS
+        description: "{{ doc('decimals')}}"
+        tests:
+          - not_null
+
+      - name: ICON
+        description: "{{ doc('icon')}}"
+
+      - name: NAME
+        description: "{{ doc('name')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: SYMBOL
+        description: "{{ doc('symbol')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: DATA
+        description: "{{ doc('data')}}"
+
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp')}}"
+
+      - name: _RES_ID
+        description: "{{ doc('_res_id')}}"

models/silver/labels/silver__nft_contract_metadata.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     unique_key = 'contract_address',
     incremental_strategy = 'delete+insert',
-    tags = ['livequery', 'nearblocks']
+    tags = ['livequery', 'nearblocks'],
 ) }}
 
 WITH livequery_results AS (
@@ -24,8 +24,6 @@ WHERE
 ),
 flatten_results AS (
     SELECT
-        page,
-        INDEX,
         VALUE :base_uri :: STRING AS base_uri,
         VALUE :contract :: STRING AS contract_address,
         VALUE :icon :: STRING AS icon,
@@ -43,3 +41,4 @@ SELECT
     *
 FROM
     flatten_results
+qualify row_number() over (partition by contract_address order by _inserted_timestamp desc) = 1

models/silver/labels/silver__nft_contract_metadata.yml

@@ -0,0 +1,52 @@
+version: 2
+
+models:
+  - name: silver__nft_contract_metadata
+    description: |-
+      NFT Contract-level metadata provided by the Nearblocks NFT endpoint.
+    tests:
+      - dbt_utils.recency:
+          datepart: day
+          field: _inserted_timestamp
+          interval: 7
+
+    columns:
+      - name: BASE_URI
+        description: "{{ doc('base_uri')}}"
+
+      - name: CONTRACT_ADDRESS
+        description: "{{ doc('contract_address')}}"
+        tests:
+          - not_null
+          - unique
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: ICON
+        description: "{{ doc('icon')}}"
+
+      - name: NAME
+        description: "{{ doc('name')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: SYMBOL
+        description: "{{ doc('symbol')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: TOKENS
+        description: "{{ doc('tokens')}}"
+
+      - name: DATA
+        description: "{{ doc('data')}}"
+
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp')}}"
+
+      - name: _RES_ID
+        description: "{{ doc('_res_id')}}"

models/silver/labels/silver__nft_series_metadata.sql

@@ -15,9 +15,9 @@ WITH livequery_response AS (
         call_succeeded
 
 {% if is_incremental() %}
-AND _inserted_timestamp >= (
+AND _request_timestamp >= (
     SELECT
-        MAX(_inserted_timestamp)
+        MAX(_request_timestamp)
     FROM
         {{ this }}
 )
@@ -29,7 +29,9 @@ FINAL AS (
         series_id,
         metadata_id,
         lq_response :data :contract_metadata :: variant AS contract_metadata,
-        lq_response :data :nft :metadata :: variant AS token_metadata
+        lq_response :data :nft :metadata :: variant AS token_metadata,
+        _inserted_timestamp,
+        _request_timestamp
     FROM
         livequery_response
 )

models/silver/labels/silver__nft_series_metadata.yml

@@ -0,0 +1,44 @@
+version: 2
+
+models:
+  - name: silver__nft_series_metadata
+    description: |-
+      NFT Series-level metadata provided by the Pagoda NFT endpoint.
+    tests:
+      - dbt_utils.recency:
+          datepart: day
+          field: _inserted_timestamp
+          interval: 5
+
+    columns:
+      - name: CONTRACT_ADDRESS
+        description: "{{ doc('contract_address')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: SERIES_ID
+        description: "{{ doc('series_id')}}"
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: METADATA_ID
+        description: "{{ doc('metadata_id')}}"
+        tests:
+          - not_null
+          - unique
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: VARCHAR
+
+      - name: CONTRACT_METADATA
+        description: "{{ doc('contract_metadata')}}"
+
+      - name: TOKEN_METADATA
+        description: "{{ doc('token_metadata')}}"
+
+      - name: _INSERTED_TIMESTAMP
+        description: "{{ doc('_inserted_timestamp')}}"
+