diff --git a/.github/workflows/dbt_run_livequery.yml b/.github/workflows/dbt_run_livequery_weekly.yml similarity index 79% rename from .github/workflows/dbt_run_livequery.yml rename to .github/workflows/dbt_run_livequery_weekly.yml index e207b69..8c09aaa 100644 --- a/.github/workflows/dbt_run_livequery.yml +++ b/.github/workflows/dbt_run_livequery_weekly.yml @@ -4,8 +4,8 @@ run-name: dbt_run_livequery on: workflow_dispatch: schedule: - # Runs hourly (see https://crontab.guru) - - cron: '0 * * * *' + # Runs weekly at 0 UTC on Mondays (see https://crontab.guru) + - cron: '0 0 * * 1' env: USE_VARS: "${{ vars.USE_VARS }}" @@ -28,7 +28,7 @@ jobs: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main with: dbt_command: > - dbt run -s tag:livequery --vars '{"PAGODA_API_KEY": "${{ secrets.PAGODA_API_KEY }}", "SQL_LIMIT": ${{ vars.PAGODA_SQL_LIMIT }}}' + dbt run -s tag:nearblocks environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/.github/workflows/dbt_run_nearblocks_api.yml b/.github/workflows/dbt_run_nearblocks_api.yml index ec21876..b865806 100644 --- a/.github/workflows/dbt_run_nearblocks_api.yml +++ b/.github/workflows/dbt_run_nearblocks_api.yml @@ -3,9 +3,10 @@ run-name: dbt_run_nearblocks_api on: workflow_dispatch: - schedule: + # Disabling schedule + # schedule: # Runs 0100 UTC daily (see https://crontab.guru) - - cron: '0 1 * * *' + # - cron: '0 1 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/macros/api_udf/create_get_nearblocks_fts.sql b/macros/api_udf/create_get_nearblocks_fts.sql index 32f7806..54a741c 100644 --- a/macros/api_udf/create_get_nearblocks_fts.sql +++ b/macros/api_udf/create_get_nearblocks_fts.sql @@ -1,3 +1,4 @@ +{# Deprecated 9/25/2023 #} {% macro create_get_nearblocks_fts() %} {% set create_table %} CREATE schema if NOT EXISTS {{ target.database }}.bronze_api; diff --git a/macros/api_udf/get_nearblocks_fts.sql b/macros/api_udf/get_nearblocks_fts.sql index 7832a79..2913601 100644 --- a/macros/api_udf/get_nearblocks_fts.sql +++ b/macros/api_udf/get_nearblocks_fts.sql @@ -1,3 +1,5 @@ +{# Deprecated 9/25/2023 #} + {% macro get_nearblocks_fts() %} {% set sql %} diff --git a/models/descriptions/base_uri.md b/models/descriptions/base_uri.md new file mode 100644 index 0000000..76045fd --- /dev/null +++ b/models/descriptions/base_uri.md @@ -0,0 +1,5 @@ +{% docs base_uri %} + +The URI associated with the token or NFT, set by the deployer or creator. + +{% enddocs %} diff --git a/models/descriptions/contract_address.md b/models/descriptions/contract_address.md new file mode 100644 index 0000000..564f611 --- /dev/null +++ b/models/descriptions/contract_address.md @@ -0,0 +1,5 @@ +{% docs contract_address %} + +The address of the deployed contract for the token, nft, or other smart contract. + +{% enddocs %} diff --git a/models/descriptions/contract_metadata.md b/models/descriptions/contract_metadata.md new file mode 100644 index 0000000..c71f277 --- /dev/null +++ b/models/descriptions/contract_metadata.md @@ -0,0 +1,5 @@ +{% docs contract_metadata %} + +Token contract metadata from the Pagoda API. Should correspond with the metadata provided by the Nearblocks API. + +{% enddocs %} diff --git a/models/descriptions/data.md b/models/descriptions/data.md new file mode 100644 index 0000000..c9afd91 --- /dev/null +++ b/models/descriptions/data.md @@ -0,0 +1,5 @@ +{% docs data %} + +A JSON object representing the response from an API. + +{% enddocs %} diff --git a/models/descriptions/icon.md b/models/descriptions/icon.md new file mode 100644 index 0000000..39124dd --- /dev/null +++ b/models/descriptions/icon.md @@ -0,0 +1,5 @@ +{% docs icon %} + +A base64 encoded image for the token or NFT's icon. + +{% enddocs %} diff --git a/models/descriptions/metadata_id.md b/models/descriptions/metadata_id.md new file mode 100644 index 0000000..861d1a8 --- /dev/null +++ b/models/descriptions/metadata_id.md @@ -0,0 +1,5 @@ +{% docs metadata_id %} + +A MD5 Hash of an NFT contract and series ID, providing an unique indicator for collecting NFT Series Metadata. + +{% enddocs %} diff --git a/models/descriptions/name.md b/models/descriptions/name.md new file mode 100644 index 0000000..c6e1725 --- /dev/null +++ b/models/descriptions/name.md @@ -0,0 +1,5 @@ +{% docs name %} + +The name of the asset. + +{% enddocs %} diff --git a/models/descriptions/series_id.md b/models/descriptions/series_id.md new file mode 100644 index 0000000..43c9a83 --- /dev/null +++ b/models/descriptions/series_id.md @@ -0,0 +1,5 @@ +{% docs series_id %} + +The series, or collection, id for a NFT project. Many platforms issues NFT collections from a marketplace contract. For example, the contract `x.paras.near` is the Paras NFT Marketplace and mints NFTs from this master-contract. A token ID 1234:123 indicates that the NFT is token `123` within series `1234`. + +{% enddocs %} diff --git a/models/descriptions/token_metadata.md b/models/descriptions/token_metadata.md new file mode 100644 index 0000000..a5f8005 --- /dev/null +++ b/models/descriptions/token_metadata.md @@ -0,0 +1,5 @@ +{% docs token_metadata %} + +NFT token-level metadata for a NFT series / collection. + +{% enddocs %} diff --git a/models/descriptions/tokens.md b/models/descriptions/tokens.md new file mode 100644 index 0000000..3244235 --- /dev/null +++ b/models/descriptions/tokens.md @@ -0,0 +1,5 @@ +{% docs tokens %} + +Count of tokens issued by the NFT Contract. + +{% enddocs %} diff --git a/models/gold/core/core__dim_token_labels.sql b/models/gold/core/core__dim_token_labels.sql index 7374896..e047e93 100644 --- a/models/gold/core/core__dim_token_labels.sql +++ b/models/gold/core/core__dim_token_labels.sql @@ -15,6 +15,7 @@ WITH token_labels AS ( {{ ref('silver__token_labels') }} ), nearblocks_fts_api AS ( +{# Deprecated 9/25/2023, TODO update this view to new token ingestion job #} SELECT token, symbol, diff --git a/models/gold/core/core__fact_token_metadata.sql b/models/gold/core/core__fact_token_metadata.sql index 50685c0..9405e4b 100644 --- a/models/gold/core/core__fact_token_metadata.sql +++ b/models/gold/core/core__fact_token_metadata.sql @@ -9,6 +9,7 @@ }, tags = ['core'] ) }} +{# Deprecated 9/25/2023 TODO deprecate and delete in favor of new view #} WITH nearblocks_ft_api AS ( diff --git a/models/gold/core/core__fact_token_metadata.yml b/models/gold/core/core__fact_token_metadata.yml index 21cc2bd..ff69859 100644 --- a/models/gold/core/core__fact_token_metadata.yml +++ b/models/gold/core/core__fact_token_metadata.yml @@ -3,7 +3,7 @@ version: 2 models: - name: core__fact_token_metadata description: |- - Holds data ingested from the Nearblocks Fungible Token API endpoint at: https://api.nearblocks.io/api-docs/#/FTs/get_v1_fts. This data includes updating pieces, such as past 24h volume and transfers, thus differs from the dimensional token label table. + Deprecating soon in favor of a new dim table. Note the new table will not contain daily data. Holds data ingested from the Nearblocks Fungible Token API endpoint at: https://api.nearblocks.io/api-docs/#/FTs/get_v1_fts. This data includes updating pieces, such as past 24h volume and transfers, thus differs from the dimensional token label table. columns: - name: DATE diff --git a/models/silver/labels/silver__api_nearblocks_fts.sql b/models/silver/labels/silver__api_nearblocks_fts.sql index 20e1c96..5804d2c 100644 --- a/models/silver/labels/silver__api_nearblocks_fts.sql +++ b/models/silver/labels/silver__api_nearblocks_fts.sql @@ -6,6 +6,8 @@ tags = ['api', 'labels'] ) }} +{# Deprecated 9/25/2023, TODO disable via config #} + WITH nearblocks_token_api AS ( SELECT diff --git a/models/silver/labels/silver__ft_contract_metadata.sql b/models/silver/labels/silver__ft_contract_metadata.sql index c39b6dc..0270474 100644 --- a/models/silver/labels/silver__ft_contract_metadata.sql +++ b/models/silver/labels/silver__ft_contract_metadata.sql @@ -24,8 +24,6 @@ WHERE ), flatten_results AS ( SELECT - page, - INDEX, VALUE :contract :: STRING AS contract_address, VALUE :decimals :: INT AS decimals, VALUE :icon :: STRING AS icon, @@ -42,3 +40,4 @@ SELECT * FROM flatten_results +qualify row_number() over (partition by contract_address order by _inserted_timestamp desc) = 1 diff --git a/models/silver/labels/silver__ft_contract_metadata.yml b/models/silver/labels/silver__ft_contract_metadata.yml new file mode 100644 index 0000000..ad52fb5 --- /dev/null +++ b/models/silver/labels/silver__ft_contract_metadata.yml @@ -0,0 +1,51 @@ +version: 2 + +models: + - name: silver__ft_contract_metadata + description: |- + Fungible Token contract metadata provided by the Nearblocks NFT endpoint. + tests: + - dbt_utils.recency: + datepart: day + field: _inserted_timestamp + interval: 14 + + columns: + - name: CONTRACT_ADDRESS + description: "{{ doc('contract_address')}}" + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: DECIMALS + description: "{{ doc('decimals')}}" + tests: + - not_null + + - name: ICON + description: "{{ doc('icon')}}" + + - name: NAME + description: "{{ doc('name')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: SYMBOL + description: "{{ doc('symbol')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: DATA + description: "{{ doc('data')}}" + + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp')}}" + + - name: _RES_ID + description: "{{ doc('_res_id')}}" diff --git a/models/silver/labels/silver__nft_contract_metadata.sql b/models/silver/labels/silver__nft_contract_metadata.sql index ac1d247..2174c2a 100644 --- a/models/silver/labels/silver__nft_contract_metadata.sql +++ b/models/silver/labels/silver__nft_contract_metadata.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'contract_address', incremental_strategy = 'delete+insert', - tags = ['livequery', 'nearblocks'] + tags = ['livequery', 'nearblocks'], ) }} WITH livequery_results AS ( @@ -24,8 +24,6 @@ WHERE ), flatten_results AS ( SELECT - page, - INDEX, VALUE :base_uri :: STRING AS base_uri, VALUE :contract :: STRING AS contract_address, VALUE :icon :: STRING AS icon, @@ -43,3 +41,4 @@ SELECT * FROM flatten_results +qualify row_number() over (partition by contract_address order by _inserted_timestamp desc) = 1 diff --git a/models/silver/labels/silver__nft_contract_metadata.yml b/models/silver/labels/silver__nft_contract_metadata.yml new file mode 100644 index 0000000..9b45d81 --- /dev/null +++ b/models/silver/labels/silver__nft_contract_metadata.yml @@ -0,0 +1,52 @@ +version: 2 + +models: + - name: silver__nft_contract_metadata + description: |- + NFT Contract-level metadata provided by the Nearblocks NFT endpoint. + tests: + - dbt_utils.recency: + datepart: day + field: _inserted_timestamp + interval: 7 + + columns: + - name: BASE_URI + description: "{{ doc('base_uri')}}" + + - name: CONTRACT_ADDRESS + description: "{{ doc('contract_address')}}" + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: ICON + description: "{{ doc('icon')}}" + + - name: NAME + description: "{{ doc('name')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: SYMBOL + description: "{{ doc('symbol')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: TOKENS + description: "{{ doc('tokens')}}" + + - name: DATA + description: "{{ doc('data')}}" + + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp')}}" + + - name: _RES_ID + description: "{{ doc('_res_id')}}" diff --git a/models/silver/labels/silver__nft_series_metadata.sql b/models/silver/labels/silver__nft_series_metadata.sql index 91d8b11..9c4cda8 100644 --- a/models/silver/labels/silver__nft_series_metadata.sql +++ b/models/silver/labels/silver__nft_series_metadata.sql @@ -15,9 +15,9 @@ WITH livequery_response AS ( call_succeeded {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND _request_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(_request_timestamp) FROM {{ this }} ) @@ -29,7 +29,9 @@ FINAL AS ( series_id, metadata_id, lq_response :data :contract_metadata :: variant AS contract_metadata, - lq_response :data :nft :metadata :: variant AS token_metadata + lq_response :data :nft :metadata :: variant AS token_metadata, + _inserted_timestamp, + _request_timestamp FROM livequery_response ) diff --git a/models/silver/labels/silver__nft_series_metadata.yml b/models/silver/labels/silver__nft_series_metadata.yml new file mode 100644 index 0000000..f814bbc --- /dev/null +++ b/models/silver/labels/silver__nft_series_metadata.yml @@ -0,0 +1,44 @@ +version: 2 + +models: + - name: silver__nft_series_metadata + description: |- + NFT Series-level metadata provided by the Pagoda NFT endpoint. + tests: + - dbt_utils.recency: + datepart: day + field: _inserted_timestamp + interval: 5 + + columns: + - name: CONTRACT_ADDRESS + description: "{{ doc('contract_address')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: SERIES_ID + description: "{{ doc('series_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: METADATA_ID + description: "{{ doc('metadata_id')}}" + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: VARCHAR + + - name: CONTRACT_METADATA + description: "{{ doc('contract_metadata')}}" + + - name: TOKEN_METADATA + description: "{{ doc('token_metadata')}}" + + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp')}}" +