tags (#94)

2026-02-06 09:26:49 +00:00 · 2023-01-19 13:08:52 -05:00 · 2023-01-19 13:08:52 -05:00 · dd4bcc2d32
commit dd4bcc2d32
parent 843df60c6f
16 changed files with 282 additions and 14 deletions
--- a/README.md
+++ b/README.md
@ -27,4 +27,48 @@ osmosis:
 - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
 - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
 - Find [dbt events](https://events.getdbt.com) near you
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
+
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
+
+## Applying Model Tags
+
+### Database / Schema level tags
+
+Database and schema tags are applied via the `add_database_or_schema_tags` macro.  These tags are inherited by their downstream objects.  To add/modify tags call the appropriate tag set function within the macro.
+
+```
+{{ set_database_tag_value('SOME_DATABASE_TAG_KEY','SOME_DATABASE_TAG_VALUE') }}
+{{ set_schema_tag_value('SOME_SCHEMA_TAG_KEY','SOME_SCHEMA_TAG_VALUE') }}
+```
+
+### Model tags
+
+To add/update a model's snowflake tags, add/modify the `meta` model property under `config`.  Only table level tags are supported at this time via DBT.
+
+```
+{{ config(
+    ...,
+    meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'SOME_PURPOSE'
+            }
+        }
+    },
+    ...
+) }}
+```
+
+By default, model tags are not pushed to snowflake on each load.  You can push a tag update for a model by specifying the `UPDATE_SNOWFLAKE_TAGS` project variable during a run.
+
+```
+dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.sql
+```
+
+### Querying for existing tags on a model in snowflake
+
+```
+select *
+from table(solana.information_schema.tag_references('solana.core.fact_blocks', 'table'));
+```
--- a/data/bronze__balance_addresses_everyday.csv
+++ b/data/bronze__balance_addresses_everyday.csv
@ -12,4 +12,10 @@ osmo1m3h30wlvsf8llruxtpukdvsy0km2kum8q25g3j,module-mint
 osmo1m5dncvfv7lvpvycr23zja93fecun2kcv226glq,module-claim
 osmo1upfuxznarpja3sywq0tzd2kktg9wv8mcc0rlm9,module-poolincentives
 osmo17xpfvakm2amg962yls6f84z3kell8c5lczssa0,module-fee_collector
-osmo1g003ql3feflq94wyfkqc50z74d4cxvfz3p5nmy,flipside
+osmo1g003ql3feflq94wyfkqc50z74d4cxvfz3p5nmy,flipside
+osmo14mqa0k55xt256q335p8zu0zcvvv7ecr20zq3wk,flipside
+osmo1cvmw32achlrv8fh0ffyhsyuu4f8lwpzshnt8wc,flipside
+osmo13qaek239lqhvwecdtmfg5jdhjh7c76eprurw6a,flipside
+osmo1a3hz6zc7zqdh2kg5dsxua0sqvz6twnmf2pw56x,flipside
+osmo1pec5kx4xlq4rmqpxfffsrvhc53483atv2wj0rf,flipside
+osmo1pqas8pt4ychqtvk0zx5r8tpe0wepldrhxmkhfc,flipside
--- a/dbt_project.yml
+++ b/dbt_project.yml
@ -39,6 +39,7 @@ models:

 vars:
  "dbt_date:time_zone": GMT
+  "UPDATE_SNOWFLAKE_TAGS": TRUE

 tests:
  +store_failures: true # all tests
@ -49,4 +50,7 @@ on-run-start:
  - '{{sp_bulk_get_asset_metadata()}}'
  - '{{sp_create_bulk_get_balances()}}'
  - '{{sp_create_bulk_get_validator_metadata()}}'
-  - '{{sp_create_bulk_get_pool_balances()}}'
+  - '{{sp_create_bulk_get_pool_balances()}}'
+
+on-run-end:
+  - '{{ apply_meta_as_tags(results) }}'
--- a/macros/tags/add_database_or_schema_tags.sql
+++ b/macros/tags/add_database_or_schema_tags.sql
@ -0,0 +1,10 @@
+{% macro add_database_or_schema_tags() %}
+    {{ set_database_tag_value(
+        'BLOCKCHAIN_NAME',
+        'OSMOSIS'
+    ) }}
+      {{ set_database_tag_value(
+        'BLOCKCHAIN_TYPE',
+        'IBC'
+    ) }}
+{% endmacro %}
--- a/macros/tags/snowflake_tagging.sql
+++ b/macros/tags/snowflake_tagging.sql
@ -0,0 +1,127 @@
+{% macro apply_meta_as_tags(results) %}
+    {% if var("UPDATE_SNOWFLAKE_TAGS") %}
+        {{ log('apply_meta_as_tags', info=False) }}
+        {{ log(results, info=False) }}
+        {% if execute %}
+
+            {%- set tags_by_schema = {} -%}
+            {% for res in results -%}
+                {% if res.node.meta.database_tags %}
+
+                    {%- set model_database = res.node.database -%}
+                    {%- set model_schema = res.node.schema -%}
+                    {%- set model_schema_full = model_database+'.'+model_schema -%}
+                    {%- set model_alias = res.node.alias -%}
+
+                    {% if model_schema_full not in tags_by_schema.keys() %}
+                        {{ log('need to fetch tags for schema '+model_schema_full, info=False) }}
+                        {%- call statement('main', fetch_result=True) -%}
+                            show tags in {{model_database}}.{{model_schema}}
+                        {%- endcall -%}
+                        {%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%}
+                        {{ log('Added tags to cache', info=False) }}
+                    {% else %}
+                        {{ log('already have tag info for schema', info=False) }}
+                    {% endif %}
+
+                    {%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%}
+                    {{ log('current_tags_in_schema:', info=False) }}
+                    {{ log(current_tags_in_schema, info=False) }}
+                    {{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }}
+
+                    {% set line -%}
+                        node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }})
+                        node full: {{ res.node}}
+                        meta: {{ res.node.meta}}
+                        materialized: {{ res.node.config.materialized }}
+                    {%- endset %}
+                    {{ log(line, info=False) }}
+
+                    {%- call statement('main', fetch_result=True) -%}
+                        select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table'))
+                    {%- endcall -%}
+                    {%- set existing_tags_for_table = load_result('main')['data'] -%}
+                    {{ log('Existing tags for table:', info=False) }}
+                    {{ log(existing_tags_for_table, info=False) }}
+
+                    {{ log('--', info=False) }}
+                    {% for table_tag in res.node.meta.database_tags.table %}
+
+                        {{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }}
+                        {% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %}
+
+                        {{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}}
+                    {% endfor %}
+                    {{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }}
+                {% endif %}
+            {% endfor %}
+        {% endif %}
+    {% endif %}
+{% endmacro %}
+
+
+{% macro create_tag_if_missing(all_tag_names,table_tag) %}
+	{% if table_tag not in all_tag_names %}
+		{{ log('Creating missing tag '+table_tag, info=False) }}
+        {%- call statement('main', fetch_result=True) -%}
+            create tag if not exists silver.{{table_tag}}
+        {%- endcall -%}
+		{{ log(load_result('main').data, info=False) }}
+	{% else %}
+		{{ log('Tag already exists: '+table_tag, info=False) }}
+	{% endif %}
+{% endmacro %}
+
+{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %}
+    {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }}
+    {%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%}
+    {{ log('Filtered tags for table:', info=False) }}
+    {{ log(existing_tag_for_table[0], info=False) }}
+    {% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %}
+        {{ log('Correct tag value already exists', info=False) }}
+    {% else %}
+        {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }}
+        {%- call statement('main', fetch_result=True) -%}
+            alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}'
+        {%- endcall -%}
+        {{ log(load_result('main').data, info=False) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %}
+    {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }}
+    {%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%}
+    {{ log('Filtered tags for column:', info=False) }}
+    {{ log(existing_tag_for_column[0], info=False) }}
+    {% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %}
+        {{ log('Correct tag value already exists', info=False) }}
+    {% else %}
+        {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }}
+        {%- call statement('main', fetch_result=True) -%}
+            alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}'
+        {%- endcall -%}
+        {{ log(load_result('main').data, info=False) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro set_database_tag_value(tag_name,tag_value) %}
+    {% set query %}
+        create tag if not exists silver.{{tag_name}}
+    {% endset %}
+    {% do run_query(query) %}
+    {% set query %}
+        alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}'
+    {% endset %}
+    {% do run_query(query) %}
+{% endmacro %}
+
+{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %}
+    {% set query %}
+        create tag if not exists silver.{{tag_name}}
+    {% endset %}
+    {% do run_query(query) %}
+    {% set query %}
+        alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}'
+    {% endset %}
+    {% do run_query(query) %}
+{% endmacro %}
--- a/models/core/core__dim_prices.sql
+++ b/models/core/core__dim_prices.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'PRICES'
+            }
+        }
+      }
 ) }}

 SELECT
--- a/models/core/core__dim_vote_options.sql
+++ b/models/core/core__dim_vote_options.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'GOVERNANCE'
+            }
+        }
+      }
 ) }}

 SELECT 
--- a/models/core/core__ez_prices.sql
+++ b/models/core/core__ez_prices.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'PRICES'
+            }
+        }
+      }
 ) }}

 WITH p_base AS (
--- a/models/core/core__fact_daily_balances.sql
+++ b/models/core/core__fact_daily_balances.sql
@ -2,7 +2,14 @@
    materialized = 'incremental',
    unique_key = "CONCAT_WS('-', date, address, balance_type, currency)",
    incremental_strategy = 'delete+insert',
-    cluster_by = ['date']
+    cluster_by = ['date'],
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'BALANCES'
+            }
+        }
+      }
 ) }}

 SELECT
--- a/models/core/core__fact_governance_proposal_deposits.sql
+++ b/models/core/core__fact_governance_proposal_deposits.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'GOVERNANCE'
+            }
+        }
+      }
 ) }}

 SELECT 
--- a/models/core/core__fact_governance_submit_proposal.sql
+++ b/models/core/core__fact_governance_submit_proposal.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'GOVERNANCE'
+            }
+        }
+      }
 ) }}

 SELECT 
--- a/models/core/core__fact_governance_votes.sql
+++ b/models/core/core__fact_governance_votes.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'GOVERNANCE'
+            }
+        }
+      }
 ) }}

 SELECT
--- a/models/core/core__fact_staking.sql
+++ b/models/core/core__fact_staking.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'STAKING'
+            }
+        }
+      }
 ) }}

 SELECT 
--- a/models/core/core__fact_staking_rewards.sql
+++ b/models/core/core__fact_staking_rewards.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'STAKING'
+            }
+        }
+      }
 ) }}

 SELECT
--- a/models/core/core__fact_superfluid_staking.sql
+++ b/models/core/core__fact_superfluid_staking.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'STAKING'
+            }
+        }
+      }
 ) }}

 SELECT
--- a/models/core/core__fact_swaps.sql
+++ b/models/core/core__fact_swaps.sql
@ -1,5 +1,12 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'view',
+      meta={
+        'database_tags':{
+            'table': {
+                'PURPOSE': 'SWAPS'
+            }
+        }
+      }
 ) }}

 SELECT