From dd4bcc2d32f13ec989d30a2a4bb44ffb4d8a3290 Mon Sep 17 00:00:00 2001 From: eric-laurello <102970824+eric-laurello@users.noreply.github.com> Date: Thu, 19 Jan 2023 13:08:52 -0500 Subject: [PATCH] tags (#94) --- README.md | 46 ++++++- data/bronze__balance_addresses_everyday.csv | 8 +- dbt_project.yml | 6 +- macros/tags/add_database_or_schema_tags.sql | 10 ++ macros/tags/snowflake_tagging.sql | 127 ++++++++++++++++++ models/core/core__dim_prices.sql | 9 +- models/core/core__dim_vote_options.sql | 9 +- models/core/core__ez_prices.sql | 9 +- models/core/core__fact_daily_balances.sql | 9 +- ...ore__fact_governance_proposal_deposits.sql | 9 +- .../core__fact_governance_submit_proposal.sql | 9 +- models/core/core__fact_governance_votes.sql | 9 +- models/core/core__fact_staking.sql | 9 +- models/core/core__fact_staking_rewards.sql | 9 +- models/core/core__fact_superfluid_staking.sql | 9 +- models/core/core__fact_swaps.sql | 9 +- 16 files changed, 282 insertions(+), 14 deletions(-) create mode 100644 macros/tags/add_database_or_schema_tags.sql create mode 100644 macros/tags/snowflake_tagging.sql diff --git a/README.md b/README.md index eea473a..5ec9918 100644 --- a/README.md +++ b/README.md @@ -27,4 +27,48 @@ osmosis: - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers - Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support - Find [dbt events](https://events.getdbt.com) near you -- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices \ No newline at end of file +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +## Applying Model Tags + +### Database / Schema level tags + +Database and schema tags are applied via the `add_database_or_schema_tags` macro. These tags are inherited by their downstream objects. To add/modify tags call the appropriate tag set function within the macro. + +``` +{{ set_database_tag_value('SOME_DATABASE_TAG_KEY','SOME_DATABASE_TAG_VALUE') }} +{{ set_schema_tag_value('SOME_SCHEMA_TAG_KEY','SOME_SCHEMA_TAG_VALUE') }} +``` + +### Model tags + +To add/update a model's snowflake tags, add/modify the `meta` model property under `config`. Only table level tags are supported at this time via DBT. + +``` +{{ config( + ..., + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'SOME_PURPOSE' + } + } + }, + ... +) }} +``` + +By default, model tags are not pushed to snowflake on each load. You can push a tag update for a model by specifying the `UPDATE_SNOWFLAKE_TAGS` project variable during a run. + +``` +dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.sql +``` + +### Querying for existing tags on a model in snowflake + +``` +select * +from table(solana.information_schema.tag_references('solana.core.fact_blocks', 'table')); +``` \ No newline at end of file diff --git a/data/bronze__balance_addresses_everyday.csv b/data/bronze__balance_addresses_everyday.csv index 5ccf3d0..d1df9a9 100644 --- a/data/bronze__balance_addresses_everyday.csv +++ b/data/bronze__balance_addresses_everyday.csv @@ -12,4 +12,10 @@ osmo1m3h30wlvsf8llruxtpukdvsy0km2kum8q25g3j,module-mint osmo1m5dncvfv7lvpvycr23zja93fecun2kcv226glq,module-claim osmo1upfuxznarpja3sywq0tzd2kktg9wv8mcc0rlm9,module-poolincentives osmo17xpfvakm2amg962yls6f84z3kell8c5lczssa0,module-fee_collector -osmo1g003ql3feflq94wyfkqc50z74d4cxvfz3p5nmy,flipside \ No newline at end of file +osmo1g003ql3feflq94wyfkqc50z74d4cxvfz3p5nmy,flipside +osmo14mqa0k55xt256q335p8zu0zcvvv7ecr20zq3wk,flipside +osmo1cvmw32achlrv8fh0ffyhsyuu4f8lwpzshnt8wc,flipside +osmo13qaek239lqhvwecdtmfg5jdhjh7c76eprurw6a,flipside +osmo1a3hz6zc7zqdh2kg5dsxua0sqvz6twnmf2pw56x,flipside +osmo1pec5kx4xlq4rmqpxfffsrvhc53483atv2wj0rf,flipside +osmo1pqas8pt4ychqtvk0zx5r8tpe0wepldrhxmkhfc,flipside \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index ed15124..fb1f178 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -39,6 +39,7 @@ models: vars: "dbt_date:time_zone": GMT + "UPDATE_SNOWFLAKE_TAGS": TRUE tests: +store_failures: true # all tests @@ -49,4 +50,7 @@ on-run-start: - '{{sp_bulk_get_asset_metadata()}}' - '{{sp_create_bulk_get_balances()}}' - '{{sp_create_bulk_get_validator_metadata()}}' - - '{{sp_create_bulk_get_pool_balances()}}' \ No newline at end of file + - '{{sp_create_bulk_get_pool_balances()}}' + +on-run-end: + - '{{ apply_meta_as_tags(results) }}' \ No newline at end of file diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql new file mode 100644 index 0000000..f548bf6 --- /dev/null +++ b/macros/tags/add_database_or_schema_tags.sql @@ -0,0 +1,10 @@ +{% macro add_database_or_schema_tags() %} + {{ set_database_tag_value( + 'BLOCKCHAIN_NAME', + 'OSMOSIS' + ) }} + {{ set_database_tag_value( + 'BLOCKCHAIN_TYPE', + 'IBC' + ) }} +{% endmacro %} diff --git a/macros/tags/snowflake_tagging.sql b/macros/tags/snowflake_tagging.sql new file mode 100644 index 0000000..bc25e69 --- /dev/null +++ b/macros/tags/snowflake_tagging.sql @@ -0,0 +1,127 @@ +{% macro apply_meta_as_tags(results) %} + {% if var("UPDATE_SNOWFLAKE_TAGS") %} + {{ log('apply_meta_as_tags', info=False) }} + {{ log(results, info=False) }} + {% if execute %} + + {%- set tags_by_schema = {} -%} + {% for res in results -%} + {% if res.node.meta.database_tags %} + + {%- set model_database = res.node.database -%} + {%- set model_schema = res.node.schema -%} + {%- set model_schema_full = model_database+'.'+model_schema -%} + {%- set model_alias = res.node.alias -%} + + {% if model_schema_full not in tags_by_schema.keys() %} + {{ log('need to fetch tags for schema '+model_schema_full, info=False) }} + {%- call statement('main', fetch_result=True) -%} + show tags in {{model_database}}.{{model_schema}} + {%- endcall -%} + {%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%} + {{ log('Added tags to cache', info=False) }} + {% else %} + {{ log('already have tag info for schema', info=False) }} + {% endif %} + + {%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%} + {{ log('current_tags_in_schema:', info=False) }} + {{ log(current_tags_in_schema, info=False) }} + {{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }} + + {% set line -%} + node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }}) + node full: {{ res.node}} + meta: {{ res.node.meta}} + materialized: {{ res.node.config.materialized }} + {%- endset %} + {{ log(line, info=False) }} + + {%- call statement('main', fetch_result=True) -%} + select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table')) + {%- endcall -%} + {%- set existing_tags_for_table = load_result('main')['data'] -%} + {{ log('Existing tags for table:', info=False) }} + {{ log(existing_tags_for_table, info=False) }} + + {{ log('--', info=False) }} + {% for table_tag in res.node.meta.database_tags.table %} + + {{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }} + {% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %} + + {{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}} + {% endfor %} + {{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} +{% endmacro %} + + +{% macro create_tag_if_missing(all_tag_names,table_tag) %} + {% if table_tag not in all_tag_names %} + {{ log('Creating missing tag '+table_tag, info=False) }} + {%- call statement('main', fetch_result=True) -%} + create tag if not exists silver.{{table_tag}} + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% else %} + {{ log('Tag already exists: '+table_tag, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }} + {%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for table:', info=False) }} + {{ log(existing_tag_for_table[0], info=False) }} + {% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }} + {%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for column:', info=False) }} + {{ log(existing_tag_for_column[0], info=False) }} + {% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_database_tag_value(tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} + +{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} \ No newline at end of file diff --git a/models/core/core__dim_prices.sql b/models/core/core__dim_prices.sql index 387e88f..b8f58b5 100644 --- a/models/core/core__dim_prices.sql +++ b/models/core/core__dim_prices.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'PRICES' + } + } + } ) }} SELECT diff --git a/models/core/core__dim_vote_options.sql b/models/core/core__dim_vote_options.sql index fe12221..38fd3af 100644 --- a/models/core/core__dim_vote_options.sql +++ b/models/core/core__dim_vote_options.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'GOVERNANCE' + } + } + } ) }} SELECT diff --git a/models/core/core__ez_prices.sql b/models/core/core__ez_prices.sql index 7fd95ed..e245483 100644 --- a/models/core/core__ez_prices.sql +++ b/models/core/core__ez_prices.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'PRICES' + } + } + } ) }} WITH p_base AS ( diff --git a/models/core/core__fact_daily_balances.sql b/models/core/core__fact_daily_balances.sql index 637cec8..b0bff94 100644 --- a/models/core/core__fact_daily_balances.sql +++ b/models/core/core__fact_daily_balances.sql @@ -2,7 +2,14 @@ materialized = 'incremental', unique_key = "CONCAT_WS('-', date, address, balance_type, currency)", incremental_strategy = 'delete+insert', - cluster_by = ['date'] + cluster_by = ['date'], + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'BALANCES' + } + } + } ) }} SELECT diff --git a/models/core/core__fact_governance_proposal_deposits.sql b/models/core/core__fact_governance_proposal_deposits.sql index 1895dc3..de0e4f1 100644 --- a/models/core/core__fact_governance_proposal_deposits.sql +++ b/models/core/core__fact_governance_proposal_deposits.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'GOVERNANCE' + } + } + } ) }} SELECT diff --git a/models/core/core__fact_governance_submit_proposal.sql b/models/core/core__fact_governance_submit_proposal.sql index ba94841..122d221 100644 --- a/models/core/core__fact_governance_submit_proposal.sql +++ b/models/core/core__fact_governance_submit_proposal.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'GOVERNANCE' + } + } + } ) }} SELECT diff --git a/models/core/core__fact_governance_votes.sql b/models/core/core__fact_governance_votes.sql index a407a72..6eba3dd 100644 --- a/models/core/core__fact_governance_votes.sql +++ b/models/core/core__fact_governance_votes.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'GOVERNANCE' + } + } + } ) }} SELECT diff --git a/models/core/core__fact_staking.sql b/models/core/core__fact_staking.sql index a0c7f54..1deb649 100644 --- a/models/core/core__fact_staking.sql +++ b/models/core/core__fact_staking.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING' + } + } + } ) }} SELECT diff --git a/models/core/core__fact_staking_rewards.sql b/models/core/core__fact_staking_rewards.sql index ed1a467..3351ce7 100644 --- a/models/core/core__fact_staking_rewards.sql +++ b/models/core/core__fact_staking_rewards.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING' + } + } + } ) }} SELECT diff --git a/models/core/core__fact_superfluid_staking.sql b/models/core/core__fact_superfluid_staking.sql index 06fefc0..655d8b0 100644 --- a/models/core/core__fact_superfluid_staking.sql +++ b/models/core/core__fact_superfluid_staking.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'STAKING' + } + } + } ) }} SELECT diff --git a/models/core/core__fact_swaps.sql b/models/core/core__fact_swaps.sql index 0601aef..619c882 100644 --- a/models/core/core__fact_swaps.sql +++ b/models/core/core__fact_swaps.sql @@ -1,5 +1,12 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'SWAPS' + } + } + } ) }} SELECT