From 06dc0c4e35260516f311ab734facb305690ae6b5 Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Mon, 23 Jan 2023 10:33:29 -0700 Subject: [PATCH] AN-2699/snowflake-tags-ethereum (#357) * added macros and updates to dbt_project/readme for snowflake tags * added database level tag * more tags * more tags * changed readme * added tags for chainlink and snapshot * more tags --- README.md | 42 ++++++ dbt_project.yml | 4 + macros/tags/add_database_or_schema_tags.sql | 4 + macros/tags/snowflake_tagging.sql | 127 ++++++++++++++++++ ...sushi__dim_distributor_reward_schedule.sql | 8 ++ models/Sushi/sushi__ez_borrowing.sql | 10 +- models/Sushi/sushi__ez_lending.sql | 10 +- models/Sushi/sushi__ez_swaps.sql | 10 +- models/aave/gold/aave__ez_borrows.sql | 10 +- models/aave/gold/aave__ez_deposits.sql | 10 +- models/aave/gold/aave__ez_flashloans.sql | 10 +- models/aave/gold/aave__ez_liquidations.sql | 10 +- models/aave/gold/aave__ez_market_stats.sql | 10 +- models/aave/gold/aave__ez_proposals.sql | 10 +- models/aave/gold/aave__ez_repayments.sql | 10 +- models/aave/gold/aave__ez_votes.sql | 10 +- models/aave/gold/aave__ez_withdraws.sql | 10 +- .../gold/chainlink__dim_oracle_feeds.sql | 10 +- .../gold/chainlink__ez_oracle_feeds.sql | 10 +- .../gold/chainlink__fact_oracle_feeds.sql | 10 +- .../gold/compound__ez_asset_details.sql | 10 +- models/compound/gold/compound__ez_borrows.sql | 10 +- .../compound/gold/compound__ez_deposits.sql | 10 +- .../gold/compound__ez_liquidations.sql | 10 +- .../gold/compound__ez_market_stats.sql | 10 +- .../gold/compound__ez_redemptions.sql | 10 +- .../compound/gold/compound__ez_repayments.sql | 10 +- models/gold/core__dim_nft_metadata.sql | 9 +- models/gold/core__ez_balance_deltas.sql | 9 +- models/gold/core__ez_current_balances.sql | 9 +- models/gold/core__ez_nft_mints.sql | 9 +- models/gold/core__ez_nft_sales.sql | 9 +- models/gold/core__ez_nft_transfers.sql | 9 +- models/gold/core__ez_snapshot.sql | 10 +- models/gold/core__fact_eth_balances.sql | 9 +- models/gold/core__fact_token_balances.sql | 9 +- models/gold/dex/core__ez_dex_swaps.sql | 10 +- models/gold/maker/maker__ez_delegations.sql | 10 +- models/gold/maker/maker__ez_deposits.sql | 10 +- models/gold/maker/maker__ez_flash_loans.sql | 10 +- .../gold/maker/maker__ez_governance_votes.sql | 10 +- models/gold/maker/maker__ez_liquidations.sql | 10 +- models/gold/maker/maker__ez_repayments.sql | 10 +- .../gold/maker/maker__ez_vault_creation.sql | 10 +- models/gold/maker/maker__ez_withdrawals.sql | 10 +- .../gold/uniswapv3__ez_lp_actions.sql | 10 +- .../gold/uniswapv3__ez_pool_stats.sql | 10 +- models/uniswapv3/gold/uniswapv3__ez_pools.sql | 10 +- .../uniswapv3__ez_position_collected_fees.sql | 10 +- .../gold/uniswapv3__ez_positions.sql | 10 +- models/uniswapv3/gold/uniswapv3__ez_swaps.sql | 10 +- 51 files changed, 591 insertions(+), 46 deletions(-) create mode 100644 macros/tags/add_database_or_schema_tags.sql create mode 100644 macros/tags/snowflake_tagging.sql diff --git a/README.md b/README.md index b1edbfd9..18a80dec 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,45 @@ dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ... * Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support * Find [dbt events](https://events.getdbt.com) near you * Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +## Applying Model Tags + +### Database / Schema level tags + +Database and schema tags are applied via the `add_database_or_schema_tags` macro. These tags are inherited by their downstream objects. To add/modify tags call the appropriate tag set function within the macro. + +``` +{{ set_database_tag_value('SOME_DATABASE_TAG_KEY','SOME_DATABASE_TAG_VALUE') }} +{{ set_schema_tag_value('SOME_SCHEMA_TAG_KEY','SOME_SCHEMA_TAG_VALUE') }} +``` + +### Model tags + +To add/update a model's snowflake tags, add/modify the `meta` model property under `config`. Only table level tags are supported at this time via DBT. + +``` +{{ config( + ..., + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'SOME_PURPOSE' + } + } + }, + ... +) }} +``` + +By default, model tags are pushed to Snowflake on each load. You can disable this by setting the `UPDATE_SNOWFLAKE_TAGS` project variable to `False` during a run. + +``` +dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks.sql +``` + +### Querying for existing tags on a model in snowflake + +``` +select * +from table(ethereum.information_schema.tag_references('ethereum.core.fact_blocks', 'table')); +``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 6acf49c1..5d5b3ad0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -37,6 +37,9 @@ on-run-start: - "{{ sp_create_cross_db_share_clones() }}" - "{{ create_udfs() }}" +on-run-end: + - '{{ apply_meta_as_tags(results) }}' + # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models @@ -49,3 +52,4 @@ vars: STREAMLINE_INVOKE_STREAMS: False STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False UPDATE_UDFS_AND_SPS: False + UPDATE_SNOWFLAKE_TAGS: True diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql new file mode 100644 index 00000000..ca3c1ac0 --- /dev/null +++ b/macros/tags/add_database_or_schema_tags.sql @@ -0,0 +1,4 @@ +{% macro add_database_or_schema_tags() %} + {{ set_database_tag_value('BLOCKCHAIN_NAME','ETHEREUM') }} + {{ set_database_tag_value('BLOCKCHAIN_TYPE','EVM') }} +{% endmacro %} \ No newline at end of file diff --git a/macros/tags/snowflake_tagging.sql b/macros/tags/snowflake_tagging.sql new file mode 100644 index 00000000..bc25e693 --- /dev/null +++ b/macros/tags/snowflake_tagging.sql @@ -0,0 +1,127 @@ +{% macro apply_meta_as_tags(results) %} + {% if var("UPDATE_SNOWFLAKE_TAGS") %} + {{ log('apply_meta_as_tags', info=False) }} + {{ log(results, info=False) }} + {% if execute %} + + {%- set tags_by_schema = {} -%} + {% for res in results -%} + {% if res.node.meta.database_tags %} + + {%- set model_database = res.node.database -%} + {%- set model_schema = res.node.schema -%} + {%- set model_schema_full = model_database+'.'+model_schema -%} + {%- set model_alias = res.node.alias -%} + + {% if model_schema_full not in tags_by_schema.keys() %} + {{ log('need to fetch tags for schema '+model_schema_full, info=False) }} + {%- call statement('main', fetch_result=True) -%} + show tags in {{model_database}}.{{model_schema}} + {%- endcall -%} + {%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%} + {{ log('Added tags to cache', info=False) }} + {% else %} + {{ log('already have tag info for schema', info=False) }} + {% endif %} + + {%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%} + {{ log('current_tags_in_schema:', info=False) }} + {{ log(current_tags_in_schema, info=False) }} + {{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }} + + {% set line -%} + node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }}) + node full: {{ res.node}} + meta: {{ res.node.meta}} + materialized: {{ res.node.config.materialized }} + {%- endset %} + {{ log(line, info=False) }} + + {%- call statement('main', fetch_result=True) -%} + select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table')) + {%- endcall -%} + {%- set existing_tags_for_table = load_result('main')['data'] -%} + {{ log('Existing tags for table:', info=False) }} + {{ log(existing_tags_for_table, info=False) }} + + {{ log('--', info=False) }} + {% for table_tag in res.node.meta.database_tags.table %} + + {{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }} + {% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %} + + {{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}} + {% endfor %} + {{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} +{% endmacro %} + + +{% macro create_tag_if_missing(all_tag_names,table_tag) %} + {% if table_tag not in all_tag_names %} + {{ log('Creating missing tag '+table_tag, info=False) }} + {%- call statement('main', fetch_result=True) -%} + create tag if not exists silver.{{table_tag}} + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% else %} + {{ log('Tag already exists: '+table_tag, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }} + {%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for table:', info=False) }} + {{ log(existing_tag_for_table[0], info=False) }} + {% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }} + {%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for column:', info=False) }} + {{ log(existing_tag_for_column[0], info=False) }} + {% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_database_tag_value(tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} + +{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} \ No newline at end of file diff --git a/models/Sushi/sushi__dim_distributor_reward_schedule.sql b/models/Sushi/sushi__dim_distributor_reward_schedule.sql index 27454a8b..d32d3073 100644 --- a/models/Sushi/sushi__dim_distributor_reward_schedule.sql +++ b/models/Sushi/sushi__dim_distributor_reward_schedule.sql @@ -1,5 +1,13 @@ {{ config( materialized = 'table', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'SUSHI', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} select merkle_root, diff --git a/models/Sushi/sushi__ez_borrowing.sql b/models/Sushi/sushi__ez_borrowing.sql index 17b096ac..dd354e94 100644 --- a/models/Sushi/sushi__ez_borrowing.sql +++ b/models/Sushi/sushi__ez_borrowing.sql @@ -4,7 +4,15 @@ persist_docs ={ "relation": true, "columns": true }, unique_key = '_log_id', - cluster_by = ['block_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'SUSHI', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} WITH borrow_txns AS ( diff --git a/models/Sushi/sushi__ez_lending.sql b/models/Sushi/sushi__ez_lending.sql index 7644bccc..e7c8007a 100644 --- a/models/Sushi/sushi__ez_lending.sql +++ b/models/Sushi/sushi__ez_lending.sql @@ -4,7 +4,15 @@ persist_docs ={ "relation": true, "columns": true }, unique_key = '_log_id', - cluster_by = ['block_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'SUSHI', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} WITH lending_txns AS ( diff --git a/models/Sushi/sushi__ez_swaps.sql b/models/Sushi/sushi__ez_swaps.sql index 5c5e9794..770cc83a 100644 --- a/models/Sushi/sushi__ez_swaps.sql +++ b/models/Sushi/sushi__ez_swaps.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'SUSHI', + 'PURPOSE': 'DEFI, DEX, SWAPS' + } + } + } ) }} SELECT diff --git a/models/aave/gold/aave__ez_borrows.sql b/models/aave/gold/aave__ez_borrows.sql index c2d00c33..c5e0d6f5 100644 --- a/models/aave/gold/aave__ez_borrows.sql +++ b/models/aave/gold/aave__ez_borrows.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = "_log_id", cluster_by = ['block_timestamp::DATE'], - tags = ['snowflake', 'ethereum', 'aave', 'aave_borrows', 'address_labels'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_borrows', 'address_labels'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH --borrows from Aave LendingPool contracts diff --git a/models/aave/gold/aave__ez_deposits.sql b/models/aave/gold/aave__ez_deposits.sql index 87d978c5..8c84c032 100644 --- a/models/aave/gold/aave__ez_deposits.sql +++ b/models/aave/gold/aave__ez_deposits.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = "_log_id", cluster_by = ['block_timestamp::DATE'], - tags = ['snowflake', 'ethereum', 'aave', 'aave_deposits', 'address_labels'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_deposits', 'address_labels'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH deposits AS( diff --git a/models/aave/gold/aave__ez_flashloans.sql b/models/aave/gold/aave__ez_flashloans.sql index 900e0792..dd36e806 100644 --- a/models/aave/gold/aave__ez_flashloans.sql +++ b/models/aave/gold/aave__ez_flashloans.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = "_log_id", cluster_by = ['block_timestamp::DATE'], - tags = ['snowflake', 'ethereum', 'aave', 'aave_flashloans', 'address_labels'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_flashloans', 'address_labels'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH flashloan AS ( diff --git a/models/aave/gold/aave__ez_liquidations.sql b/models/aave/gold/aave__ez_liquidations.sql index 642e3e9b..cdfdb826 100644 --- a/models/aave/gold/aave__ez_liquidations.sql +++ b/models/aave/gold/aave__ez_liquidations.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = "_log_id", cluster_by = ['block_timestamp::DATE'], - tags = ['snowflake', 'ethereum', 'aave', 'aave_liquidations', 'address_labels'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_liquidations', 'address_labels'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH liquidation AS( diff --git a/models/aave/gold/aave__ez_market_stats.sql b/models/aave/gold/aave__ez_market_stats.sql index 631d13a0..11bed157 100644 --- a/models/aave/gold/aave__ez_market_stats.sql +++ b/models/aave/gold/aave__ez_market_stats.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = "read_id", incremental_strategy = 'delete+insert', - tags = ['snowflake', 'ethereum', 'aave', 'aave_market_stats', 'address_labels'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_market_stats', 'address_labels'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH blocks AS ( diff --git a/models/aave/gold/aave__ez_proposals.sql b/models/aave/gold/aave__ez_proposals.sql index aab607c2..92f9f722 100644 --- a/models/aave/gold/aave__ez_proposals.sql +++ b/models/aave/gold/aave__ez_proposals.sql @@ -4,7 +4,15 @@ sort='block_number', unique_key= "CONCAT_WS('-', proposal_tx, proposal_id)", incremental_strategy='delete+insert', - tags=['snowflake', 'ethereum', 'aave', 'aave_proposals'] + tags=['snowflake', 'ethereum', 'aave', 'aave_proposals'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} diff --git a/models/aave/gold/aave__ez_repayments.sql b/models/aave/gold/aave__ez_repayments.sql index d76a79c0..5fab5cfa 100644 --- a/models/aave/gold/aave__ez_repayments.sql +++ b/models/aave/gold/aave__ez_repayments.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = "_log_id", cluster_by = ['block_timestamp::DATE'], - tags = ['snowflake', 'ethereum', 'aave', 'aave_repayments', 'address_labels'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_repayments', 'address_labels'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH repay AS( diff --git a/models/aave/gold/aave__ez_votes.sql b/models/aave/gold/aave__ez_votes.sql index fe624b61..e33b46d7 100644 --- a/models/aave/gold/aave__ez_votes.sql +++ b/models/aave/gold/aave__ez_votes.sql @@ -3,7 +3,15 @@ sort = 'block_number', unique_key = "_log_id", incremental_strategy = 'delete+insert', - tags = ['snowflake', 'ethereum', 'aave', 'aave_votes'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_votes'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH base AS ( diff --git a/models/aave/gold/aave__ez_withdraws.sql b/models/aave/gold/aave__ez_withdraws.sql index 037a8e27..cffa549d 100644 --- a/models/aave/gold/aave__ez_withdraws.sql +++ b/models/aave/gold/aave__ez_withdraws.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = "_log_id", cluster_by = ['block_timestamp::DATE'], - tags = ['snowflake', 'ethereum', 'aave', 'aave_withdraws', 'address_labels'] + tags = ['snowflake', 'ethereum', 'aave', 'aave_withdraws', 'address_labels'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'AAVE', + 'PURPOSE': 'DEFI' + } + } + } ) }} WITH withdraw AS( diff --git a/models/chainlink/gold/chainlink__dim_oracle_feeds.sql b/models/chainlink/gold/chainlink__dim_oracle_feeds.sql index fa39cd5f..7bed5122 100644 --- a/models/chainlink/gold/chainlink__dim_oracle_feeds.sql +++ b/models/chainlink/gold/chainlink__dim_oracle_feeds.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'CHAINLINK', + 'PURPOSE': 'ORACLE' + } + } + } ) }} SELECT diff --git a/models/chainlink/gold/chainlink__ez_oracle_feeds.sql b/models/chainlink/gold/chainlink__ez_oracle_feeds.sql index e75656cd..d9e12605 100644 --- a/models/chainlink/gold/chainlink__ez_oracle_feeds.sql +++ b/models/chainlink/gold/chainlink__ez_oracle_feeds.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'CHAINLINK', + 'PURPOSE': 'ORACLE' + } + } + } ) }} SELECT diff --git a/models/chainlink/gold/chainlink__fact_oracle_feeds.sql b/models/chainlink/gold/chainlink__fact_oracle_feeds.sql index 67d1b5f1..875b35f5 100644 --- a/models/chainlink/gold/chainlink__fact_oracle_feeds.sql +++ b/models/chainlink/gold/chainlink__fact_oracle_feeds.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'CHAINLINK', + 'PURPOSE': 'ORACLE' + } + } + } ) }} SELECT diff --git a/models/compound/gold/compound__ez_asset_details.sql b/models/compound/gold/compound__ez_asset_details.sql index 15fb0428..84de18bf 100644 --- a/models/compound/gold/compound__ez_asset_details.sql +++ b/models/compound/gold/compound__ez_asset_details.sql @@ -2,7 +2,15 @@ materialized = 'view', persist_docs ={ "relation": true, "columns": true }, - tags = ['compound'] + tags = ['compound'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'COMPOUND', + 'PURPOSE': 'DEFI' + } + } + } ) }} -- Pulls contract details for relevant c assets. The case when handles cETH. WITH base AS ( diff --git a/models/compound/gold/compound__ez_borrows.sql b/models/compound/gold/compound__ez_borrows.sql index 8a5dd1d8..187fe730 100644 --- a/models/compound/gold/compound__ez_borrows.sql +++ b/models/compound/gold/compound__ez_borrows.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = '_log_id', cluster_by = ['block_timestamp::DATE'], - tags = ['compound'] + tags = ['compound'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'COMPOUND', + 'PURPOSE': 'DEFI' + } + } + } ) }} -- pull all ctoken addresses and corresponding name WITH asset_details AS ( diff --git a/models/compound/gold/compound__ez_deposits.sql b/models/compound/gold/compound__ez_deposits.sql index 549e28f9..8494840c 100644 --- a/models/compound/gold/compound__ez_deposits.sql +++ b/models/compound/gold/compound__ez_deposits.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = '_log_id', cluster_by = ['block_timestamp::DATE'], - tags = ['compound'] + tags = ['compound'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'COMPOUND', + 'PURPOSE': 'DEFI' + } + } + } ) }} -- pull all ctoken addresses and corresponding name WITH asset_details AS ( diff --git a/models/compound/gold/compound__ez_liquidations.sql b/models/compound/gold/compound__ez_liquidations.sql index 84fa80f1..6cba30c3 100644 --- a/models/compound/gold/compound__ez_liquidations.sql +++ b/models/compound/gold/compound__ez_liquidations.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = '_log_id', cluster_by = ['block_timestamp::DATE'], - tags = ['compound'] + tags = ['compound'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'COMPOUND', + 'PURPOSE': 'DEFI' + } + } + } ) }} -- pull all ctoken addresses and corresponding name -- add the collateral liquidated here diff --git a/models/compound/gold/compound__ez_market_stats.sql b/models/compound/gold/compound__ez_market_stats.sql index 4efe20c9..d0801f79 100644 --- a/models/compound/gold/compound__ez_market_stats.sql +++ b/models/compound/gold/compound__ez_market_stats.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'COMPOUND', + 'PURPOSE': 'DEFI' + } + } + } ) }} -- pull all ctoken addresses and corresponding name diff --git a/models/compound/gold/compound__ez_redemptions.sql b/models/compound/gold/compound__ez_redemptions.sql index 9c9a2fd2..c0a94876 100644 --- a/models/compound/gold/compound__ez_redemptions.sql +++ b/models/compound/gold/compound__ez_redemptions.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = '_log_id', cluster_by = ['block_timestamp::DATE'], - tags = ['compound'] + tags = ['compound'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'COMPOUND', + 'PURPOSE': 'DEFI' + } + } + } ) }} -- pull all ctoken addresses and corresponding name WITH asset_details AS ( diff --git a/models/compound/gold/compound__ez_repayments.sql b/models/compound/gold/compound__ez_repayments.sql index ac04d130..1a627b7d 100644 --- a/models/compound/gold/compound__ez_repayments.sql +++ b/models/compound/gold/compound__ez_repayments.sql @@ -2,7 +2,15 @@ materialized = 'incremental', unique_key = '_log_id', cluster_by = ['block_timestamp::DATE'], - tags = ['compound'] + tags = ['compound'], + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'COMPOUND', + 'PURPOSE': 'DEFI' + } + } + } ) }} -- pull all ctoken addresses and corresponding name WITH asset_details AS ( diff --git a/models/gold/core__dim_nft_metadata.sql b/models/gold/core__dim_nft_metadata.sql index f03145aa..fc32ee91 100644 --- a/models/gold/core__dim_nft_metadata.sql +++ b/models/gold/core__dim_nft_metadata.sql @@ -1,7 +1,14 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'NFT' + } + } + } ) }} SELECT diff --git a/models/gold/core__ez_balance_deltas.sql b/models/gold/core__ez_balance_deltas.sql index 4c81df8a..b09d0fc7 100644 --- a/models/gold/core__ez_balance_deltas.sql +++ b/models/gold/core__ez_balance_deltas.sql @@ -2,7 +2,14 @@ materialized = 'view', tags = ['balances'], persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'BALANCES' + } + } + } ) }} WITH prices AS ( diff --git a/models/gold/core__ez_current_balances.sql b/models/gold/core__ez_current_balances.sql index d02705ab..85f4271e 100644 --- a/models/gold/core__ez_current_balances.sql +++ b/models/gold/core__ez_current_balances.sql @@ -2,7 +2,14 @@ materialized = 'view', tags = ['balances'], persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'BALANCES' + } + } + } ) }} WITH prices AS ( diff --git a/models/gold/core__ez_nft_mints.sql b/models/gold/core__ez_nft_mints.sql index b587ff1c..4d53006a 100644 --- a/models/gold/core__ez_nft_mints.sql +++ b/models/gold/core__ez_nft_mints.sql @@ -4,7 +4,14 @@ "columns": true }, unique_key = '_log_id', cluster_by = ['block_timestamp::DATE'], - post_hook = "{{ grant_data_share_statement('EZ_NFT_MINTS', 'TABLE') }}" + post_hook = "{{ grant_data_share_statement('EZ_NFT_MINTS', 'TABLE') }}", + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'NFT' + } + } + } ) }} WITH nft_mints AS ( diff --git a/models/gold/core__ez_nft_sales.sql b/models/gold/core__ez_nft_sales.sql index 0d8324ef..56515d72 100644 --- a/models/gold/core__ez_nft_sales.sql +++ b/models/gold/core__ez_nft_sales.sql @@ -1,7 +1,14 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'NFT' + } + } + } ) }} SELECT diff --git a/models/gold/core__ez_nft_transfers.sql b/models/gold/core__ez_nft_transfers.sql index e9f39bf3..f0fc0ea6 100644 --- a/models/gold/core__ez_nft_transfers.sql +++ b/models/gold/core__ez_nft_transfers.sql @@ -2,7 +2,14 @@ materialized = 'view', persist_docs ={ "relation": true, "columns": true }, - tags = ['core'] + tags = ['core'], + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'NFT' + } + } + } ) }} SELECT diff --git a/models/gold/core__ez_snapshot.sql b/models/gold/core__ez_snapshot.sql index d3abcc7f..34482aa5 100644 --- a/models/gold/core__ez_snapshot.sql +++ b/models/gold/core__ez_snapshot.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'SNAPSHOT', + 'PURPOSE': 'GOVERNANCE' + } + } + } ) }} SELECT diff --git a/models/gold/core__fact_eth_balances.sql b/models/gold/core__fact_eth_balances.sql index 99ce7ed8..ff604d25 100644 --- a/models/gold/core__fact_eth_balances.sql +++ b/models/gold/core__fact_eth_balances.sql @@ -1,7 +1,14 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'BALANCES' + } + } + } ) }} SELECT diff --git a/models/gold/core__fact_token_balances.sql b/models/gold/core__fact_token_balances.sql index 83b2b3d3..86dfe902 100644 --- a/models/gold/core__fact_token_balances.sql +++ b/models/gold/core__fact_token_balances.sql @@ -1,7 +1,14 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'BALANCES' + } + } + } ) }} SELECT diff --git a/models/gold/dex/core__ez_dex_swaps.sql b/models/gold/dex/core__ez_dex_swaps.sql index cfe26ba2..012dbe30 100644 --- a/models/gold/dex/core__ez_dex_swaps.sql +++ b/models/gold/dex/core__ez_dex_swaps.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'SUSHI, UNISWAP, CURVE, SYNTHETIX, BALANCER' + 'PURPOSE': 'DEX, SWAPS' + } + } + } ) }} WITH v2_swaps AS ( diff --git a/models/gold/maker/maker__ez_delegations.sql b/models/gold/maker/maker__ez_delegations.sql index eeae60ac..570b2416 100644 --- a/models/gold/maker/maker__ez_delegations.sql +++ b/models/gold/maker/maker__ez_delegations.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'GOVERNANCE, DEFI' + } + } + } ) }} SELECT diff --git a/models/gold/maker/maker__ez_deposits.sql b/models/gold/maker/maker__ez_deposits.sql index a9921f9e..b009ca93 100644 --- a/models/gold/maker/maker__ez_deposits.sql +++ b/models/gold/maker/maker__ez_deposits.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'DEFI' + } + } + } ) }} SELECT diff --git a/models/gold/maker/maker__ez_flash_loans.sql b/models/gold/maker/maker__ez_flash_loans.sql index 8b957429..d182edff 100644 --- a/models/gold/maker/maker__ez_flash_loans.sql +++ b/models/gold/maker/maker__ez_flash_loans.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'DEFI' + } + } + } ) }} SELECT diff --git a/models/gold/maker/maker__ez_governance_votes.sql b/models/gold/maker/maker__ez_governance_votes.sql index d6ccc8b5..6927c2f2 100644 --- a/models/gold/maker/maker__ez_governance_votes.sql +++ b/models/gold/maker/maker__ez_governance_votes.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'GOVERNANCE, DEFI' + } + } + } ) }} SELECT diff --git a/models/gold/maker/maker__ez_liquidations.sql b/models/gold/maker/maker__ez_liquidations.sql index fd6ff928..dd68ada8 100644 --- a/models/gold/maker/maker__ez_liquidations.sql +++ b/models/gold/maker/maker__ez_liquidations.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'DEFI' + } + } + } ) }} SELECT diff --git a/models/gold/maker/maker__ez_repayments.sql b/models/gold/maker/maker__ez_repayments.sql index 5b7dc521..0b3df373 100644 --- a/models/gold/maker/maker__ez_repayments.sql +++ b/models/gold/maker/maker__ez_repayments.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'DEFI' + } + } + } ) }} SELECT diff --git a/models/gold/maker/maker__ez_vault_creation.sql b/models/gold/maker/maker__ez_vault_creation.sql index 1bbb8417..a0afb707 100644 --- a/models/gold/maker/maker__ez_vault_creation.sql +++ b/models/gold/maker/maker__ez_vault_creation.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'DEFI' + } + } + } ) }} SELECT diff --git a/models/gold/maker/maker__ez_withdrawals.sql b/models/gold/maker/maker__ez_withdrawals.sql index 973bd687..38e63146 100644 --- a/models/gold/maker/maker__ez_withdrawals.sql +++ b/models/gold/maker/maker__ez_withdrawals.sql @@ -1,5 +1,13 @@ {{ config( - materialized = 'view' + materialized = 'view', + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'MAKER, MKR', + 'PURPOSE': 'DEFI' + } + } + } ) }} SELECT diff --git a/models/uniswapv3/gold/uniswapv3__ez_lp_actions.sql b/models/uniswapv3/gold/uniswapv3__ez_lp_actions.sql index bce6ae01..b6e4fac1 100644 --- a/models/uniswapv3/gold/uniswapv3__ez_lp_actions.sql +++ b/models/uniswapv3/gold/uniswapv3__ez_lp_actions.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'UNISWAPV3', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} SELECT diff --git a/models/uniswapv3/gold/uniswapv3__ez_pool_stats.sql b/models/uniswapv3/gold/uniswapv3__ez_pool_stats.sql index 9d3c6017..2d679c17 100644 --- a/models/uniswapv3/gold/uniswapv3__ez_pool_stats.sql +++ b/models/uniswapv3/gold/uniswapv3__ez_pool_stats.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'UNISWAPV3', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} SELECT diff --git a/models/uniswapv3/gold/uniswapv3__ez_pools.sql b/models/uniswapv3/gold/uniswapv3__ez_pools.sql index e33c90b3..903162f5 100644 --- a/models/uniswapv3/gold/uniswapv3__ez_pools.sql +++ b/models/uniswapv3/gold/uniswapv3__ez_pools.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'UNISWAPV3', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} SELECT diff --git a/models/uniswapv3/gold/uniswapv3__ez_position_collected_fees.sql b/models/uniswapv3/gold/uniswapv3__ez_position_collected_fees.sql index 29c18c8a..a6931226 100644 --- a/models/uniswapv3/gold/uniswapv3__ez_position_collected_fees.sql +++ b/models/uniswapv3/gold/uniswapv3__ez_position_collected_fees.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'UNISWAPV3', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} SELECT diff --git a/models/uniswapv3/gold/uniswapv3__ez_positions.sql b/models/uniswapv3/gold/uniswapv3__ez_positions.sql index 08d1202e..cac0e41c 100644 --- a/models/uniswapv3/gold/uniswapv3__ez_positions.sql +++ b/models/uniswapv3/gold/uniswapv3__ez_positions.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'UNISWAPV3', + 'PURPOSE': 'DEFI, DEX' + } + } + } ) }} SELECT diff --git a/models/uniswapv3/gold/uniswapv3__ez_swaps.sql b/models/uniswapv3/gold/uniswapv3__ez_swaps.sql index c3ed2ea0..f2f4178e 100644 --- a/models/uniswapv3/gold/uniswapv3__ez_swaps.sql +++ b/models/uniswapv3/gold/uniswapv3__ez_swaps.sql @@ -1,7 +1,15 @@ {{ config( materialized = 'view', persist_docs ={ "relation": true, - "columns": true } + "columns": true }, + meta={ + 'database_tags':{ + 'table': { + 'PROTOCOL': 'UNISWAPV3', + 'PURPOSE': 'DEFI, DEX, SWAPS' + } + } + } ) }} SELECT