From 8e55f678c2b129a30d51d50ce0d64d2a5904ff07 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Mon, 20 Feb 2023 13:05:59 -0800 Subject: [PATCH] - create utility macros for creating udfs (#2) --- .gitignore | 4 +- .sqlfluff | 5 ++ .sqlfluffignore | 6 ++ README.md | 27 ++++-- ...st_create_or_drop_function_from_config.sql | 17 ++++ cspell.yml | 7 ++ dbt_project.yml | 14 +++- macros/create_udfs.sql | 34 ++++++-- macros/streamline/api_integrations.sql | 11 +++ macros/streamline/configs.yaml.sql | 81 ++++++++++++++++++ macros/streamline/streamline_udfs.sql | 22 +++++ macros/streamline/utils.sql | 83 +++++++++++++++++++ packages.yml | 4 +- 13 files changed, 290 insertions(+), 25 deletions(-) create mode 100644 .sqlfluff create mode 100644 .sqlfluffignore create mode 100644 analysis/test_create_or_drop_function_from_config.sql create mode 100644 cspell.yml create mode 100644 macros/streamline/api_integrations.sql create mode 100644 macros/streamline/configs.yaml.sql create mode 100644 macros/streamline/streamline_udfs.sql create mode 100644 macros/streamline/utils.sql diff --git a/.gitignore b/.gitignore index ea7e08f..ad7b857 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ logs/ .history/ **/.DS_Store .vscode/ -dbt-env/ \ No newline at end of file +dbt-env/ +.env +.* \ No newline at end of file diff --git a/.sqlfluff b/.sqlfluff new file mode 100644 index 0000000..4144426 --- /dev/null +++ b/.sqlfluff @@ -0,0 +1,5 @@ +[sqlfluff] +templater = dbt + +[sqlfluff:templater:jinja] +apply_dbt_builtins = True \ No newline at end of file diff --git a/.sqlfluffignore b/.sqlfluffignore new file mode 100644 index 0000000..1c93f7b --- /dev/null +++ b/.sqlfluffignore @@ -0,0 +1,6 @@ +target/ +# dbt <1.0.0 +dbt_modules/ +# dbt >=1.0.0 +dbt_packages/ +macros/ \ No newline at end of file diff --git a/README.md b/README.md index e662995..51b91f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ ## Profile Set Up -#### Use the following within profiles.yml +#### Use the following within profiles.yml + ---- ```yml @@ -45,14 +46,22 @@ When False, none of the on-run-start macros are executed on model run Default values are False * Usage: -dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ... +dbt run --var 'UPDATE_UDFS_AND_SPS": True' -m ... + +Dropping and creating udfs can also be done without running a model: + +```sh +dbt run-operation create_udfs --args 'drop_:false' +dbt run-operation create_udfs --args 'drop_:true' +``` ### Resources: -- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) -- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers -- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support -- Find [dbt events](https://events.getdbt.com) near you -- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +* Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +* Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +* Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +* Find [dbt events](https://events.getdbt.com) near you +* Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices ## Applying Model Tags @@ -67,7 +76,7 @@ Database and schema tags are applied via the `add_database_or_schema_tags` macro ### Model tags -To add/update a model's snowflake tags, add/modify the `meta` model property under `config`. Only table level tags are supported at this time via DBT. +To add/update a model's snowflake tags, add/modify the `meta` model property under `config` . Only table level tags are supported at this time via DBT. ``` {{ config( @@ -94,4 +103,4 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks ``` select * from table(livequery.information_schema.tag_references('livequery.core.fact_blocks', 'table')); -``` \ No newline at end of file +``` diff --git a/analysis/test_create_or_drop_function_from_config.sql b/analysis/test_create_or_drop_function_from_config.sql new file mode 100644 index 0000000..fa4eae0 --- /dev/null +++ b/analysis/test_create_or_drop_function_from_config.sql @@ -0,0 +1,17 @@ +{% set name %} + {{- udf_configs() -}} +{% endset %} +{% set udfs = fromyaml(name) %} + +{{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=True) -}} +{{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=True) -}} +{{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=True) -}} +{{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=True) -}} +{{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=True) -}} +{{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=True) -}} +{{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=False) -}} +{{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=False) -}} +{{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=False) -}} +{{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=False) -}} +{{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=False) -}} +{{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=False) -}} \ No newline at end of file diff --git a/cspell.yml b/cspell.yml new file mode 100644 index 0000000..5e49b14 --- /dev/null +++ b/cspell.yml @@ -0,0 +1,7 @@ +version: "0.2" +language: en +words: + - fromyaml + - GETVARIABLE + - livequery + - udfs diff --git a/dbt_project.yml b/dbt_project.yml index 823bb9a..0ffe08b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -29,10 +29,10 @@ tests: on-run-start: - "{{ create_sps() }}" - # - "{{ create_udfs() }}" + - "{{ create_udfs() }}" on-run-end: - - '{{ apply_meta_as_tags(results) }}' + - "{{ apply_meta_as_tags(results) }}" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models @@ -43,5 +43,11 @@ on-run-end: vars: "dbt_date:time_zone": GMT - UPDATE_UDFS_AND_SPS: False - UPDATE_SNOWFLAKE_TAGS: True \ No newline at end of file + UPDATE_UDFS_AND_SPS: true + UPDATE_SNOWFLAKE_TAGS: True + STREAMLINE_INVOKE_STREAMS: False + STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False + REST_API_ID_PROD: hn8uqhku77 + REST_API_ID_DEV: hn8uqhku77 + API_INTEGRATION: AWS_LIVE_QUERY_DEV + AWS_REGION: us-east-1 diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 0923536..4faba28 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,8 +1,26 @@ --- {% macro create_udfs() %} --- {% if var("UPDATE_UDFS_AND_SPS") %} --- {% set sql %} --- CREATE schema if NOT EXISTS silver; --- {% endset %} --- {% do run_query(sql) %} --- {% endif %} --- {% endmacro %} +{% macro create_udfs(drop_=False) %} + {% if var("UPDATE_UDFS_AND_SPS") %} + {% set name %} + {{- udf_configs() -}} + {% endset %} + {% set udfs = fromyaml(name) %} + {% set sql %} + CREATE schema if NOT EXISTS silver; + CREATE schema if NOT EXISTS streamline; + CREATE schema if NOT EXISTS beta; + {{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=True) }} + {{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=True) }} + {{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=True) }} + {{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=True) }} + {{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=True) }} + {{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=True) }} + {{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=False) }} + {{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=False) }} + {{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=False) }} + {{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=False) }} + {{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=False) }} + {{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=False) }} + {% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql new file mode 100644 index 0000000..f6d2d38 --- /dev/null +++ b/macros/streamline/api_integrations.sql @@ -0,0 +1,11 @@ +{% macro create_aws_ethereum_api() %} + {% if target.name == "prod" %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_ethereum_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::661245089684:role/snowflake-api-ethereum' api_allowed_prefixes = ( + 'https://e03pt6v501.execute-api.us-east-1.amazonaws.com/prod/', + 'https://mryeusnrob.execute-api.us-east-1.amazonaws.com/dev/' + ) enabled = TRUE; +{% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} diff --git a/macros/streamline/configs.yaml.sql b/macros/streamline/configs.yaml.sql new file mode 100644 index 0000000..20a41ff --- /dev/null +++ b/macros/streamline/configs.yaml.sql @@ -0,0 +1,81 @@ +{% macro udf_configs() %} +streamline.introspect: + name: streamline.udf_introspect + signature: + - [echo, STRING] + func_type: SECURE EXTERNAL + return_type: TEXT + api_integration: AWS_LIVE_QUERY_DEV + sql: introspect + +beta.udf_register_secret: + name: beta.udf_register_secret + signature: + - [request_id, string] + - [key, string] + func_type: SECURE + return_type: TEXT + options: NOT NULL STRICT IMMUTABLE + sql: | + SELECT + STREAMLINE.UDF_REGISTER_SECRET(REQUEST_ID, STREAMLINE.UDF_WHOAMI(), KEY) + +beta.udf_api: + name: beta.udf_api + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [data, OBJECT] + - [secret_name, STRING] + return_type: VARIANT + func_type: SECURE + options: NOT NULL STRICT VOLATILE + sql: | + SELECT + STREAMLINE.UDF_API( + method, + url, + headers, + data, + STREAMLINE.UDF_WHOAMI(), + secret_name + ) + +streamline.udf_api: + name: streamline.udf_api + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [DATA, OBJECT] + - [user_id, STRING] + - [SECRET, STRING] + return_type: VARIANT + func_type: SECURE EXTERNAL + api_integration: AWS_LIVE_QUERY_DEV + options: NOT NULL STRICT + sql: udf_api + +streamline.udf_register_secret: + name: streamline.udf_register_secret + signature: + - [request_id, string] + - [user_id, string] + - [key, string] + return_type: TEXT + func_type: SECURE EXTERNAL + api_integration: AWS_LIVE_QUERY_DEV + options: NOT NULL STRICT + sql: secret/register + +streamline.whoami: + name: streamline.udf_whoami + signature: [] + func_type: SECURE + return_type: TEXT + options: NOT NULL STRICT IMMUTABLE MEMOIZABLE + sql: | + SELECT + COALESCE(SPLIT_PART(GETVARIABLE('QUERY_TAG_SESSION'), ',',2), CURRENT_USER()) +{% endmacro %} diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql new file mode 100644 index 0000000..4381de6 --- /dev/null +++ b/macros/streamline/streamline_udfs.sql @@ -0,0 +1,22 @@ +{% macro create_udf_introspect( + drop_ = False + ) %} + {% set name_ = 'silver.udf_introspect' %} + {% set signature = [('json', 'variant')] %} + {% set return_type = 'text' %} + {% set sql_ = construct_api_route("introspect") %} + {% if not drop_ %} + {{ create_sql_function( + name_ = name_, + signature = signature, + return_type = return_type, + sql_ = sql_, + api_integration = var("API_INTEGRATION") + ) }} + {% else %} + {{ drop_function( + name_, + signature = signature, + ) }} + {% endif %} +{% endmacro %} diff --git a/macros/streamline/utils.sql b/macros/streamline/utils.sql new file mode 100644 index 0000000..9a0031e --- /dev/null +++ b/macros/streamline/utils.sql @@ -0,0 +1,83 @@ +{% macro drop_function( + func_name, + signature + ) %} + DROP FUNCTION IF EXISTS {{ func_name }}({{ compile_signature(signature, drop_ = True) }}); +{% endmacro %} + +{%- macro construct_api_route(route) -%} + 'https://{{ var("REST_API_ID_PROD") if target.name == "prod" else var("REST_API_ID_DEV") }}.execute-api.{{ var( aws_region, "us-east-1" ) }}.amazonaws.com/{{ target.name }}/{{ route }}' +{%- endmacro -%} + +{%- macro compile_signature( + params, + drop_ = False + ) -%} + {% for name, + data_type in params -%} + {% if drop_ %} + {{ data_type -}} + {% else %} + {{ name ~ " " ~ data_type -}} + {% endif -%} + {%-if not loop.last -%}, + {%- endif -%} + {% endfor -%} +{% endmacro %} + +{% macro create_sql_function( + name_, + signature, + return_type, + sql_, + api_integration = none, + options = none, + func_type = none + ) %} + CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}( + {{- compile_signature(signature) }} + ) + RETURNS {{ return_type }} + {% if options -%} + {{ options }} + {% endif %} + {%- if api_integration -%} + api_integration = {{ api_integration }} + AS {{ construct_api_route(sql_) ~ ";" }} + {% else -%} + AS + $$ + {{ sql_ }} + $$; + {%- endif -%} +{%- endmacro -%} + +{%- macro create_or_drop_function_from_config( + config, + drop_ = False + ) -%} + {% set name_ = config ["name"] %} + {% set signature = config ["signature"] %} + {% set return_type = config ["return_type"] %} + {% set sql_ = config ["sql"] %} + {% set options = config ["options"] %} + {% set api_integration = config ["api_integration"] %} + {% set func_type = config ["func_type"] %} + + {% if not drop_ -%} + {{ create_sql_function( + name_ = name_, + signature = signature, + return_type = return_type, + sql_ = sql_, + options = options, + api_integration = api_integration, + func_type = func_type + ) }} + {%- else -%} + {{ drop_function( + name_, + signature = signature, + ) }} + {%- endif %} +{% endmacro %} diff --git a/packages.yml b/packages.yml index e799c13..ee9544f 100644 --- a/packages.yml +++ b/packages.yml @@ -1,7 +1,5 @@ packages: - package: calogica/dbt_expectations version: [">=0.8.0", "<0.9.0"] - - package: dbt-labs/dbt_external_tables - version: [">=0.8.0", "<0.9.0"] - package: dbt-labs/dbt_utils - version: [">=1.0.0", "<1.1.0"] \ No newline at end of file + version: [">=1.0.0", "<1.1.0"]