From 2c561fd8ff03763aedb1a506000b7cbca32cfa86 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Wed, 11 Oct 2023 09:23:07 -0700 Subject: [PATCH 1/2] add property for excluding from a datashare --- macros/streamline/configs.yaml.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/macros/streamline/configs.yaml.sql b/macros/streamline/configs.yaml.sql index 81f149b..a17d452 100644 --- a/macros/streamline/configs.yaml.sql +++ b/macros/streamline/configs.yaml.sql @@ -56,6 +56,7 @@ IMMUTABLE sql: | {{ fsc_utils.sql_udf_json_rpc_call() }} + exclude_from_datashare: true - name: {{ schema }}.udf_json_rpc_call signature: - [method, STRING] @@ -68,6 +69,7 @@ IMMUTABLE sql: | {{ fsc_utils.sql_udf_json_rpc_call() }} + exclude_from_datashare: true - name: {{ schema }}.udf_json_rpc_call signature: - [method, STRING] @@ -81,6 +83,7 @@ IMMUTABLE sql: | {{ fsc_utils.sql_udf_json_rpc_call(False) }} + exclude_from_datashare: true - name: {{ schema }}.udf_json_rpc_call signature: - [method, STRING] @@ -94,6 +97,7 @@ IMMUTABLE sql: | {{ fsc_utils.sql_udf_json_rpc_call(False) }} + exclude_from_datashare: true - name: {{ schema }}.udf_evm_text_signature signature: From 0de08fc0ff051176288f7eda45a4fdddd2e1b8cd Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Wed, 11 Oct 2023 09:58:56 -0700 Subject: [PATCH 2/2] add macro and model to generate UDF ddls --- macros/datashares.sql | 37 +++++++++++++++++++ models/datashare/_datashare___create_udfs.sql | 20 ++++++++++ 2 files changed, 57 insertions(+) create mode 100644 models/datashare/_datashare___create_udfs.sql diff --git a/macros/datashares.sql b/macros/datashares.sql index 744fdc8..d892dda 100644 --- a/macros/datashares.sql +++ b/macros/datashares.sql @@ -156,4 +156,41 @@ {%- set combined_ddl = gold_views_ddl + gold_tables_ddl -%} {%- do combined_ddl.insert(0, "CREATE DATABASE IF NOT EXISTS __NEW__;") -%} {{- "BEGIN\n" ~ (combined_ddl | join("\n")) ~ "\nEND" -}} +{%- endmacro -%} + +{% macro generate_datashare_udf_ddl() %} +{# + generate UDF DDL for datashare + + Return: UDF DDL for datashare + #} + {%- set schema = "UTILS" -%} + {%- set udfs = fromyaml(fsc_utils.udf_configs(schema)) -%} + {%- set combined_ddl = [] -%} + {%- for udf in udfs -%} + {% set name_ = udf ["name"] %} + {% set signature = udf ["signature"] %} + {% set return_type = udf ["return_type"] %} + {% set sql_ = udf ["sql"] %} + {% set options = udf ["options"] %} + {% set api_integration = udf ["api_integration"] %} + {% set func_type = udf ["func_type"] %} + {% set exclude_from_datashare = udf.get("exclude_from_datashare",False) %} + {% if not exclude_from_datashare %} + {%- set udf_ddl = fsc_utils.create_sql_function( + name_ = name_, + signature = signature, + return_type = return_type, + sql_ = sql_, + options = options, + api_integration = api_integration, + func_type = func_type + ).replace("\\","\\\\").replace("'","\\'") -%} + {%- do combined_ddl.append(udf_ddl) -%} + {% endif %} + {% endfor %} + {%- do combined_ddl.insert(0, "CREATE DATABASE IF NOT EXISTS __NEW__;") -%} + {%- do combined_ddl.insert(1, "USE DATABASE __NEW__;") -%} + {%- do combined_ddl.insert(2, "CREATE SCHEMA IF NOT EXISTS "~schema~";") -%} + {{- "'BEGIN','" ~ (combined_ddl | join("','")) ~ "','END'" -}} {%- endmacro -%} \ No newline at end of file diff --git a/models/datashare/_datashare___create_udfs.sql b/models/datashare/_datashare___create_udfs.sql new file mode 100644 index 0000000..cd10cc1 --- /dev/null +++ b/models/datashare/_datashare___create_udfs.sql @@ -0,0 +1,20 @@ +{{ + config( + materialized = 'incremental', + incremental_strategy = 'merge', + unique_key = 'ddl_hash', + merge_update_columns = [], + ) +}} +{% if execute %} +SELECT +CONCAT_WS('\n',{{- fsc_utils.generate_datashare_udf_ddl().strip() -}}) AS ddl, +md5(ddl) AS ddl_hash, +sysdate() as ddl_created_at +{% else %} +SELECT +null as ddl, +null as ddl_hash, +null as ddl_created_at +from dual limit 0 +{% endif %} \ No newline at end of file