From 18ffc7f1d260a9ed2ab4d1a63aec6c7cc8ad4c91 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Thu, 6 Apr 2023 17:40:57 -0500 Subject: [PATCH] AN-2979/udf-json-rpc-call (#10) --- README.md | 4 +- analysis/test_functions.sql | 2 + analysis/test_functions_py.sql | 3 + macros/streamline/configs.yaml.sql | 186 +++++++++++++++++++++++++++-- macros/streamline/functions.sql | 15 +++ 5 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 analysis/test_functions.sql create mode 100644 analysis/test_functions_py.sql create mode 100644 macros/streamline/functions.sql diff --git a/README.md b/README.md index 181f6a7..6da6c21 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ dbt run --var 'UPDATE_UDFS_AND_SPS": True' -m ... Dropping and creating udfs can also be done without running a model: ```sh -dbt run-operation create_udfs --var 'UPDATE_UDFS_AND_SPS": True' --args 'drop_:false' -dbt run-operation create_udfs --var 'UPDATE_UDFS_AND_SPS": True' --args 'drop_:true' +dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":false}' +dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":true}' ``` ## Resources diff --git a/analysis/test_functions.sql b/analysis/test_functions.sql new file mode 100644 index 0000000..767766f --- /dev/null +++ b/analysis/test_functions.sql @@ -0,0 +1,2 @@ +{{ sql_udf_json_rpc_call(True) }} +{{ sql_udf_json_rpc_call(False) }} diff --git a/analysis/test_functions_py.sql b/analysis/test_functions_py.sql new file mode 100644 index 0000000..11eef4a --- /dev/null +++ b/analysis/test_functions_py.sql @@ -0,0 +1,3 @@ + +{{ python_hex_to_int() }} +{{ python_udf_hex_to_int_with_encoding() }} \ No newline at end of file diff --git a/macros/streamline/configs.yaml.sql b/macros/streamline/configs.yaml.sql index 59d0949..702380c 100644 --- a/macros/streamline/configs.yaml.sql +++ b/macros/streamline/configs.yaml.sql @@ -16,7 +16,11 @@ signature: [] func_type: SECURE return_type: TEXT - options: NOT NULL STRICT IMMUTABLE MEMOIZABLE + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + MEMOIZABLE sql: | SELECT COALESCE(SPLIT_PART(GETVARIABLE('QUERY_TAG_SESSION'), ',',2), CURRENT_USER()) @@ -29,7 +33,9 @@ return_type: TEXT func_type: SECURE EXTERNAL api_integration: '{{ var("API_INTEGRATION") }}' - options: NOT NULL STRICT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT sql: secret/register - name: utils.udf_register_secret signature: @@ -37,7 +43,10 @@ - [key, STRING] func_type: SECURE return_type: TEXT - options: NOT NULL STRICT IMMUTABLE + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE sql: | SELECT _utils.UDF_REGISTER_SECRET(REQUEST_ID, _utils.UDF_WHOAMI(), KEY) @@ -49,7 +58,8 @@ options: | NULL LANGUAGE PYTHON - STRICT IMMUTABLE + RETURNS NULL ON NULL INPUT + IMMUTABLE RUNTIME_VERSION = '3.8' HANDLER = 'hex_to_int' sql: | @@ -62,7 +72,8 @@ options: | NULL LANGUAGE PYTHON - STRICT IMMUTABLE + RETURNS NULL ON NULL INPUT + IMMUTABLE RUNTIME_VERSION = '3.8' HANDLER = 'hex_to_int' sql: | @@ -74,14 +85,66 @@ return_type: TEXT options: | NULL - LANGUAGE SQL - STRICT IMMUTABLE + LANGUAGE SQL + RETURNS NULL ON NULL INPUT + IMMUTABLE sql: | SELECT LTRIM(regexp_replace( try_hex_decode_string(hex), '[\x00-\x1F\x7F-\x9F\xAD]', '', 1)) +- name: utils.udf_json_rpc_call + signature: + - [method, STRING] + - [params, ARRAY] + return_type: OBJECT + options: | + NULL + LANGUAGE SQL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + {{ sql_udf_json_rpc_call() }} +- name: utils.udf_json_rpc_call + signature: + - [method, STRING] + - [params, OBJECT] + return_type: OBJECT + options: | + NULL + LANGUAGE SQL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + {{ sql_udf_json_rpc_call() }} +- name: utils.udf_json_rpc_call + signature: + - [method, STRING] + - [params, OBJECT] + - [id, STRING] + return_type: OBJECT + options: | + NULL + LANGUAGE SQL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + {{ sql_udf_json_rpc_call(False) }} +- name: utils.udf_json_rpc_call + signature: + - [method, STRING] + - [params, ARRAY] + - [id, STRING] + return_type: OBJECT + options: | + NULL + LANGUAGE SQL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + {{ sql_udf_json_rpc_call(False) }} + {# LIVE SCHEMA #} @@ -94,9 +157,11 @@ - [user_id, STRING] - [SECRET, STRING] return_type: VARIANT - func_type: SECURE EXTERNAL + func_type: EXTERNAL api_integration: '{{ var("API_INTEGRATION") }}' - options: NOT NULL STRICT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT sql: udf_api - name: live.udf_api signature: @@ -106,8 +171,10 @@ - [data, OBJECT] - [secret_name, STRING] return_type: VARIANT - func_type: SECURE - options: NOT NULL STRICT VOLATILE + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE sql: | SELECT _live.UDF_API( @@ -118,6 +185,103 @@ _utils.UDF_WHOAMI(), secret_name ) +- name: live.udf_api + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [data, OBJECT] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + method, + url, + headers, + data, + _utils.UDF_WHOAMI(), + '' + ) +- name: live.udf_api + signature: + - [url, STRING] + - [data, OBJECT] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + 'POST', + url, + {'Content-Type': 'application/json'}, + data, + _utils.UDF_WHOAMI(), + '' + ) +- name: live.udf_api + signature: + - [url, STRING] + - [data, OBJECT] + - [secret_name, STRING] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + 'POST', + url, + {'Content-Type': 'application/json'}, + data, + _utils.UDF_WHOAMI(), + secret_name + ) +- name: live.udf_api + signature: + - [url, STRING] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + 'GET', + url, + {}, + {}, + _utils.UDF_WHOAMI(), + '' + ) +- name: live.udf_api + signature: + - [url, STRING] + - [secret_name, STRING] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + 'GET', + url, + {}, + {}, + _utils.UDF_WHOAMI(), + secret_name + ) {% endmacro %} diff --git a/macros/streamline/functions.sql b/macros/streamline/functions.sql new file mode 100644 index 0000000..6715bb8 --- /dev/null +++ b/macros/streamline/functions.sql @@ -0,0 +1,15 @@ +{% macro sql_udf_json_rpc_call(use_default_id=True ) %} + SELECT IFF(method IS NULL or params IS NULL, + NULL, + { + 'jsonrpc': '2.0', + 'method': method, + 'params': params + {% if use_default_id %} + , 'id': hash(method, params)::string + {% else %} + , 'id': id + {% endif %} + } + ) +{% endmacro %}