From fca84695cf2fccc3ea96d59f8651391c30e3eb83 Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Thu, 17 Jul 2025 20:23:20 +0900 Subject: [PATCH] add udf_api_v2 --- macros/core/live.yaml.sql | 205 +++++++++++++- .../external_access_integrations.sql | 9 +- models/deploy/core/_external_access.sql | 5 + models/deploy/core/live.yml | 260 +++++++++++++++++- 4 files changed, 453 insertions(+), 26 deletions(-) diff --git a/macros/core/live.yaml.sql b/macros/core/live.yaml.sql index 607f02b..3ca2499 100644 --- a/macros/core/live.yaml.sql +++ b/macros/core/live.yaml.sql @@ -130,6 +130,95 @@ secret_name ) +- name: {{ schema }}.udf_rpc + signature: + - [blockchain, STRING] + - [network, STRING] + - [method, STRING] + - [parameters, VARIANT] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes an JSON RPC call on a blockchain.$$ + sql: | + {{ sql_live_rpc_call("method", "parameters", "blockchain", "network") | indent(4) -}} + +- name: {{ schema }}.udf_allow_list + signature: [] + return_type: ARRAY + func_type: EXTERNAL + api_integration: '{{ var("API_INTEGRATION") }}' + options: | + RETURNS NULL ON NULL INPUT + VOLATILE + COMMENT = $$Returns a list of allowed domains.$$ + sql: allowed + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [headers, OBJECT] + - [secret_name, STRING] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$ + sql: | + SELECT + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) + :s3_presigned_url::STRING + ):data[0][1] as result + WHERE LOWER(COALESCE( + headers:"fsc-quantum-execution-mode"::STRING, + headers:"Fsc-Quantum-Execution-Mode"::STRING, + headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING + )) = 'async' + + UNION ALL + + SELECT + _live.udf_api_sync('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) as result + WHERE LOWER(COALESCE( + headers:"fsc-quantum-execution-mode"::STRING, + headers:"Fsc-Quantum-Execution-Mode"::STRING, + headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, + 'sync' + )) != 'async' + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [headers, OBJECT] + - [secret_name, STRING] + - [is_async, BOOLEAN] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$ + sql: | + SELECT + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) + :s3_presigned_url::STRING + ):data[0][1] as result + WHERE LOWER(COALESCE( + headers:"fsc-quantum-execution-mode"::STRING, + headers:"Fsc-Quantum-Execution-Mode"::STRING, + headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING + )) = 'async' + + UNION ALL + + SELECT + _live.udf_api_sync('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) as result + WHERE LOWER(COALESCE( + headers:"fsc-quantum-execution-mode"::STRING, + headers:"Fsc-Quantum-Execution-Mode"::STRING, + headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, + 'sync' + )) != 'async' + - name: {{ schema }}.udf_api_v2 signature: - [method, STRING] @@ -160,7 +249,8 @@ WHERE LOWER(COALESCE( headers:"fsc-quantum-execution-mode"::STRING, headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING + headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, + 'sync' )) != 'async' - name: {{ schema }}.udf_api_v2 @@ -191,27 +281,112 @@ ) END -- name: {{ schema }}.udf_rpc +- name: {{ schema }}.udf_api_v2 signature: - - [blockchain, STRING] - - [network, STRING] - [method, STRING] - - [parameters, VARIANT] + - [url, STRING] + - [headers, OBJECT] + - [data, VARIANT] return_type: VARIANT options: | VOLATILE - COMMENT = $$Executes an JSON RPC call on a blockchain.$$ + COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$ sql: | - {{ sql_live_rpc_call("method", "parameters", "blockchain", "network") | indent(4) -}} + SELECT + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async(method, url, headers, data, _utils.UDF_WHOAMI(), '') + :s3_presigned_url::STRING + ):data[0][1] as result + WHERE LOWER(COALESCE( + headers:"fsc-quantum-execution-mode"::STRING, + headers:"Fsc-Quantum-Execution-Mode"::STRING, + headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING + )) = 'async' -- name: {{ schema }}.udf_allow_list - signature: [] - return_type: ARRAY - func_type: EXTERNAL - api_integration: '{{ var("API_INTEGRATION") }}' + UNION ALL + + SELECT + _live.udf_api_sync(method, url, headers, data, _utils.UDF_WHOAMI(), '') as result + WHERE LOWER(COALESCE( + headers:"fsc-quantum-execution-mode"::STRING, + headers:"Fsc-Quantum-Execution-Mode"::STRING, + headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, + 'sync' + )) != 'async' + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [data, VARIANT] + return_type: VARIANT options: | - RETURNS NULL ON NULL INPUT VOLATILE - COMMENT = $$Returns a list of allowed domains.$$ - sql: allowed + COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$ + sql: | + SELECT + _live.udf_api_sync( + 'POST', + url, + {'Content-Type': 'application/json'}, + data, + _utils.UDF_WHOAMI(), + '' + ) + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [data, VARIANT] + - [secret_name, STRING] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$ + sql: | + SELECT + _live.udf_api_sync( + 'POST', + url, + {'Content-Type': 'application/json'}, + data, + _utils.UDF_WHOAMI(), + secret_name + ) + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$ + sql: | + SELECT + _live.udf_api_sync( + 'GET', + url, + {}, + NULL, + _utils.UDF_WHOAMI(), + '' + ) + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [secret_name, STRING] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$ + sql: | + SELECT + _live.udf_api_sync( + 'GET', + url, + {}, + {}, + _utils.UDF_WHOAMI(), + secret_name + ) {% endmacro %} diff --git a/macros/livequery/external_access_integrations.sql b/macros/livequery/external_access_integrations.sql index 74c255b..a561591 100644 --- a/macros/livequery/external_access_integrations.sql +++ b/macros/livequery/external_access_integrations.sql @@ -4,7 +4,7 @@ {% endset %} {% set network_rule_sql %} - CREATE OR REPLACE NETWORK RULE s3_express_network_rule + CREATE NETWORK RULE IF NOT EXISTS s3_express_network_rule MODE = EGRESS TYPE = HOST_PORT VALUE_LIST = ( @@ -15,17 +15,12 @@ {% endset %} {% set external_access_sql %} - CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION s3_express_external_access_integration + CREATE EXTERNAL ACCESS INTEGRATION IF NOT EXISTS s3_express_external_access_integration ALLOWED_NETWORK_RULES = (s3_express_network_rule) ENABLED = true {% endset %} {% do run_query(use_schema_sql) %} - {% do log("Schema context set to live", true) %} - {% do run_query(network_rule_sql) %} - {% do log("Network rule successfully created", true) %} - {% do run_query(external_access_sql) %} - {% do log("External S3 Express access integration successfully created", true) %} {% endmacro %} diff --git a/models/deploy/core/_external_access.sql b/models/deploy/core/_external_access.sql index 709de4a..347dc06 100644 --- a/models/deploy/core/_external_access.sql +++ b/models/deploy/core/_external_access.sql @@ -1 +1,6 @@ {{ create_s3_express_external_access_integration() }} + +-- this is to pass the model render as dbt dependency in other models +-- livequery will need s3 express access to read from the s3 bucket + +SELECT 1 diff --git a/models/deploy/core/live.yml b/models/deploy/core/live.yml index 22b8eed..978eec5 100644 --- a/models/deploy/core/live.yml +++ b/models/deploy/core/live.yml @@ -7,10 +7,10 @@ models: - test_udf: name: test__live_udf_api_batched_post_data_object args: | - 'GET', - 'https://httpbin.org/get', - {'Content-Type': 'application/json'}, - {'param1': 'value1', 'param2': 'value2'}, + 'GET', + 'https://httpbin.org/get', + {'Content-Type': 'application/json'}, + {'param1': 'value1', 'param2': 'value2'}, '' assertions: - result:status_code = 200 @@ -168,3 +168,255 @@ models: - result:data[1]:jsonrpc = '2.0' - result:data[1]:id = 2 - result:data[1]:result = '0x1' + + - name: udf_api_v2 + tests: + # Convenience overloads (always sync) + - test_udf: + name: test__live_udf_api_v2_post_data_object_sync + args: | + 'https://httpbin.org/post', {'foo': 'bar'} + assertions: + - result:data.json is not null + - result:data.json = OBJECT_CONSTRUCT('foo', 'bar') + - test_udf: + name: test__live_udf_api_v2_post_data_array_sync + args: | + 'https://httpbin.org/post', ['foo', 'bar'] + assertions: + - result:data.json is not null + - result:data.json = ARRAY_CONSTRUCT('foo', 'bar') + - test_udf: + name: test__live_udf_api_v2_post_data_string_sync + args: | + 'https://httpbin.org/post', 'foo'::VARIANT + assertions: + - result:data.json is not null + - result:data.json = 'foo' + - test_udf: + name: test__live_udf_api_v2_get_method_sync + args: | + 'https://httpbin.org/get' + assertions: + - result:status_code = 200 + - result:data.url = 'https://httpbin.org/get' + - test_udf: + name: test__live_udf_api_v2_post_batch_jsonrpc_sync + args: | + 'https://httpbin.org/post', { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'batch', + 'params': [ + {'id': 1, 'method': 'method1', 'params': {'param1': 'value1'}}, + {'id': 2, 'method': 'method2', 'params': {'param2': 'value2'}} + ] + } + assertions: + - result:status_code = 200 + - result:data.json:jsonrpc = '2.0' + - result:data.json:id = 1 + - result:data.json:method = 'batch' + - result:data.json:params is not null + - result:data.json:params[0]:id = 1 + - result:data.json:params[1]:id = 2 + + # Full signature tests - SYNC mode + - test_udf: + name: test__live_udf_api_v2_get_with_params_sync + args: | + 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1', 'param2': 'value2'}, '' + assertions: + - result:status_code = 200 + - result:data.args is not null + - result:data.args:param1 = 'value1' + - result:data.args:param2 = 'value2' + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_solana_sync + args: | + 'POST', + 'https://api.mainnet-beta.solana.com', + {'Content-Type': 'application/json'}, + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'getVersion' + }, + '' + assertions: + - result:status_code = 200 + - result:data.jsonrpc = '2.0' + - result:data.id = 1 + - result:data.result is not null + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_solana_batch_sync + args: | + 'POST', + 'https://api.mainnet-beta.solana.com', + {'Content-Type': 'application/json'}, + [ + {'jsonrpc': '2.0', 'id': 1, 'method': 'getVersion'}, + {'jsonrpc': '2.0', 'id': 2, 'method': 'getVersion'} + ], + '' + assertions: + - result:status_code = 200 + - result:data[0]:jsonrpc = '2.0' + - result:data[0]:id = 1 + - result:data[0]:result is not null + - result:data[1]:jsonrpc = '2.0' + - result:data[1]:id = 2 + - result:data[1]:result is not null + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_ethereum_batch_sync + args: | + 'POST', + 'https://ethereum-rpc.publicnode.com', + {'Content-Type': 'application/json'}, + [ + {'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []}, + {'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []} + ], + '' + assertions: + - result:status_code = 200 + - result:data[0]:jsonrpc = '2.0' + - result:data[0]:id = 1 + - result:data[0]:result is not null + - result:data[1]:jsonrpc = '2.0' + - result:data[1]:id = 2 + - result:data[1]:result = '0x1' + + # Full signature tests - ASYNC mode + - test_udf: + name: test__live_udf_api_v2_get_with_params_async + args: | + 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, {'param1': 'value1', 'param2': 'value2'}, '' + assertions: + - result:status_code = 200 + - result:data.args is not null + - result:data.args:param1 = 'value1' + - result:data.args:param2 = 'value2' + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_solana_async + args: | + 'POST', + 'https://api.mainnet-beta.solana.com', + {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'getVersion' + }, + '' + assertions: + - result:status_code = 200 + - result:data.jsonrpc = '2.0' + - result:data.id = 1 + - result:data.result is not null + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_solana_batch_async + args: | + 'POST', + 'https://api.mainnet-beta.solana.com', + {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, + [ + {'jsonrpc': '2.0', 'id': 1, 'method': 'getVersion'}, + {'jsonrpc': '2.0', 'id': 2, 'method': 'getVersion'} + ], + '' + assertions: + - result:status_code = 200 + - result:data[0]:jsonrpc = '2.0' + - result:data[0]:id = 1 + - result:data[0]:result is not null + - result:data[1]:jsonrpc = '2.0' + - result:data[1]:id = 2 + - result:data[1]:result is not null + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_ethereum_batch_async + args: | + 'POST', + 'https://ethereum-rpc.publicnode.com', + {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, + [ + {'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []}, + {'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []} + ], + '' + assertions: + - result:status_code = 200 + - result:data[0]:jsonrpc = '2.0' + - result:data[0]:id = 1 + - result:data[0]:result is not null + - result:data[1]:jsonrpc = '2.0' + - result:data[1]:id = 2 + - result:data[1]:result = '0x1' + + # Explicit is_async boolean parameter tests + - test_udf: + name: test__live_udf_api_v2_get_with_headers_is_async_true + args: | + 'https://httpbin.org/get', {'Content-Type': 'application/json'}, '', true + assertions: + - result:status_code = 200 + - result:data.url = 'https://httpbin.org/get' + - test_udf: + name: test__live_udf_api_v2_get_with_headers_is_async_false + args: | + 'https://httpbin.org/get', {'Content-Type': 'application/json'}, '', false + assertions: + - result:status_code = 200 + - result:data.url = 'https://httpbin.org/get' + - test_udf: + name: test__live_udf_api_v2_full_signature_is_async_true + args: | + 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1'}, '', true + assertions: + - result:status_code = 200 + - result:data.args is not null + - result:data.args:param1 = 'value1' + - test_udf: + name: test__live_udf_api_v2_full_signature_is_async_false + args: | + 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1'}, '', false + assertions: + - result:status_code = 200 + - result:data.args is not null + - result:data.args:param1 = 'value1' + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_is_async_true + args: | + 'POST', + 'https://api.mainnet-beta.solana.com', + {'Content-Type': 'application/json'}, + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'getVersion' + }, + '', + true + assertions: + - result:status_code = 200 + - result:data.jsonrpc = '2.0' + - result:data.id = 1 + - result:data.result is not null + - test_udf: + name: test__live_udf_api_v2_post_jsonrpc_is_async_false + args: | + 'POST', + 'https://api.mainnet-beta.solana.com', + {'Content-Type': 'application/json'}, + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'getVersion' + }, + '', + false + assertions: + - result:status_code = 200 + - result:data.jsonrpc = '2.0' + - result:data.id = 1 + - result:data.result is not null