add udf_api_v2

macros/core/live.yaml.sql

@@ -130,6 +130,95 @@
           secret_name
       )
 
+- name: {{ schema }}.udf_rpc
+  signature:
+    - [blockchain, STRING]
+    - [network, STRING]
+    - [method, STRING]
+    - [parameters, VARIANT]
+  return_type: VARIANT
+  options: |
+    VOLATILE
+    COMMENT = $$Executes an JSON RPC call on a blockchain.$$
+  sql: |
+    {{ sql_live_rpc_call("method", "parameters", "blockchain", "network") | indent(4) -}}
+
+- name: {{ schema }}.udf_allow_list
+  signature: []
+  return_type: ARRAY
+  func_type: EXTERNAL
+  api_integration: '{{ var("API_INTEGRATION") }}'
+  options: |
+    RETURNS NULL ON NULL INPUT
+    VOLATILE
+    COMMENT = $$Returns a list of allowed domains.$$
+  sql: allowed
+
+- name: {{ schema }}.udf_api_v2
+  signature:
+    - [url, STRING]
+    - [headers, OBJECT]
+    - [secret_name, STRING]
+  return_type: VARIANT
+  options: |
+    VOLATILE
+    COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$
+  sql: |
+    SELECT
+        utils.udf_redirect_s3_presigned_url(
+            _live.udf_api_async('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name)
+            :s3_presigned_url::STRING
+        ):data[0][1] as result
+    WHERE LOWER(COALESCE(
+        headers:"fsc-quantum-execution-mode"::STRING,
+        headers:"Fsc-Quantum-Execution-Mode"::STRING,
+        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING
+    )) = 'async'
+
+    UNION ALL
+
+    SELECT
+        _live.udf_api_sync('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) as result
+    WHERE LOWER(COALESCE(
+        headers:"fsc-quantum-execution-mode"::STRING,
+        headers:"Fsc-Quantum-Execution-Mode"::STRING,
+        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING,
+        'sync'
+    )) != 'async'
+
+- name: {{ schema }}.udf_api_v2
+  signature:
+    - [url, STRING]
+    - [headers, OBJECT]
+    - [secret_name, STRING]
+    - [is_async, BOOLEAN]
+  return_type: VARIANT
+  options: |
+    VOLATILE
+    COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$
+  sql: |
+    SELECT
+        utils.udf_redirect_s3_presigned_url(
+            _live.udf_api_async('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name)
+            :s3_presigned_url::STRING
+        ):data[0][1] as result
+    WHERE LOWER(COALESCE(
+        headers:"fsc-quantum-execution-mode"::STRING,
+        headers:"Fsc-Quantum-Execution-Mode"::STRING,
+        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING
+    )) = 'async'
+
+    UNION ALL
+
+    SELECT
+        _live.udf_api_sync('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) as result
+    WHERE LOWER(COALESCE(
+        headers:"fsc-quantum-execution-mode"::STRING,
+        headers:"Fsc-Quantum-Execution-Mode"::STRING,
+        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING,
+        'sync'
+    )) != 'async'
+
 - name: {{ schema }}.udf_api_v2
   signature:
     - [method, STRING]
@@ -160,7 +249,8 @@
     WHERE LOWER(COALESCE(
         headers:"fsc-quantum-execution-mode"::STRING,
         headers:"Fsc-Quantum-Execution-Mode"::STRING,
-        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING
+        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING,
+        'sync'
     )) != 'async'
 
 - name: {{ schema }}.udf_api_v2
@@ -191,27 +281,112 @@
             )
     END
 
-- name: {{ schema }}.udf_rpc
+- name: {{ schema }}.udf_api_v2
   signature:
-    - [blockchain, STRING]
-    - [network, STRING]
     - [method, STRING]
-    - [parameters, VARIANT]
+    - [url, STRING]
+    - [headers, OBJECT]
+    - [data, VARIANT]
   return_type: VARIANT
   options: |
     VOLATILE
-    COMMENT = $$Executes an JSON RPC call on a blockchain.$$
+    COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$
   sql: |
-    {{ sql_live_rpc_call("method", "parameters", "blockchain", "network") | indent(4) -}}
+    SELECT
+        utils.udf_redirect_s3_presigned_url(
+            _live.udf_api_async(method, url, headers, data, _utils.UDF_WHOAMI(), '')
+            :s3_presigned_url::STRING
+        ):data[0][1] as result
+    WHERE LOWER(COALESCE(
+        headers:"fsc-quantum-execution-mode"::STRING,
+        headers:"Fsc-Quantum-Execution-Mode"::STRING,
+        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING
+    )) = 'async'
 
-- name: {{ schema }}.udf_allow_list
-  signature: []
-  return_type: ARRAY
-  func_type: EXTERNAL
-  api_integration: '{{ var("API_INTEGRATION") }}'
+    UNION ALL
+
+    SELECT
+        _live.udf_api_sync(method, url, headers, data, _utils.UDF_WHOAMI(), '') as result
+    WHERE LOWER(COALESCE(
+        headers:"fsc-quantum-execution-mode"::STRING,
+        headers:"Fsc-Quantum-Execution-Mode"::STRING,
+        headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING,
+        'sync'
+    )) != 'async'
+
+- name: {{ schema }}.udf_api_v2
+  signature:
+    - [url, STRING]
+    - [data, VARIANT]
+  return_type: VARIANT
   options: |
-    RETURNS NULL ON NULL INPUT
     VOLATILE
-    COMMENT = $$Returns a list of allowed domains.$$
-  sql: allowed
+    COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$
+  sql: |
+    SELECT
+        _live.udf_api_sync(
+          'POST',
+          url,
+          {'Content-Type': 'application/json'},
+          data,
+          _utils.UDF_WHOAMI(),
+          ''
+        )
+
+- name: {{ schema }}.udf_api_v2
+  signature:
+    - [url, STRING]
+    - [data, VARIANT]
+    - [secret_name, STRING]
+  return_type: VARIANT
+  options: |
+    VOLATILE
+    COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$
+  sql: |
+    SELECT
+        _live.udf_api_sync(
+          'POST',
+          url,
+          {'Content-Type': 'application/json'},
+          data,
+          _utils.UDF_WHOAMI(),
+          secret_name
+        )
+
+- name: {{ schema }}.udf_api_v2
+  signature:
+    - [url, STRING]
+  return_type: VARIANT
+  options: |
+    VOLATILE
+    COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$
+  sql: |
+    SELECT
+        _live.udf_api_sync(
+          'GET',
+          url,
+          {},
+          NULL,
+          _utils.UDF_WHOAMI(),
+          ''
+        )
+
+- name: {{ schema }}.udf_api_v2
+  signature:
+    - [url, STRING]
+    - [secret_name, STRING]
+  return_type: VARIANT
+  options: |
+    VOLATILE
+    COMMENT = $$Executes an LiveQuery Sync or Async Externall Function.$$
+  sql: |
+    SELECT
+        _live.udf_api_sync(
+          'GET',
+          url,
+          {},
+          {},
+          _utils.UDF_WHOAMI(),
+          secret_name
+        )
 {% endmacro %}

macros/livequery/external_access_integrations.sql

@@ -4,7 +4,7 @@
   {% endset %}
 
   {% set network_rule_sql %}
-    CREATE OR REPLACE NETWORK RULE s3_express_network_rule
+    CREATE NETWORK RULE IF NOT EXISTS s3_express_network_rule
     MODE = EGRESS
     TYPE = HOST_PORT
     VALUE_LIST = (
@@ -15,17 +15,12 @@
   {% endset %}
 
   {% set external_access_sql %}
-    CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION s3_express_external_access_integration
+    CREATE EXTERNAL ACCESS INTEGRATION IF NOT EXISTS s3_express_external_access_integration
     ALLOWED_NETWORK_RULES = (s3_express_network_rule)
     ENABLED = true
   {% endset %}
 
   {% do run_query(use_schema_sql) %}
-  {% do log("Schema context set to live", true) %}
-
   {% do run_query(network_rule_sql) %}
-  {% do log("Network rule successfully created", true) %}
-
   {% do run_query(external_access_sql) %}
-  {% do log("External S3 Express access integration successfully created", true) %}
 {% endmacro %}

models/deploy/core/_external_access.sql

@@ -1 +1,6 @@
 {{ create_s3_express_external_access_integration() }}
+
+-- this is to pass the model render as dbt dependency in other models
+-- livequery will need s3 express access to read from the s3 bucket
+
+SELECT 1

models/deploy/core/live.yml

@@ -7,10 +7,10 @@ models:
           - test_udf:
               name: test__live_udf_api_batched_post_data_object
               args: |
-                'GET', 
-                'https://httpbin.org/get', 
-                {'Content-Type': 'application/json'}, 
-                {'param1': 'value1', 'param2': 'value2'}, 
+                'GET',
+                'https://httpbin.org/get',
+                {'Content-Type': 'application/json'},
+                {'param1': 'value1', 'param2': 'value2'},
                 ''
               assertions:
                 - result:status_code = 200
@@ -168,3 +168,255 @@ models:
                 - result:data[1]:jsonrpc = '2.0'
                 - result:data[1]:id = 2
                 - result:data[1]:result = '0x1'
+
+      - name: udf_api_v2
+        tests:
+          # Convenience overloads (always sync)
+          - test_udf:
+              name: test__live_udf_api_v2_post_data_object_sync
+              args: |
+                'https://httpbin.org/post', {'foo': 'bar'}
+              assertions:
+                - result:data.json is not null
+                - result:data.json = OBJECT_CONSTRUCT('foo', 'bar')
+          - test_udf:
+              name: test__live_udf_api_v2_post_data_array_sync
+              args: |
+                'https://httpbin.org/post', ['foo', 'bar']
+              assertions:
+                - result:data.json is not null
+                - result:data.json = ARRAY_CONSTRUCT('foo', 'bar')
+          - test_udf:
+              name: test__live_udf_api_v2_post_data_string_sync
+              args: |
+                'https://httpbin.org/post', 'foo'::VARIANT
+              assertions:
+                - result:data.json is not null
+                - result:data.json = 'foo'
+          - test_udf:
+              name: test__live_udf_api_v2_get_method_sync
+              args: |
+                'https://httpbin.org/get'
+              assertions:
+                - result:status_code = 200
+                - result:data.url = 'https://httpbin.org/get'
+          - test_udf:
+              name: test__live_udf_api_v2_post_batch_jsonrpc_sync
+              args: |
+                'https://httpbin.org/post', {
+                  'jsonrpc': '2.0',
+                  'id': 1,
+                  'method': 'batch',
+                  'params': [
+                    {'id': 1, 'method': 'method1', 'params': {'param1': 'value1'}},
+                    {'id': 2, 'method': 'method2', 'params': {'param2': 'value2'}}
+                  ]
+                }
+              assertions:
+                - result:status_code = 200
+                - result:data.json:jsonrpc = '2.0'
+                - result:data.json:id = 1
+                - result:data.json:method = 'batch'
+                - result:data.json:params is not null
+                - result:data.json:params[0]:id = 1
+                - result:data.json:params[1]:id = 2
+
+          # Full signature tests - SYNC mode
+          - test_udf:
+              name: test__live_udf_api_v2_get_with_params_sync
+              args: |
+                'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1', 'param2': 'value2'}, ''
+              assertions:
+                - result:status_code = 200
+                - result:data.args is not null
+                - result:data.args:param1 = 'value1'
+                - result:data.args:param2 = 'value2'
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_solana_sync
+              args: |
+                'POST',
+                'https://api.mainnet-beta.solana.com',
+                {'Content-Type': 'application/json'},
+                {
+                  'jsonrpc': '2.0',
+                  'id': 1,
+                  'method': 'getVersion'
+                },
+                ''
+              assertions:
+                - result:status_code = 200
+                - result:data.jsonrpc = '2.0'
+                - result:data.id = 1
+                - result:data.result is not null
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_solana_batch_sync
+              args: |
+                'POST',
+                'https://api.mainnet-beta.solana.com',
+                {'Content-Type': 'application/json'},
+                [
+                  {'jsonrpc': '2.0', 'id': 1, 'method': 'getVersion'},
+                  {'jsonrpc': '2.0', 'id': 2, 'method': 'getVersion'}
+                ],
+                ''
+              assertions:
+                - result:status_code = 200
+                - result:data[0]:jsonrpc = '2.0'
+                - result:data[0]:id = 1
+                - result:data[0]:result is not null
+                - result:data[1]:jsonrpc = '2.0'
+                - result:data[1]:id = 2
+                - result:data[1]:result is not null
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_ethereum_batch_sync
+              args: |
+                'POST',
+                'https://ethereum-rpc.publicnode.com',
+                {'Content-Type': 'application/json'},
+                [
+                  {'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []},
+                  {'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []}
+                ],
+                ''
+              assertions:
+                - result:status_code = 200
+                - result:data[0]:jsonrpc = '2.0'
+                - result:data[0]:id = 1
+                - result:data[0]:result is not null
+                - result:data[1]:jsonrpc = '2.0'
+                - result:data[1]:id = 2
+                - result:data[1]:result = '0x1'
+
+          # Full signature tests - ASYNC mode
+          - test_udf:
+              name: test__live_udf_api_v2_get_with_params_async
+              args: |
+                'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, {'param1': 'value1', 'param2': 'value2'}, ''
+              assertions:
+                - result:status_code = 200
+                - result:data.args is not null
+                - result:data.args:param1 = 'value1'
+                - result:data.args:param2 = 'value2'
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_solana_async
+              args: |
+                'POST',
+                'https://api.mainnet-beta.solana.com',
+                {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'},
+                {
+                  'jsonrpc': '2.0',
+                  'id': 1,
+                  'method': 'getVersion'
+                },
+                ''
+              assertions:
+                - result:status_code = 200
+                - result:data.jsonrpc = '2.0'
+                - result:data.id = 1
+                - result:data.result is not null
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_solana_batch_async
+              args: |
+                'POST',
+                'https://api.mainnet-beta.solana.com',
+                {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'},
+                [
+                  {'jsonrpc': '2.0', 'id': 1, 'method': 'getVersion'},
+                  {'jsonrpc': '2.0', 'id': 2, 'method': 'getVersion'}
+                ],
+                ''
+              assertions:
+                - result:status_code = 200
+                - result:data[0]:jsonrpc = '2.0'
+                - result:data[0]:id = 1
+                - result:data[0]:result is not null
+                - result:data[1]:jsonrpc = '2.0'
+                - result:data[1]:id = 2
+                - result:data[1]:result is not null
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_ethereum_batch_async
+              args: |
+                'POST',
+                'https://ethereum-rpc.publicnode.com',
+                {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'},
+                [
+                  {'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []},
+                  {'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []}
+                ],
+                ''
+              assertions:
+                - result:status_code = 200
+                - result:data[0]:jsonrpc = '2.0'
+                - result:data[0]:id = 1
+                - result:data[0]:result is not null
+                - result:data[1]:jsonrpc = '2.0'
+                - result:data[1]:id = 2
+                - result:data[1]:result = '0x1'
+
+          # Explicit is_async boolean parameter tests
+          - test_udf:
+              name: test__live_udf_api_v2_get_with_headers_is_async_true
+              args: |
+                'https://httpbin.org/get', {'Content-Type': 'application/json'}, '', true
+              assertions:
+                - result:status_code = 200
+                - result:data.url = 'https://httpbin.org/get'
+          - test_udf:
+              name: test__live_udf_api_v2_get_with_headers_is_async_false
+              args: |
+                'https://httpbin.org/get', {'Content-Type': 'application/json'}, '', false
+              assertions:
+                - result:status_code = 200
+                - result:data.url = 'https://httpbin.org/get'
+          - test_udf:
+              name: test__live_udf_api_v2_full_signature_is_async_true
+              args: |
+                'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1'}, '', true
+              assertions:
+                - result:status_code = 200
+                - result:data.args is not null
+                - result:data.args:param1 = 'value1'
+          - test_udf:
+              name: test__live_udf_api_v2_full_signature_is_async_false
+              args: |
+                'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1'}, '', false
+              assertions:
+                - result:status_code = 200
+                - result:data.args is not null
+                - result:data.args:param1 = 'value1'
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_is_async_true
+              args: |
+                'POST',
+                'https://api.mainnet-beta.solana.com',
+                {'Content-Type': 'application/json'},
+                {
+                  'jsonrpc': '2.0',
+                  'id': 1,
+                  'method': 'getVersion'
+                },
+                '',
+                true
+              assertions:
+                - result:status_code = 200
+                - result:data.jsonrpc = '2.0'
+                - result:data.id = 1
+                - result:data.result is not null
+          - test_udf:
+              name: test__live_udf_api_v2_post_jsonrpc_is_async_false
+              args: |
+                'POST',
+                'https://api.mainnet-beta.solana.com',
+                {'Content-Type': 'application/json'},
+                {
+                  'jsonrpc': '2.0',
+                  'id': 1,
+                  'method': 'getVersion'
+                },
+                '',
+                false
+              assertions:
+                - result:status_code = 200
+                - result:data.jsonrpc = '2.0'
+                - result:data.id = 1
+                - result:data.result is not null