From bafc04a85675d5a8445de437bfdfe9fa0b33c42e Mon Sep 17 00:00:00 2001 From: Jensen Yap Date: Wed, 13 Aug 2025 02:00:11 +0900 Subject: [PATCH] [STREAM-1324] Enhance UDF compatibility checks and refactor UDF definitions (#128) --- macros/core/_live.yaml.sql | 2 + macros/core/live.yaml.sql | 241 ++++++++++-------- macros/livequery/udf_compatibility.sql | 77 ++++-- macros/marketplace/alchemy/util_udfs.yaml.sql | 16 +- macros/marketplace/claude/util_udfs.yaml.sql | 12 +- .../marketplace/github/actions_udtfs.yml.sql | 20 +- macros/marketplace/github/utils_udfs.yaml.sql | 18 +- macros/marketplace/helius/apis_udfs.yaml.sql | 18 +- macros/marketplace/helius/util_udfs.yaml.sql | 12 +- macros/marketplace/slack/utils_udfs.yaml.sql | 6 +- models/deploy/core/_external_access.sql | 2 + models/deploy/core/live.yml | 22 +- 12 files changed, 249 insertions(+), 197 deletions(-) diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index 49ceba4..58f5c26 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -31,6 +31,7 @@ NOT NULL sql: udf_api +{% if is_udf_api_v2_compatible() %} - name: {{ schema }}.udf_api_sync signature: - [method, STRING] @@ -66,5 +67,6 @@ options: | NOT NULL sql: 'v2/udf_api' +{% endif %} {% endmacro %} diff --git a/macros/core/live.yaml.sql b/macros/core/live.yaml.sql index b573c39..e7146e2 100644 --- a/macros/core/live.yaml.sql +++ b/macros/core/live.yaml.sql @@ -154,38 +154,7 @@ COMMENT = $$Returns a list of allowed domains.$$ sql: allowed -- name: {{ schema }}.udf_api_v2 - signature: - - [url, STRING] - - [headers, OBJECT] - - [secret_name, STRING] - return_type: VARIANT - options: | - VOLATILE - COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ - sql: | - SELECT - utils.udf_redirect_s3_presigned_url( - _live.udf_api_async('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) - :s3_presigned_url::STRING - ):data[0][1] as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING - )) = 'async' - - UNION ALL - - SELECT - _live.udf_api_sync('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, - 'sync' - )) != 'async' - +{% if is_udf_api_v2_compatible() %} - name: {{ schema }}.udf_api_v2 signature: - [url, STRING] @@ -198,60 +167,19 @@ COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ sql: | SELECT - utils.udf_redirect_s3_presigned_url( - _live.udf_api_async('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) - :s3_presigned_url::STRING - ):data[0][1] as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING - )) = 'async' - - UNION ALL - - SELECT - _live.udf_api_sync('GET', url, headers, {}, _utils.UDF_WHOAMI(), secret_name) as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, - 'sync' - )) != 'async' - -- name: {{ schema }}.udf_api_v2 - signature: - - [method, STRING] - - [url, STRING] - - [headers, OBJECT] - - [data, VARIANT] - - [secret_name, STRING] - return_type: VARIANT - options: | - VOLATILE - COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ - sql: | - SELECT - utils.udf_redirect_s3_presigned_url( - _live.udf_api_async(method, url, headers, data, _utils.UDF_WHOAMI(), secret_name) - :s3_presigned_url::STRING - ):data[0][1] as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING - )) = 'async' - - UNION ALL - - SELECT - _live.udf_api_sync(method, url, headers, data, _utils.UDF_WHOAMI(), secret_name) as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, - 'sync' - )) != 'async' + CASE is_async + WHEN TRUE + THEN + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async( + 'GET', URL, HEADERS, {}, _utils.UDF_WHOAMI(), SECRET_NAME + ):s3_presigned_url :: STRING + ):data[0][1] + ELSE + _live.udf_api_sync( + 'GET', URL, HEADERS, {}, _utils.UDF_WHOAMI(), SECRET_NAME + ) + END - name: {{ schema }}.udf_api_v2 signature: @@ -287,32 +215,26 @@ - [url, STRING] - [headers, OBJECT] - [data, VARIANT] + - [is_async, BOOLEAN] return_type: VARIANT options: | VOLATILE COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ sql: | SELECT - utils.udf_redirect_s3_presigned_url( - _live.udf_api_async(method, url, headers, data, _utils.UDF_WHOAMI(), '') - :s3_presigned_url::STRING - ):data[0][1] as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING - )) = 'async' - - UNION ALL - - SELECT - _live.udf_api_sync(method, url, headers, data, _utils.UDF_WHOAMI(), '') as result - WHERE LOWER(COALESCE( - headers:"fsc-quantum-execution-mode"::STRING, - headers:"Fsc-Quantum-Execution-Mode"::STRING, - headers:"FSC-QUANTUM-EXECUTION-MODE"::STRING, - 'sync' - )) != 'async' + CASE is_async + WHEN TRUE + THEN + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async( + METHOD, URL, HEADERS, DATA, _utils.UDF_WHOAMI(), '' + ):s3_presigned_url :: STRING + ):data[0][1] + ELSE + _live.udf_api_sync( + METHOD, URL, HEADERS, DATA, _utils.UDF_WHOAMI(), '' + ) + END - name: {{ schema }}.udf_api_v2 signature: @@ -321,7 +243,7 @@ return_type: VARIANT options: | VOLATILE - COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ + COMMENT = $$Executes a Quick Post LiveQuery Sync External Function.$$ sql: | SELECT _live.udf_api_sync( @@ -337,11 +259,36 @@ signature: - [url, STRING] - [data, VARIANT] - - [secret_name, STRING] + - [is_async, BOOLEAN] return_type: VARIANT options: | VOLATILE COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ + sql: | + SELECT + CASE is_async + WHEN TRUE + THEN + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async( + 'GET', URL, {'Content-Type': 'application/json'}, data, _utils.UDF_WHOAMI(), '' + ):s3_presigned_url :: STRING + ):data[0][1] + ELSE + _live.udf_api_sync( + 'GET', URL, {'Content-Type': 'application/json'}, data, _utils.UDF_WHOAMI(), '' + ) + END + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [data, VARIANT] + - [secret_name, STRING] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes a Quick Post LiveQuery Sync External Function.$$ sql: | SELECT _live.udf_api_sync( @@ -356,10 +303,36 @@ - name: {{ schema }}.udf_api_v2 signature: - [url, STRING] + - [data, VARIANT] + - [secret_name, STRING] + - [is_async, BOOLEAN] return_type: VARIANT options: | VOLATILE COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ + sql: | + SELECT + CASE is_async + WHEN TRUE + THEN + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async( + 'GET', URL, {'Content-Type': 'application/json'}, data, _utils.UDF_WHOAMI(), secret_name + ):s3_presigned_url :: STRING + ):data[0][1] + ELSE + _live.udf_api_sync( + 'GET', URL, {'Content-Type': 'application/json'}, data, _utils.UDF_WHOAMI(), secret_name + ) + END + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes a Quick GET LiveQuery Sync External Function.$$ sql: | SELECT _live.udf_api_sync( @@ -374,11 +347,35 @@ - name: {{ schema }}.udf_api_v2 signature: - [url, STRING] - - [secret_name, STRING] + - [is_async, BOOLEAN] return_type: VARIANT options: | VOLATILE COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ + sql: | + SELECT + CASE is_async + WHEN TRUE + THEN + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async( + 'GET', URL, {'Content-Type': 'application/json'}, {}, _utils.UDF_WHOAMI(), '' + ):s3_presigned_url :: STRING + ):data[0][1] + ELSE + _live.udf_api_sync( + 'GET', URL, {'Content-Type': 'application/json'}, {}, _utils.UDF_WHOAMI(), '' + ) + END + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [secret_name, STRING] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes a Quick GET LiveQuery Sync External Function.$$ sql: | SELECT _live.udf_api_sync( @@ -389,4 +386,30 @@ _utils.UDF_WHOAMI(), secret_name ) + +- name: {{ schema }}.udf_api_v2 + signature: + - [url, STRING] + - [secret_name, STRING] + - [is_async, BOOLEAN] + return_type: VARIANT + options: | + VOLATILE + COMMENT = $$Executes an LiveQuery Sync or Async External Function.$$ + sql: | + SELECT + CASE is_async + WHEN TRUE + THEN + utils.udf_redirect_s3_presigned_url( + _live.udf_api_async( + 'GET', URL, {'Content-Type': 'application/json'}, {}, _utils.UDF_WHOAMI(), secret_name + ):s3_presigned_url :: STRING + ):data[0][1] + ELSE + _live.udf_api_sync( + 'GET', URL, {'Content-Type': 'application/json'}, {}, _utils.UDF_WHOAMI(), secret_name + ) + END +{% endif %} {% endmacro %} diff --git a/macros/livequery/udf_compatibility.sql b/macros/livequery/udf_compatibility.sql index 15478aa..e24d5c8 100644 --- a/macros/livequery/udf_compatibility.sql +++ b/macros/livequery/udf_compatibility.sql @@ -1,26 +1,63 @@ -{% macro check_udf_api_v2_exists() -%} -{# - Check if live.udf_api_v2 function exists at compile time - Returns true/false to control which UDF call to render - - Usage: - {% set v2_exists = check_udf_api_v2_exists() %} - {% if v2_exists %} - live.udf_api_v2(...) - {% else %} - live.udf_api(...) - {% endif %} -#} -{% set check_v2_query %} - SELECT COUNT(*) FROM information_schema.functions - WHERE function_name = 'UDF_API_V2' AND function_schema = 'LIVE' +{% macro get_streamline_stack_version() -%} +{# Determine the environment based on target.name #} +{% if target.name == 'dev' %} + {% set env = 'stg' %} +{% elif target.name == 'prod' %} + {% set env = 'prod' %} +{% else %} + {% set env = 'stg' %} +{% endif %} + +{# Extract database prefix if it follows pattern _ #} +{% set database_parts = target.database.split('_') %} +{% if database_parts|length > 1 and database_parts[-1].lower() == target.name.lower() %} + {% set database_prefix = database_parts[:-1]|join('_') %} +{% else %} + {% set database_prefix = target.database %} +{% endif %} + + +{% set streamline_stack_version_query %} + SELECT + TAGS:streamline_runtime_version::STRING as runtime_version, + TAGS:streamline_infrastructure_version::STRING as infrastructure_version + FROM TABLE(STREAMLINE.AWS.CLOUDFORMATION_DESCRIBE_STACKS('{{ env }}', '{{ database_prefix.lower() }}-api-{{ env }}')); {% endset %} {% if execute %} - {% set v2_exists = run_query(check_v2_query).rows[0][0] > 0 %} + {% set result = run_query(streamline_stack_version_query) %} + {% if result.rows|length > 0 %} + {% set runtime_version = result.rows[0][0] %} + {% set infrastructure_version = result.rows[0][1] %} + {% set streamline_stack_version = {'runtime_version': runtime_version, 'infrastructure_version': infrastructure_version} %} + {% else %} + {% set streamline_stack_version = {'runtime_version': false, 'infrastructure_version': false} %} + {% endif %} {% else %} - {% set v2_exists = false %} + {% set streamline_stack_version = {'runtime_version': false, 'infrastructure_version': false} %} {% endif %} -{{ return(v2_exists) }} -{%- endmacro -%} \ No newline at end of file +{{ return(streamline_stack_version) }} +{%- endmacro -%} + +{% macro is_udf_api_v2_compatible() -%} +{% set versions = get_streamline_stack_version() %} + +{% if execute and versions.runtime_version %} + {# Extract version number from runtime_version string (e.g., "v3.1.2" -> "3.1.2") #} + {% set version_str = versions.runtime_version.replace('v', '') %} + {% set version_parts = version_str.split('.') %} + + {# Convert to comparable format: major.minor.patch #} + {% set major = version_parts[0] | int %} + {% set minor = version_parts[1] | int if version_parts|length > 1 else 0 %} + {% set patch = version_parts[2] | int if version_parts|length > 2 else 0 %} + + {# Check if version is >= 3.0.0 #} + {% set is_compatible = major >= 3 %} +{% else %} + {% set is_compatible = false %} +{% endif %} + +{{ return(is_compatible) }} +{%- endmacro -%} diff --git a/macros/marketplace/alchemy/util_udfs.yaml.sql b/macros/marketplace/alchemy/util_udfs.yaml.sql index 95a77c7..9aa52be 100644 --- a/macros/marketplace/alchemy/util_udfs.yaml.sql +++ b/macros/marketplace/alchemy/util_udfs.yaml.sql @@ -13,8 +13,8 @@ options: | COMMENT = $$Used to issue a 'GET' request to the Alchemy NFT API.$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'GET', @@ -52,8 +52,8 @@ options: | COMMENT = $$Used to issue a 'GET' request to the Alchemy NFT API.$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'GET', @@ -90,8 +90,8 @@ options: | COMMENT = $$Used to issue a 'POST' request to the Alchemy NFT API.$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', @@ -122,8 +122,8 @@ options: | COMMENT = $$Used to issue an RPC call to Alchemy.$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', diff --git a/macros/marketplace/claude/util_udfs.yaml.sql b/macros/marketplace/claude/util_udfs.yaml.sql index 189d944..20f4426 100644 --- a/macros/marketplace/claude/util_udfs.yaml.sql +++ b/macros/marketplace/claude/util_udfs.yaml.sql @@ -11,8 +11,8 @@ options: | COMMENT = $$Make calls to Claude API [API docs: Claude](https://docs.anthropic.com/claude/reference/getting-started-with-the-api)$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', @@ -55,8 +55,8 @@ options: | COMMENT = $$Make GET requests to Claude API [API docs: Get](https://docs.anthropic.com/claude/reference/get)$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'GET', @@ -99,8 +99,8 @@ options: | COMMENT = $$Make DELETE requests to Claude API [API docs: Delete](https://docs.anthropic.com/claude/reference/delete)$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'DELETE', diff --git a/macros/marketplace/github/actions_udtfs.yml.sql b/macros/marketplace/github/actions_udtfs.yml.sql index 512978c..98da147 100644 --- a/macros/marketplace/github/actions_udtfs.yml.sql +++ b/macros/marketplace/github/actions_udtfs.yml.sql @@ -240,23 +240,11 @@ conclusion AS job_conclusion, html_url AS job_url, workflow_name, - steps AS failed_steps + steps AS failed_steps, + {{ schema_name -}}.job_logs(owner, repo, job_id::TEXT) AS logs FROM TABLE({{ schema_name -}}.tf_workflow_run_jobs(owner, repo, run_id)) WHERE conclusion = 'failure' ), - jobs_with_logs AS ( - SELECT - run_id, - job_id, - job_name, - job_status, - job_conclusion, - job_url, - workflow_name, - failed_steps, - {{ schema_name -}}.job_logs(owner, repo, job_id::TEXT) AS logs - FROM failed_jobs - ), error_sections AS ( SELECT run_id, @@ -269,7 +257,7 @@ failed_steps, logs, ARRAY_AGG(section.value) AS failed_step_logs - FROM jobs_with_logs, + FROM failed_jobs, LATERAL FLATTEN(INPUT => SPLIT(logs, '##[group]')) section WHERE CONTAINS(section.value, '##[error]') GROUP BY run_id, job_id, job_name, job_status, job_conclusion, job_url, workflow_name, failed_steps, logs @@ -285,7 +273,7 @@ failed_steps, logs, COALESCE(failed_step_logs, ARRAY_CONSTRUCT()) AS failed_step_logs - FROM jobs_with_logs + FROM failed_jobs LEFT JOIN error_sections USING (run_id, job_id) - name: {{ schema_name -}}.tf_failure_analysis_with_ai diff --git a/macros/marketplace/github/utils_udfs.yaml.sql b/macros/marketplace/github/utils_udfs.yaml.sql index f2dc1d2..a26bfd9 100644 --- a/macros/marketplace/github/utils_udfs.yaml.sql +++ b/macros/marketplace/github/utils_udfs.yaml.sql @@ -10,13 +10,13 @@ options: | COMMENT = $$Verify token [Authenticating to the REST API](https://docs.github.com/en/rest/overview/authenticating-to-the-rest-api?apiVersion=2022-11-28).$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'GET', 'https://api.github.com/octocat', - {'Authorization': 'Bearer {TOKEN}', 'X-GitHub-Api-Version': '2022-11-28', 'fsc-quantum-execution-mode': 'async'}, + {'Authorization': 'Bearer {TOKEN}', 'X-GitHub-Api-Version': '2022-11-28'}, {}, IFF(_utils.udf_whoami() <> CURRENT_USER(), '_FSC_SYS/GITHUB', 'Vault/github/api'), TRUE @@ -55,8 +55,8 @@ options: | COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'GET', @@ -85,8 +85,8 @@ options: | COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', @@ -115,8 +115,8 @@ options: | COMMENT = $$List all workflow runs for a workflow. You can replace workflow_id with the workflow file name. You can use parameters to narrow the list of results. [Docs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-workflow).$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'PUT', diff --git a/macros/marketplace/helius/apis_udfs.yaml.sql b/macros/marketplace/helius/apis_udfs.yaml.sql index 894f140..67d91eb 100644 --- a/macros/marketplace/helius/apis_udfs.yaml.sql +++ b/macros/marketplace/helius/apis_udfs.yaml.sql @@ -22,15 +22,15 @@ options: | COMMENT = $$Returns the native Solana balance (in lamports) and all token balances for a given address. [Helius docs here](https://docs.helius.xyz/solana-apis/balances-api).$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'GET', - CASE - WHEN NETWORK = 'devnet' THEN + CASE + WHEN NETWORK = 'devnet' THEN concat('https://api-devnet.helius.xyz/v0/addresses/', ADDRESS, '/balances?api-key={API_KEY}') - ELSE + ELSE concat('https://api.helius.xyz/v0/addresses/', ADDRESS, '/balances?api-key={API_KEY}') END, {'fsc-quantum-execution-mode': 'async'}, @@ -41,10 +41,10 @@ {%- else -%} live.udf_api( 'GET', - CASE - WHEN NETWORK = 'devnet' THEN + CASE + WHEN NETWORK = 'devnet' THEN concat('https://api-devnet.helius.xyz/v0/addresses/', ADDRESS, '/balances?api-key={API_KEY}') - ELSE + ELSE concat('https://api.helius.xyz/v0/addresses/', ADDRESS, '/balances?api-key={API_KEY}') END, {}, @@ -65,4 +65,4 @@ sql: | SELECT {{ utils_schema_name -}}.post_api(NETWORK, '/v0/transactions', {'transactions': TRANSACTIONS}) as response -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/marketplace/helius/util_udfs.yaml.sql b/macros/marketplace/helius/util_udfs.yaml.sql index 947f6dd..27800a6 100644 --- a/macros/marketplace/helius/util_udfs.yaml.sql +++ b/macros/marketplace/helius/util_udfs.yaml.sql @@ -13,8 +13,8 @@ options: | COMMENT = $$Used to issue an HTTP GET request to Helius.$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'GET', @@ -55,8 +55,8 @@ options: | COMMENT = $$Used to issue an HTTP POST request to Helius.$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', @@ -97,8 +97,8 @@ options: | COMMENT = $$Used to issue an RPC call to Helius.$$ sql: | - SELECT - {% set v2_exists = check_udf_api_v2_exists() %} + SELECT + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', diff --git a/macros/marketplace/slack/utils_udfs.yaml.sql b/macros/marketplace/slack/utils_udfs.yaml.sql index df1dabf..7ed1457 100644 --- a/macros/marketplace/slack/utils_udfs.yaml.sql +++ b/macros/marketplace/slack/utils_udfs.yaml.sql @@ -17,7 +17,7 @@ WHEN PAYLOAD IS NULL THEN OBJECT_CONSTRUCT('ok', false, 'error', 'payload is required') ELSE - {% set v2_exists = check_udf_api_v2_exists() %} + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', @@ -58,7 +58,7 @@ WHEN PAYLOAD IS NULL THEN OBJECT_CONSTRUCT('ok', false, 'error', 'payload is required') ELSE - {% set v2_exists = check_udf_api_v2_exists() %} + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', @@ -123,7 +123,7 @@ WHEN PAYLOAD IS NULL THEN OBJECT_CONSTRUCT('ok', false, 'error', 'payload is required') ELSE - {% set v2_exists = check_udf_api_v2_exists() %} + {% set v2_exists = is_udf_api_v2_compatible() %} {% if v2_exists -%} live.udf_api_v2( 'POST', diff --git a/models/deploy/core/_external_access.sql b/models/deploy/core/_external_access.sql index 347dc06..2740d86 100644 --- a/models/deploy/core/_external_access.sql +++ b/models/deploy/core/_external_access.sql @@ -1,4 +1,6 @@ +{% if is_udf_api_v2_compatible() %} {{ create_s3_express_external_access_integration() }} +{% endif %} -- this is to pass the model render as dbt dependency in other models -- livequery will need s3 express access to read from the s3 bucket diff --git a/models/deploy/core/live.yml b/models/deploy/core/live.yml index 0a3f0bc..c8fce38 100644 --- a/models/deploy/core/live.yml +++ b/models/deploy/core/live.yml @@ -231,7 +231,7 @@ models: - test_udf: name: test__live_udf_api_v2_get_with_params_sync args: | - 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1', 'param2': 'value2'}, '' + 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1', 'param2': 'value2'}, FALSE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data.args is not null ELSE true END @@ -248,7 +248,7 @@ models: 'id': 1, 'method': 'getVersion' }, - '' + FALSE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data.jsonrpc = '2.0' ELSE true END @@ -264,7 +264,7 @@ models: {'jsonrpc': '2.0', 'id': 1, 'method': 'getVersion'}, {'jsonrpc': '2.0', 'id': 2, 'method': 'getVersion'} ], - '' + FALSE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data[0]:jsonrpc = '2.0' ELSE true END @@ -283,7 +283,7 @@ models: {'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []}, {'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []} ], - '' + FALSE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data[0]:jsonrpc = '2.0' ELSE true END @@ -297,7 +297,7 @@ models: - test_udf: name: test__live_udf_api_v2_get_with_params_async args: | - 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, {'param1': 'value1', 'param2': 'value2'}, '' + 'GET', 'https://httpbin.org/get', {'Content-Type': 'application/json'}, {'param1': 'value1', 'param2': 'value2'}, TRUE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data.args is not null ELSE true END @@ -308,13 +308,13 @@ models: args: | 'POST', 'https://api.mainnet-beta.solana.com', - {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, + {'Content-Type': 'application/json'}, { 'jsonrpc': '2.0', 'id': 1, 'method': 'getVersion' }, - '' + TRUE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data.jsonrpc = '2.0' ELSE true END @@ -325,12 +325,12 @@ models: args: | 'POST', 'https://api.mainnet-beta.solana.com', - {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, + {'Content-Type': 'application/json'}, [ {'jsonrpc': '2.0', 'id': 1, 'method': 'getVersion'}, {'jsonrpc': '2.0', 'id': 2, 'method': 'getVersion'} ], - '' + TRUE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data[0]:jsonrpc = '2.0' ELSE true END @@ -344,12 +344,12 @@ models: args: | 'POST', 'https://ethereum-rpc.publicnode.com', - {'Content-Type': 'application/json', 'fsc-quantum-execution-mode': 'async'}, + {'Content-Type': 'application/json'}, [ {'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []}, {'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []} ], - '' + TRUE assertions: - result:status_code IN (200, 502, 503) - CASE WHEN result:status_code = 200 THEN result:data[0]:jsonrpc = '2.0' ELSE true END