From b8dc1b09a4d5c3f295610dd3336367d842ad71b0 Mon Sep 17 00:00:00 2001 From: Shah Newaz Khan Date: Wed, 21 May 2025 23:01:11 -0700 Subject: [PATCH] add udf_api_batched (#122) * add udf_api_batched * add makefile & max_batch_rows to dbt vars * update dev endpoint | add udf_api_batched to _live * add udf_api_batched to live * add update inner macros to account for max_batch_rows * fix pointer to livequery_dev * add udf_api_batched tests --- Makefile | 28 ++++++++++++++++++ dbt_project.yml | 6 +++- macros/core/_live.yaml.sql | 17 +++++++++++ macros/core/live.yaml.sql | 21 +++++++++++++ macros/livequery/manage_udfs.sql | 15 ++++++---- models/deploy/core/live.yml | 51 ++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..af35cde --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +SHELL := /bin/bash + +dbt-console: + docker-compose run dbt_console + +.PHONY: dbt-console + +rm_logs: + @if [ -d logs ]; then \ + rm -r logs 2>/dev/null || echo "Warning: Could not remove logs directory"; \ + else \ + echo "Logs directory does not exist"; \ + fi + + +deploy_core: rm_logs + dbt run --select livequery_models.deploy.core.live \ + --vars '{UPDATE_UDFS_AND_SPS: true}' \ + --profiles-dir ~/.dbt \ + --profile livequery \ + --target dev + +test_core: rm_logs + dbt test --select live \ + --profiles-dir ~/.dbt \ + --profile livequery \ + --target dev + diff --git a/dbt_project.yml b/dbt_project.yml index 3d209be..563cd47 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -67,14 +67,16 @@ vars: API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}' EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}' ROLES: '{{ var("config")[target.name]["ROLES"] }}' + MAX_BATCH_ROWS: '{{ var("config")[target.name]["MAX_BATCH_ROWS"] }}' config: # The keys correspond to dbt profiles and are case sensitive dev: API_INTEGRATION: AWS_LIVE_QUERY_STG - EXTERNAL_FUNCTION_URI: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/ + EXTERNAL_FUNCTION_URI: yn4219e0o6.execute-api.us-east-1.amazonaws.com/stg/ ROLES: - INTERNAL_DEV + MAX_BATCH_ROWS: 10 prod: API_INTEGRATION: AWS_LIVE_QUERY EXTERNAL_FUNCTION_URI: bqco8lkjsb.execute-api.us-east-1.amazonaws.com/prod/ @@ -83,8 +85,10 @@ vars: - VELOCITY_ETHEREUM - INTERNAL_DEV - BI_ANALYTICS_READER + MAX_BATCH_ROWS: 10 hosted: API_INTEGRATION: AWS_LIVEQUERY EXTERNAL_FUNCTION_URI: dlcb3tpiz8.execute-api.us-east-1.amazonaws.com/hosted/ ROLES: - DATA_READER + MAX_BATCH_ROWS: 10 \ No newline at end of file diff --git a/macros/core/_live.yaml.sql b/macros/core/_live.yaml.sql index b6c0aa9..52b9ba6 100644 --- a/macros/core/_live.yaml.sql +++ b/macros/core/_live.yaml.sql @@ -1,5 +1,21 @@ {% macro config_core__live(schema="_live") %} +- name: {{ schema }}.udf_api_batched + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [DATA, VARIANT] + - [user_id, STRING] + - [SECRET, STRING] + return_type: VARIANT + func_type: EXTERNAL + api_integration: '{{ var("API_INTEGRATION") }}' + max_batch_rows: '{{ var("MAX_BATCH_ROWS") }}' + options: | + NOT NULL + sql: udf_api + - name: {{ schema }}.udf_api signature: - [method, STRING] @@ -14,4 +30,5 @@ options: | NOT NULL sql: udf_api + {% endmacro %} \ No newline at end of file diff --git a/macros/core/live.yaml.sql b/macros/core/live.yaml.sql index 94b9da2..71628ec 100644 --- a/macros/core/live.yaml.sql +++ b/macros/core/live.yaml.sql @@ -1,5 +1,26 @@ {% macro config_core_live(schema="live") %} +- name: {{ schema }}.udf_api_batched + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [data, VARIANT] + - [secret_name, STRING] + return_type: VARIANT + options: | + VOLATILE + sql: | + SELECT + _live.UDF_API( + method, + url, + headers, + data, + _utils.UDF_WHOAMI(), + secret_name + ) + - name: {{ schema }}.udf_api signature: - [method, STRING] diff --git a/macros/livequery/manage_udfs.sql b/macros/livequery/manage_udfs.sql index 5691ebc..2e87116 100644 --- a/macros/livequery/manage_udfs.sql +++ b/macros/livequery/manage_udfs.sql @@ -33,7 +33,8 @@ sql_, api_integration = none, options = none, - func_type = none + func_type = none, + max_batch_rows = none ) %} CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}( {{- livequery_models.compile_signature(signature) }} @@ -44,9 +45,12 @@ {{ options }} {% endif %} {%- if api_integration -%} - api_integration = {{ api_integration }} - AS {{ livequery_models.construct_api_route(sql_) ~ ";" }} - {% else -%} + api_integration = {{ api_integration -}} + {%- if max_batch_rows -%} + {{ "\n max_batch_rows = " ~ max_batch_rows -}} + {%- endif -%} + {{ "\n AS " ~ livequery_models.construct_api_route(sql_) ~ ";" -}} + {%- else -%} AS $$ {{ sql_ }} @@ -65,7 +69,7 @@ {% set options = config ["options"] %} {% set api_integration = config ["api_integration"] %} {% set func_type = config ["func_type"] %} - + {% set max_batch_rows = config ["max_batch_rows"] %} {% if not drop_ -%} {{ livequery_models.create_sql_function( name_ = name_, @@ -74,6 +78,7 @@ sql_ = sql_, options = options, api_integration = api_integration, + max_batch_rows = max_batch_rows, func_type = func_type ) }} {%- else -%} diff --git a/models/deploy/core/live.yml b/models/deploy/core/live.yml index 7e0a4d8..22b8eed 100644 --- a/models/deploy/core/live.yml +++ b/models/deploy/core/live.yml @@ -2,6 +2,57 @@ version: 2 models: - name: live columns: + - name: udf_api_batched + tests: + - test_udf: + name: test__live_udf_api_batched_post_data_object + args: | + 'GET', + 'https://httpbin.org/get', + {'Content-Type': 'application/json'}, + {'param1': 'value1', 'param2': 'value2'}, + '' + assertions: + - result:status_code = 200 + - result:data.args is not null + - result:data.args:param1 = 'value1' + - result:data.args:param2 = 'value2' + - test_udf: + name: test__live_udf_api_batched_post_jsonrpc_ethereum_batch + args: | + 'POST', + 'https://ethereum-rpc.publicnode.com', + {'Content-Type': 'application/json'}, + [ + {'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []}, + {'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []} + ], + '' + assertions: + - result:status_code = 200 + - result:data[0]:jsonrpc = '2.0' + - result:data[0]:id = 1 + - result:data[0]:result is not null + - result:data[1]:jsonrpc = '2.0' + - result:data[1]:id = 2 + - result:data[1]:result = '0x1' + - test_udf: + name: test__live_udf_api_batched_post_jsonrpc_solana + args: | + 'POST', + 'https://api.mainnet-beta.solana.com', + {'Content-Type': 'application/json'}, + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'getVersion' + }, + '' + assertions: + - result:status_code = 200 + - result:data.jsonrpc = '2.0' + - result:data.id = 1 + - result:data.result is not null - name: udf_api tests: - test_udf: