flip/livequery-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/livequery-models.git synced 2026-02-06 10:56:46 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

add udf_api_batched (#122)

Browse Source 
* add udf_api_batched

* add makefile & max_batch_rows to dbt vars

* update dev endpoint | add udf_api_batched to _live

* add udf_api_batched to live

* add update inner macros to account for max_batch_rows

* fix pointer to livequery_dev

* add udf_api_batched tests
This commit is contained in:
Shah Newaz Khan 2025-05-21 23:01:11 -07:00 committed by GitHub
parent da46dc4a8e
commit b8dc1b09a4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 132 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
Makefile Normal file
View File

@ -0,0 +1,28 @@
SHELL := /bin/bash
dbt-console:
docker-compose run dbt_console
.PHONY: dbt-console
rm_logs:
@if [ -d logs ]; then \
rm -r logs 2>/dev/null || echo "Warning: Could not remove logs directory"; \
else \
echo "Logs directory does not exist"; \
fi
deploy_core: rm_logs
dbt run --select livequery_models.deploy.core.live \
--vars '{UPDATE_UDFS_AND_SPS: true}' \
--profiles-dir ~/.dbt \
--profile livequery \
--target dev
test_core: rm_logs
dbt test --select live \
--profiles-dir ~/.dbt \
--profile livequery \
--target dev

6
dbt_project.yml
View File

@ -67,14 +67,16 @@ vars:
API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}'
EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}'
ROLES: '{{ var("config")[target.name]["ROLES"] }}'
MAX_BATCH_ROWS: '{{ var("config")[target.name]["MAX_BATCH_ROWS"] }}'
config:
# The keys correspond to dbt profiles and are case sensitive
dev:
API_INTEGRATION: AWS_LIVE_QUERY_STG
EXTERNAL_FUNCTION_URI: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/
EXTERNAL_FUNCTION_URI: yn4219e0o6.execute-api.us-east-1.amazonaws.com/stg/
ROLES:
- INTERNAL_DEV
MAX_BATCH_ROWS: 10
prod:
API_INTEGRATION: AWS_LIVE_QUERY
EXTERNAL_FUNCTION_URI: bqco8lkjsb.execute-api.us-east-1.amazonaws.com/prod/
@ -83,8 +85,10 @@ vars:
- VELOCITY_ETHEREUM
- INTERNAL_DEV
- BI_ANALYTICS_READER
MAX_BATCH_ROWS: 10
hosted:
API_INTEGRATION: AWS_LIVEQUERY
EXTERNAL_FUNCTION_URI: dlcb3tpiz8.execute-api.us-east-1.amazonaws.com/hosted/
ROLES:
- DATA_READER
MAX_BATCH_ROWS: 10

17
macros/core/_live.yaml.sql
View File

@ -1,5 +1,21 @@
{% macro config_core__live(schema="_live") %}
- name: {{ schema }}.udf_api_batched
signature:
- [method, STRING]
- [url, STRING]
- [headers, OBJECT]
- [DATA, VARIANT]
- [user_id, STRING]
- [SECRET, STRING]
return_type: VARIANT
func_type: EXTERNAL
api_integration: '{{ var("API_INTEGRATION") }}'
max_batch_rows: '{{ var("MAX_BATCH_ROWS") }}'
options: |
NOT NULL
sql: udf_api
- name: {{ schema }}.udf_api
signature:
- [method, STRING]
@ -14,4 +30,5 @@
options: |
NOT NULL
sql: udf_api
{% endmacro %}

21
macros/core/live.yaml.sql
View File

@ -1,5 +1,26 @@
{% macro config_core_live(schema="live") %}
- name: {{ schema }}.udf_api_batched
signature:
- [method, STRING]
- [url, STRING]
- [headers, OBJECT]
- [data, VARIANT]
- [secret_name, STRING]
return_type: VARIANT
options: |
VOLATILE
sql: |
SELECT
_live.UDF_API(
method,
url,
headers,
data,
_utils.UDF_WHOAMI(),
secret_name
)
- name: {{ schema }}.udf_api
signature:
- [method, STRING]

15
macros/livequery/manage_udfs.sql
View File

@ -33,7 +33,8 @@
sql_,
api_integration = none,
options = none,
func_type = none
func_type = none,
max_batch_rows = none
) %}
CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}(
{{- livequery_models.compile_signature(signature) }}
@ -44,9 +45,12 @@
{{ options }}
{% endif %}
{%- if api_integration -%}
api_integration = {{ api_integration }}
AS {{ livequery_models.construct_api_route(sql_) ~ ";" }}
{% else -%}
api_integration = {{ api_integration -}}
{%- if max_batch_rows -%}
{{ "\n max_batch_rows = " ~ max_batch_rows -}}
{%- endif -%}
{{ "\n AS " ~ livequery_models.construct_api_route(sql_) ~ ";" -}}
{%- else -%}
AS
$$
{{ sql_ }}
@ -65,7 +69,7 @@
{% set options = config ["options"] %}
{% set api_integration = config ["api_integration"] %}
{% set func_type = config ["func_type"] %}
{% set max_batch_rows = config ["max_batch_rows"] %}
{% if not drop_ -%}
{{ livequery_models.create_sql_function(
name_ = name_,
@ -74,6 +78,7 @@
sql_ = sql_,
options = options,
api_integration = api_integration,
max_batch_rows = max_batch_rows,
func_type = func_type
) }}
{%- else -%}

51
models/deploy/core/live.yml
View File

@ -2,6 +2,57 @@ version: 2
models:
- name: live
columns:
- name: udf_api_batched
tests:
- test_udf:
name: test__live_udf_api_batched_post_data_object
args: |
'GET',
'https://httpbin.org/get',
{'Content-Type': 'application/json'},
{'param1': 'value1', 'param2': 'value2'},
''
assertions:
- result:status_code = 200
- result:data.args is not null
- result:data.args:param1 = 'value1'
- result:data.args:param2 = 'value2'
- test_udf:
name: test__live_udf_api_batched_post_jsonrpc_ethereum_batch
args: |
'POST',
'https://ethereum-rpc.publicnode.com',
{'Content-Type': 'application/json'},
[
{'jsonrpc': '2.0', 'id': 1, 'method': 'eth_blockNumber', 'params': []},
{'jsonrpc': '2.0', 'id': 2, 'method': 'eth_chainId', 'params': []}
],
''
assertions:
- result:status_code = 200
- result:data[0]:jsonrpc = '2.0'
- result:data[0]:id = 1
- result:data[0]:result is not null
- result:data[1]:jsonrpc = '2.0'
- result:data[1]:id = 2
- result:data[1]:result = '0x1'
- test_udf:
name: test__live_udf_api_batched_post_jsonrpc_solana
args: |
'POST',
'https://api.mainnet-beta.solana.com',
{'Content-Type': 'application/json'},
{
'jsonrpc': '2.0',
'id': 1,
'method': 'getVersion'
},
''
assertions:
- result:status_code = 200
- result:data.jsonrpc = '2.0'
- result:data.id = 1
- result:data.result is not null
- name: udf_api
tests:
- test_udf: