From 37e5f5693c38b274845caf12bd7330681054caaf Mon Sep 17 00:00:00 2001 From: shah Date: Mon, 14 Apr 2025 13:09:11 -0700 Subject: [PATCH] add deploy/core & updated test_udf --- data/live__verified_api_endpoints.csv | 12 ++ models/deploy/core/_live.sql | 2 + models/deploy/core/_utils.sql | 2 + models/deploy/core/_utils.yml | 11 ++ models/deploy/core/live.sql | 5 + models/deploy/core/live.yml | 27 +++ models/deploy/core/secrets.sql | 4 + models/deploy/core/utils.sql | 3 + models/deploy/core/utils.yml | 269 ++++++++++++++++++++++++++ models/live__allow_list.sql | 11 ++ packages.yml | 3 +- tests/generic/test_udf.sql | 37 +++- 12 files changed, 379 insertions(+), 7 deletions(-) create mode 100644 data/live__verified_api_endpoints.csv create mode 100644 models/deploy/core/_live.sql create mode 100644 models/deploy/core/_utils.sql create mode 100644 models/deploy/core/_utils.yml create mode 100644 models/deploy/core/live.sql create mode 100644 models/deploy/core/live.yml create mode 100644 models/deploy/core/secrets.sql create mode 100644 models/deploy/core/utils.sql create mode 100644 models/deploy/core/utils.yml create mode 100644 models/live__allow_list.sql diff --git a/data/live__verified_api_endpoints.csv b/data/live__verified_api_endpoints.csv new file mode 100644 index 0000000..00db711 --- /dev/null +++ b/data/live__verified_api_endpoints.csv @@ -0,0 +1,12 @@ +Project,Endpoint,Type,Documentation +Snapshot,https://hub.snapshot.org/graphql,GraphQL,https://docs.snapshot.org/graphql-api +Defillama,https://api.llama.fi/,REST,https://defillama.com/docs/api +Defillama,https://yields.llama.fi/,REST,https://defillama.com/docs/api +Defillama,https://stablecoins.llama.fi/,REST,https://defillama.com/docs/api +Defillama,https://bridges.llama.fi/,REST,https://defillama.com/docs/api +Defillama,https://coins.llama.fi/,REST,https://defillama.com/docs/api +zkSync,https://api.zksync.io/api/v0.2/,REST,https://docs.zksync.io/apiv02-docs/ +DeepNFT Value,https://api.deepnftvalue.com/v1,REST,https://deepnftvalue.readme.io/reference/getting-started-with-deepnftvalue-api +Zapper,https://api.zapper.fi/v2/,REST,https://api.zapper.fi/api/static/index.html#/Apps/AppsController_getApps +Helius,https://api.helius.xyz,REST,https://docs.helius.xyz/introduction/why-helius +Stargaze Name Service,https://rest.stargaze-apis.com,REST,https://github.com/public-awesome/names/blob/main/API.md \ No newline at end of file diff --git a/models/deploy/core/_live.sql b/models/deploy/core/_live.sql new file mode 100644 index 0000000..f37331e --- /dev/null +++ b/models/deploy/core/_live.sql @@ -0,0 +1,2 @@ +{% set config = config_core__live %} +{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/_utils.sql b/models/deploy/core/_utils.sql new file mode 100644 index 0000000..ac40ddf --- /dev/null +++ b/models/deploy/core/_utils.sql @@ -0,0 +1,2 @@ +{% set config = config_core__utils %} +{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/_utils.yml b/models/deploy/core/_utils.yml new file mode 100644 index 0000000..80fcb8b --- /dev/null +++ b/models/deploy/core/_utils.yml @@ -0,0 +1,11 @@ +version: 2 +models: + - name: _utils + columns: + - name: udf_introspect + tests: + - test_udf: + name: test___utils_udf_introspect + args: "'hello'" + assertions: + - result = 'hello' \ No newline at end of file diff --git a/models/deploy/core/live.sql b/models/deploy/core/live.sql new file mode 100644 index 0000000..9e088f2 --- /dev/null +++ b/models/deploy/core/live.sql @@ -0,0 +1,5 @@ +-- depends_on: {{ ref('_utils') }} +-- depends_on: {{ ref('utils') }} +-- depends_on: {{ ref('_live') }} +{% set config = config_core_live %} +{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/live.yml b/models/deploy/core/live.yml new file mode 100644 index 0000000..cb4bbf1 --- /dev/null +++ b/models/deploy/core/live.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: live + columns: + - name: udf_api + tests: + - test_udf: + name: test__live_udf_api_post_data_object + args: | + 'https://httpbin.org/post', {'foo': 'bar'} + assertions: + - result:data.json is not null + - result:data.json = OBJECT_CONSTRUCT('foo', 'bar') + - test_udf: + name: test__live_udf_api_post_data_array + args: | + 'https://httpbin.org/post', ['foo', 'bar'] + assertions: + - result:data.json is not null + - result:data.json = ARRAY_CONSTRUCT('foo', 'bar') + - test_udf: + name: test__live_udf_api_post_data_string + args: | + 'https://httpbin.org/post', 'foo'::VARIANT + assertions: + - result:data.json is not null + - result:data.json = 'foo' \ No newline at end of file diff --git a/models/deploy/core/secrets.sql b/models/deploy/core/secrets.sql new file mode 100644 index 0000000..668ca14 --- /dev/null +++ b/models/deploy/core/secrets.sql @@ -0,0 +1,4 @@ +-- depends_on: {{ ref('_utils') }} +-- depends_on: {{ ref('live') }} +{% set config = config_core_secrets %} +{{ ephemeral_deploy_core(config) }} \ No newline at end of file diff --git a/models/deploy/core/utils.sql b/models/deploy/core/utils.sql new file mode 100644 index 0000000..fbabfe1 --- /dev/null +++ b/models/deploy/core/utils.sql @@ -0,0 +1,3 @@ + -- depends_on: {{ ref('_utils') }} +{% set config = config_core_utils %} +{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/utils.yml b/models/deploy/core/utils.yml new file mode 100644 index 0000000..1d46b78 --- /dev/null +++ b/models/deploy/core/utils.yml @@ -0,0 +1,269 @@ +version: 2 +models: + - name: utils + columns: + - name: udf_json_rpc_call + tests: + - test_udf: + name: test__utils_udf_json_rpc_call + args: "'foo', [], 1" + assertions: + - > + result = { + 'jsonrpc': '2.0', + 'method': 'foo', + 'params': [], + 'id': '1' + } + - name: udf_urlencode + tests: + - test_udf: + name: test__utils_udf_urlencode_dict_true_1 + args: > + {'a':'b'}, TRUE + assertions: + - result = 'a=b' + - test_udf: + name: test__utils_udf_urlencode_dict_true_2 + args: > + {'a':'a', 'b':'b'}, TRUE + assertions: + - result = 'a=a&b=b' + - test_udf: + name: test__utils_udf_urlencode_dict_true_space + args: > + {'space': 'abc 123'}, TRUE + assertions: + - result = 'space=abc+123' + - test_udf: + name: test__utils_udf_urlencode_dict_true_special + args: > + {'special!': ' !@#$,+"'}, TRUE + assertions: + - result = 'special%21=+%21%40%23%24%2C%2B%22' + - test_udf: + name: test__utils_udf_urlencode_dict_true_array + args: > + {'array': [0, 1, 2]}, TRUE + assertions: + - result = 'array=0&array=1&array=2' + - test_udf: + name: test__utils_udf_urlencode_dict_false_1 + args: > + {'a':'b'}, FALSE + assertions: + - result = 'a=b' + - test_udf: + name: test__utils_udf_urlencode_dict_false_2 + args: > + {'a':'b', 'b':'b'}, FALSE + assertions: + - result = 'a=b&b=b' + - test_udf: + name: test__utils_udf_urlencode_dict_false_space + args: > + {'space': 'abc 123'}, FALSE + assertions: + - result = 'space=abc+123' + - test_udf: + name: test__utils_udf_urlencode_dict_false_special + args: > + {'special!': ' !@#$,+"'}, FALSE + assertions: + - result = 'special%21=+%21%40%23%24%2C%2B%22' + - test_udf: + name: test__utils_udf_urlencode_dict_false_array + args: > + {'array': [0, 1, 2]}, FALSE + assertions: + - result = 'array=%5B0%2C+1%2C+2%5D' + - test_udf: + name: test__utils_udf_urlencode_dict_1 + args: > + {'a':'b'} + assertions: + - result = 'a=b' + - test_udf: + name: test__utils_udf_urlencode_dict_2 + args: > + {'a':'b', 'b':'b'} + assertions: + - result = 'a=b&b=b' + - test_udf: + name: test__utils_udf_urlencode_dict_space + args: > + {'space': 'abc 123'} + assertions: + - result = 'space=abc+123' + - test_udf: + name: test__utils_udf_urlencode_dict_special + args: > + {'special!': ' !@#$,+"'} + assertions: + - result = 'special%21=+%21%40%23%24%2C%2B%22' + - test_udf: + name: test__utils_udf_urlencode_dict_array + args: > + {'array': [0, 1, 2]} + assertions: + - result = 'array=%5B0%2C+1%2C+2%5D' + # write tests but use arrays of arrays instead of dictionaries + - test_udf: + name: test__utils_udf_urlencode_array_true_1 + args: > + [['a', 'b']], TRUE + assertions: + - result = 'a=b' + - test_udf: + name: test__utils_udf_urlencode_array_true_2 + args: > + [['a', 'a'], ['b', 'b']], TRUE + assertions: + - result = 'a=a&b=b' + - test_udf: + name: test__utils_udf_urlencode_array_true_space + args: > + [['space', 'abc 123']], TRUE + assertions: + - result = 'space=abc+123' + - test_udf: + name: test__utils_udf_urlencode_array_true_special + args: > + [['special!', ' !@#$,+"']], TRUE + assertions: + - result = 'special%21=+%21%40%23%24%2C%2B%22' + - test_udf: + name: test__utils_udf_urlencode_array_true_array + args: > + [['array', [0, 1, 2]]], TRUE + assertions: + - result = 'array=0&array=1&array=2' + - test_udf: + name: test__utils_udf_urlencode_array_false_1 + args: > + [['a', 'b']], FALSE + assertions: + - result = 'a=b' + - test_udf: + name: test__utils_udf_urlencode_array_false_2 + args: > + [['a', 'a'], ['b', 'b']], FALSE + assertions: + - result = 'a=a&b=b' + - test_udf: + name: test__utils_udf_urlencode_array_false_space + args: > + [['space', 'abc 123']], FALSE + assertions: + - result = 'space=abc+123' + - test_udf: + name: test__utils_udf_urlencode_array_false_special + args: > + [['special!', ' !@#$,+"']], FALSE + assertions: + - result = 'special%21=+%21%40%23%24%2C%2B%22' + - test_udf: + name: test__utils_udf_urlencode_array_false_array + args: > + [['array', [0, 1, 2]]], FALSE + assertions: + - result = 'array=%5B0%2C+1%2C+2%5D' + - test_udf: + name: test__utils_udf_urlencode_array_1 + args: > + [['a', 'b']] + assertions: + - result = 'a=b' + - test_udf: + name: test__utils_udf_urlencode_array_2 + args: > + [['a', 'a'], ['b', 'b']] + assertions: + - result = 'a=a&b=b' + - test_udf: + name: test__utils_udf_urlencode_array_space + args: > + [['space', 'abc 123']] + assertions: + - result = 'space=abc+123' + - test_udf: + name: test__utils_udf_urlencode_array_special + args: > + [['special!', ' !@#$,+"']] + assertions: + - result = 'special%21=+%21%40%23%24%2C%2B%22' + - test_udf: + name: test__utils_udf_urlencode_array_array + args: > + [['array', [0, 1, 2]]] + assertions: + - result = 'array=%5B0%2C+1%2C+2%5D' + - name: udf_int_to_binary + tests: + - test_udf: + name: test__utils_udf_int_to_binary + args: 123456789 + assertions: + - result = '111010110111100110100010101' + - name: udf_int_to_binary + tests: + - test_udf: + name: test__utils_udf_int_to_binary_large_number + args: "'123456789123456789123456789123456789123456789'" + assertions: + - result = '101100010010011011011100101001111010001001110011010111111101111101010111011001001101000001111110001010100001011011010000100000001000101111100010101' + - name: udf_binary_to_int + tests: + - test_udf: + name: test__utils_udf_binary_to_int + args: '111010110111100110100010101' + assertions: + - result = '123456789' + - name: udf_binary_to_int + tests: + - test_udf: + name: test__utils_udf_binary_to_int_large_number + args: "'110110110100110110100101110101100110100000101111100010101'" + assertions: + - result = '123456789123456789' + - name: udf_evm_decode_log + tests: + - test_udf: + name: test__utils_udf_evm_decode_log + args: > + { + 'anonymous': false, + 'inputs': [ + { + 'indexed': false, + 'name': 'nonce', + 'type': 'uint256' + } + ], + 'name': 'NonceChanged', + 'type': 'event' + }::variant, + { + 'address': '0x55032650b14df07b85bf18a3a3ec8e0af2e028d5', + 'data': '0x000000000000000000000000000000000000000000000000000000000000279c', + 'topics': [ + '0x7220970e1f1f12864ecccd8942690a837c7a8dd45d158cb891eb45a8a69134aa' + ] + }::variant + assertions: + - > + result = [ + { + 'address': '0x55032650b14df07b85bF18A3a3eC8E0Af2e028d5', + 'data': [ + { + 'decoded': true, + 'name': 'nonce', + 'type': 'uint256', + 'value': 10140 + } + ], + 'decoded': true, + 'name': 'NonceChanged' + } + ] \ No newline at end of file diff --git a/models/live__allow_list.sql b/models/live__allow_list.sql new file mode 100644 index 0000000..611fda3 --- /dev/null +++ b/models/live__allow_list.sql @@ -0,0 +1,11 @@ +-- depends_on: {{ ref('live') }} +{{ config( + materialized = 'view', + grants = {'+select': fromyaml(var('ROLES'))} +) }} +SELECT '*.' || t.value AS allowed_domains +FROM table(flatten(input => {{ this.database }}.live.udf_allow_list())) AS t +ORDER BY + split_part(allowed_domains, '.', -1), + split_part(allowed_domains, '.', -2), + split_part(allowed_domains, '.', -3) diff --git a/packages.yml b/packages.yml index 172201a..6b7cfda 100644 --- a/packages.yml +++ b/packages.yml @@ -3,5 +3,4 @@ packages: version: [">=0.8.0", "<0.9.0"] - package: dbt-labs/dbt_utils version: [">=1.0.0", "<1.1.0"] - - git: https://github.com/FlipsideCrypto/livequery-base.git - revision: v1.0.1 \ No newline at end of file + \ No newline at end of file diff --git a/tests/generic/test_udf.sql b/tests/generic/test_udf.sql index 407abca..2754fb5 100644 --- a/tests/generic/test_udf.sql +++ b/tests/generic/test_udf.sql @@ -1,12 +1,39 @@ {% test test_udf(model, column_name, args, assertions) %} {# This is a generic test for UDFs. - The udfs are deployed using ephemeral models, so we need to - use the ephemeral model name to get the udf name. + The udfs are deployed using ephemeral models, as of dbt-core > 1.8 + we need to use `this.identifier` to extract the schema from for base_test_udf(). #} - {%- set schema = model | replace("__dbt__cte__", "") -%} - {%- set schema = schema.split("__") | first -%} - {%- set udf = schema ~ "." ~ column_name -%} + + {% set schema = none %} + + {% if execute %} + {# Extract schema based on standard pattern `test___ #} + {% set test_identifier = this.identifier %} + + {% if test_identifier.startswith('test_') %} + {% set test_identifier = test_identifier[5:] %} + {% endif %} + + {# Handle schemas with underscore prefix #} + {% if test_identifier.startswith('_') %} + {# For identifiers like _utils_ #} + {% set parts = test_identifier.split('_') %} + {% if parts | length > 2 %} + {% set schema = '_' ~ parts[1] %} + {% else %} + {% set schema = parts[0] %} + {% endif %} + {% else %} + {# For identifiers without underscore prefix #} + {% set parts = test_identifier.split('_') %} + {% if parts | length > 0 %} + {% set schema = parts[0] %} + {% endif %} + {% endif %} + {% endif %} + + {% set udf = schema ~ "." ~ column_name %} {{ base_test_udf(model, udf, args, assertions) }} {% endtest %} \ No newline at end of file