From d786d4b85bae7880c060e302f1c2643e161576fa Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 28 Aug 2024 15:53:18 -0600 Subject: [PATCH 1/2] evm udf, decoded_traces and wait function --- macros/create_streamline_udfs.sql | 1 + macros/streamline/udfs.sql | 24 +++++++++++++++++ macros/streamline/utils.sql | 43 +++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/macros/create_streamline_udfs.sql b/macros/create_streamline_udfs.sql index a9bc226..9b2bad9 100644 --- a/macros/create_streamline_udfs.sql +++ b/macros/create_streamline_udfs.sql @@ -8,5 +8,6 @@ {% if var("UPDATE_UDFS_AND_SPS") %} {{ create_udf_bulk_rest_api_v2() }} {{ create_udf_bulk_decode_logs() }} + {{ create_udf_bulk_decode_traces() }} {% endif %} {% endmacro %} diff --git a/macros/streamline/udfs.sql b/macros/streamline/udfs.sql index f25e9eb..c21da4b 100644 --- a/macros/streamline/udfs.sql +++ b/macros/streamline/udfs.sql @@ -46,6 +46,30 @@ {% do adapter.execute(sql) %} {% endmacro %} +{% macro create_udf_bulk_decode_traces() %} + {{ log("Creating udf udf_bulk_decode_traces_v2 for target:" ~ target.name ~ ", schema: " ~ target.schema ~ ", DB: " ~ target.database, info=True) }} + {{ log("role:" ~ target.role ~ ", user:" ~ target.user, info=True) }} + + {% set sql %} + CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_decode_traces_v2(json object) returns array api_integration = + {% if target.name == "prod" %} + {{ log("Creating prod udf_bulk_decode_traces_v2", info=True) }} + {{ var("API_INTEGRATION") }} AS 'https://{{ var("EXTERNAL_FUNCTION_URI") | lower }}bulk_decode_traces' + {% elif target.name == "dev" %} + {{ log("Creating dev udf_bulk_decode_traces_v2", info=True) }} + {{ var("API_INTEGRATION") }} AS 'https://{{ var("EXTERNAL_FUNCTION_URI") | lower }}bulk_decode_traces' + {% elif target.name == "sbx" %} + {{ log("Creating stg udf_bulk_decode_traces_v2", info=True) }} + {{ var("API_INTEGRATION") }} AS 'https://{{ var("EXTERNAL_FUNCTION_URI") | lower }}bulk_decode_traces' + {% else %} + {{ log("Creating default (dev) udf_bulk_decode_traces_v2", info=True) }} + {{ var("config")["dev"]["API_INTEGRATION"] }} AS 'https://{{ var("config")["dev"]["EXTERNAL_FUNCTION_URI"] | lower }}bulk_decode_traces' + {% endif %}; + {% endset %} + {{ log(sql, info=True) }} + {% do adapter.execute(sql) %} +{% endmacro %} + {% macro create_aws_api_integrations() %} {{ log("Creating api integration for target:" ~ target.name ~ ", schema: " ~ target.schema ~ ", DB: " ~ target.database, info=True) }} {{ log("role:" ~ target.role ~ ", user:" ~ target.user, info=True) }} diff --git a/macros/streamline/utils.sql b/macros/streamline/utils.sql index b9411b8..ed27f2b 100644 --- a/macros/streamline/utils.sql +++ b/macros/streamline/utils.sql @@ -118,4 +118,47 @@ SELECT NULL {% endif %} +{% endmacro %} + +{% macro if_data_call_wait() %} + {% if var( + "STREAMLINE_INVOKE_STREAMS" + ) %} + {% set query %} + SELECT + 1 + WHERE + EXISTS( + SELECT + 1 + FROM + {{ model.schema ~ "." ~ model.alias }} + LIMIT + 1 + ) {% endset %} + {% if execute %} + {% set results = run_query( + query + ) %} + {% if results %} + {{ log( + "Waiting...", + info = True + ) }} + + {% set wait_query %} + SELECT + system$wait( + {{ var( + "WAIT", + 400 + ) }} + ) {% endset %} + {% do run_query(wait_query) %} + {% else %} + SELECT + NULL; + {% endif %} + {% endif %} + {% endif %} {% endmacro %} \ No newline at end of file From 8c99db499671ff6f514bd0695f7b1f20bce8d80d Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 3 Sep 2024 13:53:59 -0600 Subject: [PATCH 2/2] remove arn and api_integration macro, docs --- README.md | 8 ++------ macros/streamline/udfs.sql | 13 ------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index fe057aa..04966a6 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,6 @@ The `Streamline V 2.0` functions are a set of macros and UDFs that are designed # Setup variables in dbt_project.yml API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}' EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}' - API_AWS_ROLE_ARN: '{{ var("config")[target.name]["API_AWS_ROLE_ARN"] if var("config")[target.name] else var("config")["dev"]["API_AWS_ROLE_ARN"] }}' ROLES: '{{ var("config")[target.name]["ROLES"] }}' config: @@ -220,7 +219,6 @@ The `Streamline V 2.0` functions are a set of macros and UDFs that are designed dev: API_INTEGRATION: AWS_CROSSCHAIN_API_STG EXTERNAL_FUNCTION_URI: q0bnjqvs9a.execute-api.us-east-1.amazonaws.com/stg/ - API_AWS_ROLE_ARN: arn:aws:iam::704693948482:role/crosschain-api-stg-rolesnowflakeudfsAF733095-CCDPFHsmlGmu ROLES: - AWS_LAMBDA_CROSSCHAIN_API - INTERNAL_DEV @@ -228,7 +226,6 @@ The `Streamline V 2.0` functions are a set of macros and UDFs that are designed prod: API_INTEGRATION: AWS_CROSSCHAIN_API_PROD EXTERNAL_FUNCTION_URI: 35hm1qhag9.execute-api.us-east-1.amazonaws.com/prod/ - API_AWS_ROLE_ARN: arn:aws:iam::924682671219:role/crosschain-api-prod-rolesnowflakeudfsAF733095-jBvAIUHiR70D ROLES: - AWS_LAMBDA_CROSSCHAIN_API - INTERNAL_DEV @@ -243,10 +240,9 @@ The `Streamline V 2.0` functions are a set of macros and UDFs that are designed - [create_udf_bulk_decode_logs](/macros/streamline/udfs.sql#L25): This macro is used to create a `udf` name `udf_bulk_decode_logs_v2 ` in the `streamline` schema of the databae this is invoked in. This function returns a `variant` type and uses an API integration. The API integration and the external function URI are determined based on the target environment (`prod`, `dev`, or `sbx`). The [macro interpolates](/macros/streamline/udfs.sql#L32) the `API_INTEGRATION` and `EXTERNAL_FUNCTION_URI` vars from the `dbt_project.yml` file. -- [create_streamline_udfs](macros/create_streamline_udfs.sql#L1). This macro runs [create_aws_api_integrations](/macros/streamline/udfs.sql#L49) and [create_udf_bulk_rest_api_v2](/macros/streamline/udfs.sql#L1) when ran with `--vars '{UPDATE_UDFS_AND_SPS: true}'`. +- [create_streamline_udfs](macros/create_streamline_udfs.sql#L1). This macro runs [create_udf_bulk_rest_api_v2](/macros/streamline/udfs.sql#L1) when ran with `--vars '{UPDATE_UDFS_AND_SPS: true}'`. -- [create_evm_streamline_udfs](macros/create_streamline_udfs.sql#L8). This macro runs [create_aws_api_integrations](/macros/streamline/udfs.sql#L49), [create_udf_bulk_rest_api_v2](/macros/streamline/udfs.sql#L1), and [create_udf_bulk_decode_logs](/macros/streamline/udfs.sql#L25) when ran with `--vars '{UPDATE_UDFS_AND_SPS: true}'`. This is designed to be used on the EVM chains due to the inclusion of `create_udf_bulk_decode_logs`. - +- [create_evm_streamline_udfs](macros/create_streamline_udfs.sql#L8). This macro runs [create_udf_bulk_rest_api_v2](/macros/streamline/udfs.sql#L1), [create_udf_bulk_decode_logs](/macros/streamline/udfs.sql#L25), and [create_udf_bulk_decode_traces](/macros/streamline/udfs.sql#L49) when ran with `--vars '{UPDATE_UDFS_AND_SPS: true}'`. This is designed to be used on the EVM chains due to the inclusion of `create_udf_bulk_decode_logs` and `create_udf_bulk_decode_traces`. ## **LiveQuery Functions** diff --git a/macros/streamline/udfs.sql b/macros/streamline/udfs.sql index c21da4b..f459c25 100644 --- a/macros/streamline/udfs.sql +++ b/macros/streamline/udfs.sql @@ -69,16 +69,3 @@ {{ log(sql, info=True) }} {% do adapter.execute(sql) %} {% endmacro %} - -{% macro create_aws_api_integrations() %} - {{ log("Creating api integration for target:" ~ target.name ~ ", schema: " ~ target.schema ~ ", DB: " ~ target.database, info=True) }} - {{ log("role:" ~ target.role ~ ", user:" ~ target.user, info=True) }} - - {% set sql %} - CREATE OR REPLACE api integration {{ var("API_INTEGRATION") }} api_provider = aws_api_gateway api_aws_role_arn = '{{ var("API_AWS_ROLE_ARN") }}' api_allowed_prefixes = ( - 'https://{{ var("EXTERNAL_FUNCTION_URI") | lower }}' - ) enabled = TRUE; - {% endset %} - {{ log(sql, info=True) }} - {% do adapter.execute(sql) %} -{% endmacro %} \ No newline at end of file