lq & udf_api overload deploy (#186)

* lq overload deploy success

* fixed make directive line continuation

---------

Co-authored-by: shah <info@shahnewazkhan.ca>

Makefile

@@ -86,4 +86,12 @@ tx_results_history:
 	-m 1+models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet22.sql \
 	--profile flow \
 	--target $(DBT_TARGET) \
-	--profiles-dir ~/.dbt
+	--profiles-dir ~/.dbt
+
+lq_overloads:
+	dbt run \
+	-s models/deploy/core/ \
+	--profile flow \
+	--target $(DBT_TARGET) \
+	--profiles-dir ~/.dbt \
+	--vars '{"UPDATE_LQ_UDFS":True}'

dbt_project.yml

@@ -39,6 +39,9 @@ on-run-end:
 # as tables. These settings can be overridden in the individual model files
 # using the `{{ config(...) }}` macro.
 models:
+  flow_models:
+    deploy:
+      +materialized: ephemeral
   +copy_grants: true
   +persist_docs:
     relation: true
@@ -60,6 +63,10 @@ vars:
   STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
   STREAMLINE_INVOKE_STREAMS: False
   STREAMLINE_RUN_HISTORY: False
+  API_INTEGRATION: AWS_FLOW_API{{ '_DEV_2' if target.name != 'prod' else '_PROD'  }}
+  DROP_UDFS_AND_SPS: False
+  REST_API_PREFIX_PROD: quxfxtl934.execute-api.us-east-1.amazonaws.com/prod/
+  REST_API_PREFIX_DEV: ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/
 
 dispatch:
   - macro_namespace: dbt

macros/create_udfs.sql

@@ -11,7 +11,6 @@
             ) }}
             {{ create_udf_get_chainhead() }}
             {{ create_udf_bulk_grpc() }}
-            {{ create_udf_api() }}
             
             {{ run_create_udf_array_disjunctive_union() }}
             

macros/streamline/_live.yaml.sql

@@ -0,0 +1,17 @@
+{% macro config_core__live(schema="_live") %}
+- name: {{ schema }}.udf_api
+  signature:
+    - [method, STRING]
+    - [url, STRING]
+    - [headers, OBJECT]
+    - [DATA, OBJECT]
+    - [user_id, STRING]
+    - [SECRET, STRING]
+  return_type: VARIANT
+  func_type: EXTERNAL
+  api_integration: '{{ var("API_INTEGRATION") }}'
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+  sql: udf_api
+{% endmacro %}

macros/streamline/_utils.yaml.sql

@@ -0,0 +1,29 @@
+{% macro config_core__utils(schema="_utils") %}
+
+- name: {{ schema }}.udf_register_secret
+  signature:
+    - [request_id, STRING]
+    - [user_id, STRING]
+    - [key, STRING]
+  return_type: TEXT
+  func_type: SECURE EXTERNAL
+  api_integration: '{{ var("API_INTEGRATION") }}'
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+  sql: secret/register
+
+- name: {{ schema }}.udf_whoami
+  signature: []
+  func_type: SECURE
+  return_type: TEXT
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    IMMUTABLE
+    MEMOIZABLE
+  sql: |
+    SELECT
+      COALESCE(PARSE_JSON(GETVARIABLE('LIVEQUERY_CONTEXT')):userId::STRING, CURRENT_USER())
+
+{% endmacro %}

macros/streamline/live.yaml.sql

@@ -0,0 +1,104 @@
+{% macro config_core_live(schema="live") %}
+
+- name: {{ schema }}.udf_api
+  signature:
+    - [method, STRING]
+    - [url, STRING]
+    - [headers, OBJECT]
+    - [data, OBJECT]
+    - [secret_name, STRING]
+  return_type: VARIANT
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    VOLATILE
+  sql: |
+    SELECT
+      _live.UDF_API(
+          method,
+          url,
+          headers,
+          data,
+          _utils.UDF_WHOAMI(),
+          secret_name
+      )
+- name: {{ schema }}.udf_api
+  signature:
+    - [method, STRING]
+    - [url, STRING]
+    - [headers, OBJECT]
+    - [data, OBJECT]
+  return_type: VARIANT
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    VOLATILE
+  sql: |
+    SELECT
+      _live.UDF_API(
+          method,
+          url,
+          headers,
+          data,
+          _utils.UDF_WHOAMI(),
+          ''
+      )
+- name: {{ schema }}.udf_api
+  signature:
+    - [url, STRING]
+    - [data, OBJECT]
+  return_type: VARIANT
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    VOLATILE
+  sql: |
+    SELECT
+      _live.UDF_API(
+          'POST',
+          url,
+          {'Content-Type': 'application/json'},
+          data,
+          _utils.UDF_WHOAMI(),
+          ''
+      )
+- name: {{ schema }}.udf_api
+  signature:
+    - [url, STRING]
+    - [data, OBJECT]
+    - [secret_name, STRING]
+  return_type: VARIANT
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    VOLATILE
+  sql: |
+    SELECT
+      _live.UDF_API(
+          'POST',
+          url,
+          {'Content-Type': 'application/json'},
+          data,
+          _utils.UDF_WHOAMI(),
+          secret_name
+      )
+- name: {{ schema }}.udf_api
+  signature:
+    - [url, STRING]
+  return_type: VARIANT
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    VOLATILE
+  sql: |
+    SELECT
+      _live.UDF_API(
+          'GET',
+          url,
+          {},
+          {},
+          _utils.UDF_WHOAMI(),
+          ''
+      )
+      
+{% endmacro %}

macros/streamline/manage_udfs.sql

@@ -0,0 +1,151 @@
+{% macro drop_function(
+        func_name,
+        signature
+    ) %}
+    DROP FUNCTION IF EXISTS {{ func_name }}({{ compile_signature(signature, drop_ = True) }});
+{% endmacro %}
+
+{%- macro construct_api_route(route) -%}
+    'https://{{ var("REST_API_PREFIX_PROD") | lower if target.name == "prod" else var("REST_API_PREFIX_DEV") | lower }}{{ route }}'
+{%- endmacro -%}
+
+{%- macro compile_signature(
+        params,
+        drop_ = False
+    ) -%}
+    {% for p in params -%}
+        {%- set name = p.0 -%}
+        {%- set data_type = p.1 -%}
+        {% if drop_ %}
+            {{ data_type -}}
+        {% else %}
+            {{ name ~ " " ~ data_type -}}
+        {%- endif -%}
+        {%-if not loop.last -%},
+        {%- endif -%}
+    {% endfor -%}
+{%- endmacro -%}
+
+{% macro create_sql_function(
+        name_,
+        signature,
+        return_type,
+        sql_,
+        api_integration = none,
+        options = none,
+        func_type = none
+    ) %}
+    CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}(
+            {{- compile_signature(signature) }}
+    )
+    COPY GRANTS
+    RETURNS {{ return_type }}
+    {% if options -%}
+        {{ options }}
+    {% endif %}
+    {%- if api_integration -%}
+    api_integration = {{ api_integration }}
+    AS {{ construct_api_route(sql_) ~ ";" }}
+    {% else -%}
+    AS
+    $$
+    {{ sql_ }}
+    $$;
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro create_or_drop_function_from_config(
+        config,
+        drop_ = False
+    ) -%}
+    {% set name_ = config ["name"] %}
+    {% set signature = config ["signature"] %}
+    {% set return_type = config ["return_type"] if config ["return_type"] is string else config ["return_type"][0] %}
+    {% set sql_ = config ["sql"] %}
+    {% set options = config ["options"] %}
+    {% set api_integration = config ["api_integration"] %}
+    {% set func_type = config ["func_type"] %}
+
+    {% if not drop_ -%}
+        {{ create_sql_function(
+            name_ = name_,
+            signature = signature,
+            return_type = return_type,
+            sql_ = sql_,
+            options = options,
+            api_integration = api_integration,
+            func_type = func_type
+        ) }}
+    {%- else -%}
+        {{ drop_function(
+            name_,
+            signature = signature,
+        ) }}
+    {%- endif %}
+{% endmacro %}
+
+{% macro crud_udfs(config_func, schema, drop_) %}
+{#
+    Generate create or drop statements for a list of udf configs for a given schema
+
+    config_func: function that returns a list of udf configs
+    drop_: whether to drop or create the udfs
+ #}
+    {% set udfs = fromyaml(config_func())%}
+    {%- for udf in udfs -%}
+        {% if udf["name"].split(".") | first == schema %}
+            CREATE SCHEMA IF NOT EXISTS {{ schema }};
+            {{- create_or_drop_function_from_config(udf, drop_=drop_) -}}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+
+{% macro ephemeral_deploy_core(config) %}
+{#
+    This macro is used to deploy functions using ephemeral models.
+    It should only be used within an ephemeral model.
+ #}
+    
+    {% if execute and (var("UPDATE_LQ_UDFS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
+        {% set sql %}
+            {{- crud_udfs(config, this.schema, var("DROP_UDFS_AND_SPS")) -}}
+        {%- endset -%}
+        {%- if var("DROP_UDFS_AND_SPS") -%}
+            {%- do log("Drop core udfs: " ~ this.database ~ "." ~ this.schema, true) -%}
+        {%- else -%}
+            {%- do log("Deploy core udfs: " ~ this.database ~ "." ~ this.schema, true) -%}
+        {%- endif -%}
+        {%- do run_query(sql ~ apply_grants_by_schema(this.schema)) -%}
+    {% else -%}
+        SELECT '{{ model.schema }}' as schema_
+    {%- endif -%}
+{%- endmacro -%}
+
+{% macro apply_grants_by_schema(schema) %}
+{#
+    Generates SQL to grant permissions to roles for a given schema.
+    This gets run automatically when a deployment is made to prod.
+
+    This can be manually run to grant permissions to a new schema:
+    `dbt run-operation apply_grants_by_schema --args '{"schema": "my_schema"}'`
+ #}
+    {% if target.name == "prod" %}
+        {%- set outer = namespace(sql="") -%}
+        {% for role in [ "INTERNAL_DEV", "BI_ANALYTICS_READER"] %}
+                {% set sql -%}
+                    {% if schema.startswith("_") %}
+                        REVOKE USAGE ON SCHEMA {{ target.database }}.{{ schema }} FROM {{ role }};
+                        REVOKE USAGE ON ALL FUNCTIONS IN SCHEMA {{ target.database }}.{{ schema }} FROM {{ role }};
+                    {%- else -%}
+                        GRANT USAGE ON SCHEMA {{ target.database }}.{{ schema }} TO {{ role }};
+                        GRANT USAGE ON ALL FUNCTIONS IN SCHEMA {{ target.database }}.{{ schema }} TO {{ role }};
+
+                        GRANT SELECT ON ALL TABLES IN SCHEMA {{ target.database }}.{{ schema }} TO {{ role }};
+                        GRANT SELECT ON ALL VIEWS IN SCHEMA {{ target.database }}.{{ schema }} TO {{ role }};
+                    {%- endif -%}
+                {%- endset -%}
+                {%- set outer.sql = outer.sql ~ sql -%}
+        {%- endfor -%}
+        {{ outer.sql }}
+    {%- endif -%}
+{%- endmacro -%}

macros/streamline/utils.sql

@@ -26,4 +26,24 @@
             '"}}'
         )
     )
+{% endmacro %}
+
+
+{% macro config_core_utils(schema="utils") %}
+
+
+- name: {{ schema }}.udf_register_secret
+  signature:
+    - [request_id, STRING]
+    - [key, STRING]
+  func_type: SECURE
+  return_type: TEXT
+  options: |
+    NOT NULL
+    RETURNS NULL ON NULL INPUT
+    IMMUTABLE
+  sql: |
+    SELECT
+      _utils.UDF_REGISTER_SECRET(REQUEST_ID, _utils.UDF_WHOAMI(), KEY)
+
 {% endmacro %}

models/deploy/core/_live.sql

@@ -0,0 +1,2 @@
+{% set config = config_core__live %}
+{{ ephemeral_deploy_core(config) }}

models/deploy/core/_utils.sql

@@ -0,0 +1,2 @@
+{% set config = config_core__utils %}
+{{ ephemeral_deploy_core(config) }}

models/deploy/core/live.sql

@@ -0,0 +1,5 @@
+-- depends_on: {{ ref('_utils') }}
+-- depends_on: {{ ref('utils') }}
+-- depends_on: {{ ref('_live') }}
+{% set config = config_core_live %}
+{{ ephemeral_deploy_core(config) }}

models/deploy/core/utils.sql

@@ -0,0 +1,3 @@
+ -- depends_on: {{ ref('_utils') }}
+{% set config = config_core_utils %}
+{{ ephemeral_deploy_core(config) }}