diff --git a/Makefile b/Makefile index 1c03a70..3f1b688 100644 --- a/Makefile +++ b/Makefile @@ -86,4 +86,12 @@ tx_results_history: -m 1+models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet22.sql \ --profile flow \ --target $(DBT_TARGET) \ - --profiles-dir ~/.dbt \ No newline at end of file + --profiles-dir ~/.dbt + +lq_overloads: + dbt run \ + -s models/deploy/core/ \ + --profile flow \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt \ + --vars '{"UPDATE_LQ_UDFS":True}' \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 8873e04..8f0ee00 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -39,6 +39,9 @@ on-run-end: # as tables. These settings can be overridden in the individual model files # using the `{{ config(...) }}` macro. models: + flow_models: + deploy: + +materialized: ephemeral +copy_grants: true +persist_docs: relation: true @@ -60,6 +63,10 @@ vars: STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False STREAMLINE_INVOKE_STREAMS: False STREAMLINE_RUN_HISTORY: False + API_INTEGRATION: AWS_FLOW_API{{ '_DEV_2' if target.name != 'prod' else '_PROD' }} + DROP_UDFS_AND_SPS: False + REST_API_PREFIX_PROD: quxfxtl934.execute-api.us-east-1.amazonaws.com/prod/ + REST_API_PREFIX_DEV: ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/ dispatch: - macro_namespace: dbt diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 270becf..3f7959b 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -11,7 +11,6 @@ ) }} {{ create_udf_get_chainhead() }} {{ create_udf_bulk_grpc() }} - {{ create_udf_api() }} {{ run_create_udf_array_disjunctive_union() }} diff --git a/macros/streamline/_live.yaml.sql b/macros/streamline/_live.yaml.sql new file mode 100644 index 0000000..b94a92a --- /dev/null +++ b/macros/streamline/_live.yaml.sql @@ -0,0 +1,17 @@ +{% macro config_core__live(schema="_live") %} +- name: {{ schema }}.udf_api + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [DATA, OBJECT] + - [user_id, STRING] + - [SECRET, STRING] + return_type: VARIANT + func_type: EXTERNAL + api_integration: '{{ var("API_INTEGRATION") }}' + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + sql: udf_api +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/_utils.yaml.sql b/macros/streamline/_utils.yaml.sql new file mode 100644 index 0000000..3422965 --- /dev/null +++ b/macros/streamline/_utils.yaml.sql @@ -0,0 +1,29 @@ +{% macro config_core__utils(schema="_utils") %} + +- name: {{ schema }}.udf_register_secret + signature: + - [request_id, STRING] + - [user_id, STRING] + - [key, STRING] + return_type: TEXT + func_type: SECURE EXTERNAL + api_integration: '{{ var("API_INTEGRATION") }}' + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + sql: secret/register + +- name: {{ schema }}.udf_whoami + signature: [] + func_type: SECURE + return_type: TEXT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + MEMOIZABLE + sql: | + SELECT + COALESCE(PARSE_JSON(GETVARIABLE('LIVEQUERY_CONTEXT')):userId::STRING, CURRENT_USER()) + +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/live.yaml.sql b/macros/streamline/live.yaml.sql new file mode 100644 index 0000000..4582aac --- /dev/null +++ b/macros/streamline/live.yaml.sql @@ -0,0 +1,104 @@ +{% macro config_core_live(schema="live") %} + +- name: {{ schema }}.udf_api + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [data, OBJECT] + - [secret_name, STRING] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + method, + url, + headers, + data, + _utils.UDF_WHOAMI(), + secret_name + ) +- name: {{ schema }}.udf_api + signature: + - [method, STRING] + - [url, STRING] + - [headers, OBJECT] + - [data, OBJECT] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + method, + url, + headers, + data, + _utils.UDF_WHOAMI(), + '' + ) +- name: {{ schema }}.udf_api + signature: + - [url, STRING] + - [data, OBJECT] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + 'POST', + url, + {'Content-Type': 'application/json'}, + data, + _utils.UDF_WHOAMI(), + '' + ) +- name: {{ schema }}.udf_api + signature: + - [url, STRING] + - [data, OBJECT] + - [secret_name, STRING] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + 'POST', + url, + {'Content-Type': 'application/json'}, + data, + _utils.UDF_WHOAMI(), + secret_name + ) +- name: {{ schema }}.udf_api + signature: + - [url, STRING] + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + VOLATILE + sql: | + SELECT + _live.UDF_API( + 'GET', + url, + {}, + {}, + _utils.UDF_WHOAMI(), + '' + ) + +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/manage_udfs.sql b/macros/streamline/manage_udfs.sql new file mode 100644 index 0000000..66b3369 --- /dev/null +++ b/macros/streamline/manage_udfs.sql @@ -0,0 +1,151 @@ +{% macro drop_function( + func_name, + signature + ) %} + DROP FUNCTION IF EXISTS {{ func_name }}({{ compile_signature(signature, drop_ = True) }}); +{% endmacro %} + +{%- macro construct_api_route(route) -%} + 'https://{{ var("REST_API_PREFIX_PROD") | lower if target.name == "prod" else var("REST_API_PREFIX_DEV") | lower }}{{ route }}' +{%- endmacro -%} + +{%- macro compile_signature( + params, + drop_ = False + ) -%} + {% for p in params -%} + {%- set name = p.0 -%} + {%- set data_type = p.1 -%} + {% if drop_ %} + {{ data_type -}} + {% else %} + {{ name ~ " " ~ data_type -}} + {%- endif -%} + {%-if not loop.last -%}, + {%- endif -%} + {% endfor -%} +{%- endmacro -%} + +{% macro create_sql_function( + name_, + signature, + return_type, + sql_, + api_integration = none, + options = none, + func_type = none + ) %} + CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}( + {{- compile_signature(signature) }} + ) + COPY GRANTS + RETURNS {{ return_type }} + {% if options -%} + {{ options }} + {% endif %} + {%- if api_integration -%} + api_integration = {{ api_integration }} + AS {{ construct_api_route(sql_) ~ ";" }} + {% else -%} + AS + $$ + {{ sql_ }} + $$; + {%- endif -%} +{%- endmacro -%} + +{%- macro create_or_drop_function_from_config( + config, + drop_ = False + ) -%} + {% set name_ = config ["name"] %} + {% set signature = config ["signature"] %} + {% set return_type = config ["return_type"] if config ["return_type"] is string else config ["return_type"][0] %} + {% set sql_ = config ["sql"] %} + {% set options = config ["options"] %} + {% set api_integration = config ["api_integration"] %} + {% set func_type = config ["func_type"] %} + + {% if not drop_ -%} + {{ create_sql_function( + name_ = name_, + signature = signature, + return_type = return_type, + sql_ = sql_, + options = options, + api_integration = api_integration, + func_type = func_type + ) }} + {%- else -%} + {{ drop_function( + name_, + signature = signature, + ) }} + {%- endif %} +{% endmacro %} + +{% macro crud_udfs(config_func, schema, drop_) %} +{# + Generate create or drop statements for a list of udf configs for a given schema + + config_func: function that returns a list of udf configs + drop_: whether to drop or create the udfs + #} + {% set udfs = fromyaml(config_func())%} + {%- for udf in udfs -%} + {% if udf["name"].split(".") | first == schema %} + CREATE SCHEMA IF NOT EXISTS {{ schema }}; + {{- create_or_drop_function_from_config(udf, drop_=drop_) -}} + {%- endif -%} + {%- endfor -%} +{%- endmacro -%} + +{% macro ephemeral_deploy_core(config) %} +{# + This macro is used to deploy functions using ephemeral models. + It should only be used within an ephemeral model. + #} + + {% if execute and (var("UPDATE_LQ_UDFS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %} + {% set sql %} + {{- crud_udfs(config, this.schema, var("DROP_UDFS_AND_SPS")) -}} + {%- endset -%} + {%- if var("DROP_UDFS_AND_SPS") -%} + {%- do log("Drop core udfs: " ~ this.database ~ "." ~ this.schema, true) -%} + {%- else -%} + {%- do log("Deploy core udfs: " ~ this.database ~ "." ~ this.schema, true) -%} + {%- endif -%} + {%- do run_query(sql ~ apply_grants_by_schema(this.schema)) -%} + {% else -%} + SELECT '{{ model.schema }}' as schema_ + {%- endif -%} +{%- endmacro -%} + +{% macro apply_grants_by_schema(schema) %} +{# + Generates SQL to grant permissions to roles for a given schema. + This gets run automatically when a deployment is made to prod. + + This can be manually run to grant permissions to a new schema: + `dbt run-operation apply_grants_by_schema --args '{"schema": "my_schema"}'` + #} + {% if target.name == "prod" %} + {%- set outer = namespace(sql="") -%} + {% for role in [ "INTERNAL_DEV", "BI_ANALYTICS_READER"] %} + {% set sql -%} + {% if schema.startswith("_") %} + REVOKE USAGE ON SCHEMA {{ target.database }}.{{ schema }} FROM {{ role }}; + REVOKE USAGE ON ALL FUNCTIONS IN SCHEMA {{ target.database }}.{{ schema }} FROM {{ role }}; + {%- else -%} + GRANT USAGE ON SCHEMA {{ target.database }}.{{ schema }} TO {{ role }}; + GRANT USAGE ON ALL FUNCTIONS IN SCHEMA {{ target.database }}.{{ schema }} TO {{ role }}; + + GRANT SELECT ON ALL TABLES IN SCHEMA {{ target.database }}.{{ schema }} TO {{ role }}; + GRANT SELECT ON ALL VIEWS IN SCHEMA {{ target.database }}.{{ schema }} TO {{ role }}; + {%- endif -%} + {%- endset -%} + {%- set outer.sql = outer.sql ~ sql -%} + {%- endfor -%} + {{ outer.sql }} + {%- endif -%} +{%- endmacro -%} \ No newline at end of file diff --git a/macros/streamline/utils.sql b/macros/streamline/utils.sql index b66b028..ac23d7b 100644 --- a/macros/streamline/utils.sql +++ b/macros/streamline/utils.sql @@ -26,4 +26,24 @@ '"}}' ) ) +{% endmacro %} + + +{% macro config_core_utils(schema="utils") %} + + +- name: {{ schema }}.udf_register_secret + signature: + - [request_id, STRING] + - [key, STRING] + func_type: SECURE + return_type: TEXT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + SELECT + _utils.UDF_REGISTER_SECRET(REQUEST_ID, _utils.UDF_WHOAMI(), KEY) + {% endmacro %} \ No newline at end of file diff --git a/models/deploy/core/_live.sql b/models/deploy/core/_live.sql new file mode 100644 index 0000000..f37331e --- /dev/null +++ b/models/deploy/core/_live.sql @@ -0,0 +1,2 @@ +{% set config = config_core__live %} +{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/_utils.sql b/models/deploy/core/_utils.sql new file mode 100644 index 0000000..ac40ddf --- /dev/null +++ b/models/deploy/core/_utils.sql @@ -0,0 +1,2 @@ +{% set config = config_core__utils %} +{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/live.sql b/models/deploy/core/live.sql new file mode 100644 index 0000000..9e088f2 --- /dev/null +++ b/models/deploy/core/live.sql @@ -0,0 +1,5 @@ +-- depends_on: {{ ref('_utils') }} +-- depends_on: {{ ref('utils') }} +-- depends_on: {{ ref('_live') }} +{% set config = config_core_live %} +{{ ephemeral_deploy_core(config) }} diff --git a/models/deploy/core/utils.sql b/models/deploy/core/utils.sql new file mode 100644 index 0000000..fbabfe1 --- /dev/null +++ b/models/deploy/core/utils.sql @@ -0,0 +1,3 @@ + -- depends_on: {{ ref('_utils') }} +{% set config = config_core_utils %} +{{ ephemeral_deploy_core(config) }}