flip/fsc-utils
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/fsc-utils.git synced 2026-02-06 02:46:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

- create utility macros for creating udfs (#2)

Browse Source
This commit is contained in:
Julius Remigio 2023-02-20 13:05:59 -08:00 committed by GitHub
parent 67e1b91150
commit 8e55f678c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 290 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -14,4 +14,6 @@ logs/
.history/
**/.DS_Store
.vscode/
dbt-env/
dbt-env/
.env
.*

5
.sqlfluff Normal file
View File

@ -0,0 +1,5 @@
[sqlfluff]
templater = dbt
[sqlfluff:templater:jinja]
apply_dbt_builtins = True

6
.sqlfluffignore Normal file
View File

@ -0,0 +1,6 @@
target/
# dbt <1.0.0
dbt_modules/
# dbt >=1.0.0
dbt_packages/
macros/

27
README.md
View File

@ -1,6 +1,7 @@
## Profile Set Up
#### Use the following within profiles.yml
#### Use the following within profiles.yml
----
```yml
@ -45,14 +46,22 @@ When False, none of the on-run-start macros are executed on model run
Default values are False
* Usage:
dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ...
dbt run --var 'UPDATE_UDFS_AND_SPS": True' -m ...
Dropping and creating udfs can also be done without running a model:
```sh
dbt run-operation create_udfs --args 'drop_:false'
dbt run-operation create_udfs --args 'drop_:true'
```
### Resources:
- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
- Find [dbt events](https://events.getdbt.com) near you
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
* Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
* Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
* Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
* Find [dbt events](https://events.getdbt.com) near you
* Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices
## Applying Model Tags
@ -67,7 +76,7 @@ Database and schema tags are applied via the `add_database_or_schema_tags` macro
### Model tags
To add/update a model's snowflake tags, add/modify the `meta` model property under `config`. Only table level tags are supported at this time via DBT.
To add/update a model's snowflake tags, add/modify the `meta` model property under `config` . Only table level tags are supported at this time via DBT.
```
{{ config(
@ -94,4 +103,4 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/core/core__fact_blocks
```
select *
from table(livequery.information_schema.tag_references('livequery.core.fact_blocks', 'table'));
```
```

17
analysis/test_create_or_drop_function_from_config.sql Normal file
View File

@ -0,0 +1,17 @@
{% set name %}
{{- udf_configs() -}}
{% endset %}
{% set udfs = fromyaml(name) %}
{{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=True) -}}
{{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=True) -}}
{{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=True) -}}
{{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=True) -}}
{{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=True) -}}
{{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=True) -}}
{{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=False) -}}
{{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=False) -}}
{{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=False) -}}
{{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=False) -}}
{{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=False) -}}
{{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=False) -}}

7
cspell.yml Normal file
View File

@ -0,0 +1,7 @@
version: "0.2"
language: en
words:
- fromyaml
- GETVARIABLE
- livequery
- udfs

14
dbt_project.yml
View File

@ -29,10 +29,10 @@ tests:
on-run-start:
- "{{ create_sps() }}"
# - "{{ create_udfs() }}"
- "{{ create_udfs() }}"
on-run-end:
- '{{ apply_meta_as_tags(results) }}'
- "{{ apply_meta_as_tags(results) }}"
# Configuring models
# Full documentation: https://docs.getdbt.com/docs/configuring-models
@ -43,5 +43,11 @@ on-run-end:
vars:
"dbt_date:time_zone": GMT
UPDATE_UDFS_AND_SPS: False
UPDATE_SNOWFLAKE_TAGS: True
UPDATE_UDFS_AND_SPS: true
UPDATE_SNOWFLAKE_TAGS: True
STREAMLINE_INVOKE_STREAMS: False
STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
REST_API_ID_PROD: hn8uqhku77
REST_API_ID_DEV: hn8uqhku77
API_INTEGRATION: AWS_LIVE_QUERY_DEV
AWS_REGION: us-east-1

34
macros/create_udfs.sql
View File

@ -1,8 +1,26 @@
-- {% macro create_udfs() %}
-- {% if var("UPDATE_UDFS_AND_SPS") %}
-- {% set sql %}
-- CREATE schema if NOT EXISTS silver;
-- {% endset %}
-- {% do run_query(sql) %}
-- {% endif %}
-- {% endmacro %}
{% macro create_udfs(drop_=False) %}
{% if var("UPDATE_UDFS_AND_SPS") %}
{% set name %}
{{- udf_configs() -}}
{% endset %}
{% set udfs = fromyaml(name) %}
{% set sql %}
CREATE schema if NOT EXISTS silver;
CREATE schema if NOT EXISTS streamline;
CREATE schema if NOT EXISTS beta;
{{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=True) }}
{{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=True) }}
{{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=True) }}
{{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=True) }}
{{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=True) }}
{{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=True) }}
{{- create_or_drop_function_from_config(udfs["streamline.introspect"], drop_=False) }}
{{- create_or_drop_function_from_config(udfs["streamline.whoami"], drop_=False) }}
{{- create_or_drop_function_from_config(udfs["streamline.udf_register_secret"], drop_=False) }}
{{- create_or_drop_function_from_config(udfs["beta.udf_register_secret"], drop_=False) }}
{{- create_or_drop_function_from_config(udfs["streamline.udf_api"], drop_=False) }}
{{- create_or_drop_function_from_config(udfs["beta.udf_api"], drop_=False) }}
{% endset %}
{% do run_query(sql) %}
{% endif %}
{% endmacro %}

11
macros/streamline/api_integrations.sql Normal file
View File

@ -0,0 +1,11 @@
{% macro create_aws_ethereum_api() %}
{% if target.name == "prod" %}
{% set sql %}
CREATE api integration IF NOT EXISTS aws_ethereum_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::661245089684:role/snowflake-api-ethereum' api_allowed_prefixes = (
'https://e03pt6v501.execute-api.us-east-1.amazonaws.com/prod/',
'https://mryeusnrob.execute-api.us-east-1.amazonaws.com/dev/'
) enabled = TRUE;
{% endset %}
{% do run_query(sql) %}
{% endif %}
{% endmacro %}

81
macros/streamline/configs.yaml.sql Normal file
View File

@ -0,0 +1,81 @@
{% macro udf_configs() %}
streamline.introspect:
name: streamline.udf_introspect
signature:
- [echo, STRING]
func_type: SECURE EXTERNAL
return_type: TEXT
api_integration: AWS_LIVE_QUERY_DEV
sql: introspect
beta.udf_register_secret:
name: beta.udf_register_secret
signature:
- [request_id, string]
- [key, string]
func_type: SECURE
return_type: TEXT
options: NOT NULL STRICT IMMUTABLE
sql: |
SELECT
STREAMLINE.UDF_REGISTER_SECRET(REQUEST_ID, STREAMLINE.UDF_WHOAMI(), KEY)
beta.udf_api:
name: beta.udf_api
signature:
- [method, STRING]
- [url, STRING]
- [headers, OBJECT]
- [data, OBJECT]
- [secret_name, STRING]
return_type: VARIANT
func_type: SECURE
options: NOT NULL STRICT VOLATILE
sql: |
SELECT
STREAMLINE.UDF_API(
method,
url,
headers,
data,
STREAMLINE.UDF_WHOAMI(),
secret_name
)
streamline.udf_api:
name: streamline.udf_api
signature:
- [method, STRING]
- [url, STRING]
- [headers, OBJECT]
- [DATA, OBJECT]
- [user_id, STRING]
- [SECRET, STRING]
return_type: VARIANT
func_type: SECURE EXTERNAL
api_integration: AWS_LIVE_QUERY_DEV
options: NOT NULL STRICT
sql: udf_api
streamline.udf_register_secret:
name: streamline.udf_register_secret
signature:
- [request_id, string]
- [user_id, string]
- [key, string]
return_type: TEXT
func_type: SECURE EXTERNAL
api_integration: AWS_LIVE_QUERY_DEV
options: NOT NULL STRICT
sql: secret/register
streamline.whoami:
name: streamline.udf_whoami
signature: []
func_type: SECURE
return_type: TEXT
options: NOT NULL STRICT IMMUTABLE MEMOIZABLE
sql: |
SELECT
COALESCE(SPLIT_PART(GETVARIABLE('QUERY_TAG_SESSION'), ',',2), CURRENT_USER())
{% endmacro %}

22
macros/streamline/streamline_udfs.sql Normal file
View File

@ -0,0 +1,22 @@
{% macro create_udf_introspect(
drop_ = False
) %}
{% set name_ = 'silver.udf_introspect' %}
{% set signature = [('json', 'variant')] %}
{% set return_type = 'text' %}
{% set sql_ = construct_api_route("introspect") %}
{% if not drop_ %}
{{ create_sql_function(
name_ = name_,
signature = signature,
return_type = return_type,
sql_ = sql_,
api_integration = var("API_INTEGRATION")
) }}
{% else %}
{{ drop_function(
name_,
signature = signature,
) }}
{% endif %}
{% endmacro %}

83
macros/streamline/utils.sql Normal file
View File

@ -0,0 +1,83 @@
{% macro drop_function(
func_name,
signature
) %}
DROP FUNCTION IF EXISTS {{ func_name }}({{ compile_signature(signature, drop_ = True) }});
{% endmacro %}
{%- macro construct_api_route(route) -%}
'https://{{ var("REST_API_ID_PROD") if target.name == "prod" else var("REST_API_ID_DEV") }}.execute-api.{{ var( aws_region, "us-east-1" ) }}.amazonaws.com/{{ target.name }}/{{ route }}'
{%- endmacro -%}
{%- macro compile_signature(
params,
drop_ = False
) -%}
{% for name,
data_type in params -%}
{% if drop_ %}
{{ data_type -}}
{% else %}
{{ name ~ " " ~ data_type -}}
{% endif -%}
{%-if not loop.last -%},
{%- endif -%}
{% endfor -%}
{% endmacro %}
{% macro create_sql_function(
name_,
signature,
return_type,
sql_,
api_integration = none,
options = none,
func_type = none
) %}
CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}(
{{- compile_signature(signature) }}
)
RETURNS {{ return_type }}
{% if options -%}
{{ options }}
{% endif %}
{%- if api_integration -%}
api_integration = {{ api_integration }}
AS {{ construct_api_route(sql_) ~ ";" }}
{% else -%}
AS
$$
{{ sql_ }}
$$;
{%- endif -%}
{%- endmacro -%}
{%- macro create_or_drop_function_from_config(
config,
drop_ = False
) -%}
{% set name_ = config ["name"] %}
{% set signature = config ["signature"] %}
{% set return_type = config ["return_type"] %}
{% set sql_ = config ["sql"] %}
{% set options = config ["options"] %}
{% set api_integration = config ["api_integration"] %}
{% set func_type = config ["func_type"] %}
{% if not drop_ -%}
{{ create_sql_function(
name_ = name_,
signature = signature,
return_type = return_type,
sql_ = sql_,
options = options,
api_integration = api_integration,
func_type = func_type
) }}
{%- else -%}
{{ drop_function(
name_,
signature = signature,
) }}
{%- endif %}
{% endmacro %}

4
packages.yml
View File

@ -1,7 +1,5 @@
packages:
- package: calogica/dbt_expectations
version: [">=0.8.0", "<0.9.0"]
- package: dbt-labs/dbt_external_tables
version: [">=0.8.0", "<0.9.0"]
- package: dbt-labs/dbt_utils
version: [">=1.0.0", "<1.1.0"]
version: [">=1.0.0", "<1.1.0"]