flip/fsc-utils
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/fsc-utils.git synced 2026-02-06 10:56:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

removed unnecessary files and renamed

Browse Source
This commit is contained in:
drethereum 2023-05-22 18:05:09 -06:00
parent adda21a75f
commit 0ffa7c5355
13 changed files with 17 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
.github/workflows/dbt_run_dev_refresh.yml vendored
View File

@ -1,44 +0,0 @@
name: dbt_run_dev_refresh
run-name: dbt_run_dev_refresh
on:
workflow_dispatch:
schedule:
# Runs "at 9:00 UTC" (see https://crontab.guru)
- cron: '0 9 * * *'
env:
DBT_PROFILES_DIR: ./
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v1
with:
python-version: "3.7.x"
- name: install dependencies
run: |
pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
dbt deps
- name: Run DBT Jobs
run: |
dbt run-operation run_sp_create_prod_clone

15
dbt_project.yml
View File

@ -1,7 +1,7 @@
# Name your project! Project names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
name: "reference_models"
name: "fsc_utils"
version: "1.0.0"
config-version: 2
@ -27,13 +27,6 @@ clean-targets: # directories to be removed by `dbt clean`
tests:
+store_failures: true # all tests
on-run-start:
- "{{ create_sps() }}"
- "{{ create_udfs() }}"
on-run-end:
- "{{ apply_meta_as_tags(results) }}"
# Configuring models
# Full documentation: https://docs.getdbt.com/docs/configuring-models
@ -44,10 +37,4 @@ on-run-end:
vars:
"dbt_date:time_zone": GMT
UPDATE_UDFS_AND_SPS: false
UPDATE_SNOWFLAKE_TAGS: true
STREAMLINE_INVOKE_STREAMS: False
STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
REST_API_ID_PROD: hn8uqhku77
REST_API_ID_DEV: hn8uqhku77
API_INTEGRATION: AWS_LIVE_QUERY{{ '_DEV' if target.name != 'prod' else '' }}
AWS_REGION: us-east-1

7
macros/create_sps.sql
View File

@ -1,8 +1,5 @@
{% macro create_sps() %}
{# {% macro create_sps() %}
{% if var("UPDATE_UDFS_AND_SPS") %}
{% if target.database == 'REFERENCE' %}
CREATE schema IF NOT EXISTS _internal;
{{ sp_create_prod_clone('_internal') }};
{% endif %}
{% endif %}
{% endmacro %}
{% endmacro %} #}

4
macros/create_udfs.sql
View File

@ -1,13 +1,13 @@
{% macro create_udfs(drop_=False,schema="utils") %}
{% if var("UPDATE_UDFS_AND_SPS") %}
{% set name %}
{{- reference_models.udf_configs(schema) -}}
{{- fsc_utils.udf_configs(schema) -}}
{% endset %}
{% set udfs = fromyaml(name) %}
{% set sql %}
CREATE schema if NOT EXISTS {{ schema }};
{%- for udf in udfs -%}
{{- reference_models.create_or_drop_function_from_config(udf, drop_=drop_) -}}
{{- fsc_utils.create_or_drop_function_from_config(udf, drop_=drop_) -}}
{% endfor %}
{% endset %}
{% do run_query(sql) %}

5
macros/hex_to_str_udf.sql
View File

@ -1,5 +0,0 @@
{% macro udf_hex_to_string(hex_string) %}
LTRIM(regexp_replace(
try_hex_decode_string({{ hex_string }}),
'[\x00-\x1F\x7F-\x9F\xAD]', '', 1))
{% endmacro %}

10
macros/run_sp_create_prod_clone.sql
View File

@ -1,10 +0,0 @@
{% macro run_sp_create_prod_clone() %}
{% set clone_query %}
call reference._internal.create_prod_clone(
'reference',
'reference_dev',
'reference_dev_owner'
);
{% endset %}
{% do run_query(clone_query) %}
{% endmacro %}

44
macros/sp_create_prod_clone.sql
View File

@ -1,44 +0,0 @@
{% macro sp_create_prod_clone(target_schema) -%}
create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string)
returns boolean
language javascript
execute as caller
as
$$
snowflake.execute({sqlText: `BEGIN TRANSACTION;`});
try {
snowflake.execute({sqlText: `CREATE OR REPLACE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`});
snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`})
var existing_tags = snowflake.execute({sqlText: `SHOW TAGS IN DATABASE ${DESTINATION_DB_NAME};`});
while (existing_tags.next()) {
var schema = existing_tags.getColumnValue(4);
var tag_name = existing_tags.getColumnValue(2)
snowflake.execute({sqlText: `GRANT OWNERSHIP ON TAG ${DESTINATION_DB_NAME}.${schema}.${tag_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
}
snowflake.execute({sqlText: `COMMIT;`});
} catch (err) {
snowflake.execute({sqlText: `ROLLBACK;`});
throw(err);
}
return true
$$
{%- endmacro %}

4
macros/streamline/configs.yaml.sql
View File

@ -15,7 +15,7 @@
RUNTIME_VERSION = '3.8'
HANDLER = 'hex_to_int'
sql: |
{{ reference_models.python_hex_to_int() | indent(4) }}
{{ fsc_utils.python_hex_to_int() | indent(4) }}
- name: {{ schema }}.udf_hex_to_int
signature:
- [encoding, STRING]
@ -28,7 +28,7 @@
RUNTIME_VERSION = '3.8'
HANDLER = 'hex_to_int'
sql: |
{{ reference_models.python_udf_hex_to_int_with_encoding() | indent(4) }}
{{ fsc_utils.python_udf_hex_to_int_with_encoding() | indent(4) }}
- name: {{ schema }}.udf_hex_to_string
signature:

10
macros/streamline/utils.sql
View File

@ -2,7 +2,7 @@
func_name,
signature
) %}
DROP FUNCTION IF EXISTS {{ func_name }}({{ reference_models.compile_signature(signature, drop_ = True) }});
DROP FUNCTION IF EXISTS {{ func_name }}({{ fsc_utils.compile_signature(signature, drop_ = True) }});
{% endmacro %}
{%- macro construct_api_route(route) -%}
@ -35,7 +35,7 @@
func_type = none
) %}
CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}(
{{- reference_models.compile_signature(signature) }}
{{- fsc_utils.compile_signature(signature) }}
)
COPY GRANTS
RETURNS {{ return_type }}
@ -44,7 +44,7 @@
{% endif %}
{%- if api_integration -%}
api_integration = {{ api_integration }}
AS {{ reference_models.construct_api_route(sql_) ~ ";" }}
AS {{ fsc_utils.construct_api_route(sql_) ~ ";" }}
{% else -%}
AS
$$
@ -66,7 +66,7 @@
{% set func_type = config ["func_type"] %}
{% if not drop_ -%}
{{ reference_models.create_sql_function(
{{ fsc_utils.create_sql_function(
name_ = name_,
signature = signature,
return_type = return_type,
@ -76,7 +76,7 @@
func_type = func_type
) }}
{%- else -%}
{{ reference_models.drop_function(
{{ fsc_utils.drop_function(
name_,
signature = signature,
) }}

3
macros/tags/add_database_or_schema_tags.sql
View File

@ -1,3 +0,0 @@
{% macro add_database_or_schema_tags() %}
{{ set_database_tag_value('BLOCKCHAIN_NAME','LIVEQUERY') }}
{% endmacro %}

127
macros/tags/snowflake_tagging.sql
View File

@ -1,127 +0,0 @@
{% macro apply_meta_as_tags(results) %}
{% if var("UPDATE_SNOWFLAKE_TAGS") %}
{{ log('apply_meta_as_tags', info=False) }}
{{ log(results, info=False) }}
{% if execute %}
{%- set tags_by_schema = {} -%}
{% for res in results -%}
{% if res.node.meta.database_tags %}
{%- set model_database = res.node.database -%}
{%- set model_schema = res.node.schema -%}
{%- set model_schema_full = model_database+'.'+model_schema -%}
{%- set model_alias = res.node.alias -%}
{% if model_schema_full not in tags_by_schema.keys() %}
{{ log('need to fetch tags for schema '+model_schema_full, info=False) }}
{%- call statement('main', fetch_result=True) -%}
show tags in {{model_database}}.{{model_schema}}
{%- endcall -%}
{%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%}
{{ log('Added tags to cache', info=False) }}
{% else %}
{{ log('already have tag info for schema', info=False) }}
{% endif %}
{%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%}
{{ log('current_tags_in_schema:', info=False) }}
{{ log(current_tags_in_schema, info=False) }}
{{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }}
{% set line -%}
node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }})
node full: {{ res.node}}
meta: {{ res.node.meta}}
materialized: {{ res.node.config.materialized }}
{%- endset %}
{{ log(line, info=False) }}
{%- call statement('main', fetch_result=True) -%}
select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table'))
{%- endcall -%}
{%- set existing_tags_for_table = load_result('main')['data'] -%}
{{ log('Existing tags for table:', info=False) }}
{{ log(existing_tags_for_table, info=False) }}
{{ log('--', info=False) }}
{% for table_tag in res.node.meta.database_tags.table %}
{{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }}
{% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %}
{{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}}
{% endfor %}
{{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }}
{% endif %}
{% endfor %}
{% endif %}
{% endif %}
{% endmacro %}
{% macro create_tag_if_missing(all_tag_names,table_tag) %}
{% if table_tag not in all_tag_names %}
{{ log('Creating missing tag '+table_tag, info=False) }}
{%- call statement('main', fetch_result=True) -%}
create tag if not exists silver.{{table_tag}}
{%- endcall -%}
{{ log(load_result('main').data, info=False) }}
{% else %}
{{ log('Tag already exists: '+table_tag, info=False) }}
{% endif %}
{% endmacro %}
{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %}
{{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }}
{%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%}
{{ log('Filtered tags for table:', info=False) }}
{{ log(existing_tag_for_table[0], info=False) }}
{% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %}
{{ log('Correct tag value already exists', info=False) }}
{% else %}
{{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }}
{%- call statement('main', fetch_result=True) -%}
alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}'
{%- endcall -%}
{{ log(load_result('main').data, info=False) }}
{% endif %}
{% endmacro %}
{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %}
{{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }}
{%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%}
{{ log('Filtered tags for column:', info=False) }}
{{ log(existing_tag_for_column[0], info=False) }}
{% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %}
{{ log('Correct tag value already exists', info=False) }}
{% else %}
{{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }}
{%- call statement('main', fetch_result=True) -%}
alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}'
{%- endcall -%}
{{ log(load_result('main').data, info=False) }}
{% endif %}
{% endmacro %}
{% macro set_database_tag_value(tag_name,tag_value) %}
{% set query %}
create tag if not exists silver.{{tag_name}}
{% endset %}
{% do run_query(query) %}
{% set query %}
alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}'
{% endset %}
{% do run_query(query) %}
{% endmacro %}
{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %}
{% set query %}
create tag if not exists silver.{{tag_name}}
{% endset %}
{% do run_query(query) %}
{% set query %}
alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}'
{% endset %}
{% do run_query(query) %}
{% endmacro %}

6
models/doc_descriptions/general/__overview__.md
View File

@ -1,9 +1,9 @@
{% docs __overview__ %}
# Welcome to the Flipside Crypto Reference Models Documentation!
# Welcome to the Flipside Crypto Utility Functions Documentation!
## **What does this documentation cover?**
The documentation included here details the design of the Reference functions available via [Flipside Crypto](https://flipsidecrypto.xyz/). For more information on how these functions are built, please see [the github repository.](https://github.com/FlipsideCrypto/reference-models)
The documentation included here details the design of the utility functions available via [Flipside Crypto](https://flipsidecrypto.xyz/). For more information on how these functions are built, please see [the github repository.](https://github.com/FlipsideCrypto/fsc-utils)
### **Overview of Available Functions**
@ -64,7 +64,7 @@ Note that you can also right-click on models to interactively filter and explore
### **More information**
- [Flipside](https://flipsidecrypto.xyz/)
- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials)
- [Github](https://github.com/FlipsideCrypto/reference-models)
- [Github](https://github.com/FlipsideCrypto/fsc-utils)
- [What is dbt?](https://docs.getdbt.com/docs/introduction)
{% enddocs %}

4
profiles.yml
View File

@ -13,7 +13,7 @@ reference:
schema: SILVER
threads: 4
client_session_keep_alive: False
query_tag: reference_curator
query_tag: fsc_utils_curator
prod:
type: snowflake
account: "{{ env_var('ACCOUNT') }}"
@ -26,4 +26,4 @@ reference:
schema: SILVER
threads: 4
client_session_keep_alive: False
query_tag: reference_curator
query_tag: fsc_utils_curator