diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml deleted file mode 100644 index ea29450..0000000 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_dev_refresh -run-name: dbt_run_dev_refresh - -on: - workflow_dispatch: - schedule: - # Runs "at 9:00 UTC" (see https://crontab.guru) - - cron: '0 9 * * *' - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run-operation run_sp_create_prod_clone \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index fb47932..4a7b7aa 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "reference_models" +name: "fsc_utils" version: "1.0.0" config-version: 2 @@ -27,13 +27,6 @@ clean-targets: # directories to be removed by `dbt clean` tests: +store_failures: true # all tests -on-run-start: - - "{{ create_sps() }}" - - "{{ create_udfs() }}" - -on-run-end: - - "{{ apply_meta_as_tags(results) }}" - # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models @@ -44,10 +37,4 @@ on-run-end: vars: "dbt_date:time_zone": GMT UPDATE_UDFS_AND_SPS: false - UPDATE_SNOWFLAKE_TAGS: true - STREAMLINE_INVOKE_STREAMS: False - STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False - REST_API_ID_PROD: hn8uqhku77 - REST_API_ID_DEV: hn8uqhku77 - API_INTEGRATION: AWS_LIVE_QUERY{{ '_DEV' if target.name != 'prod' else '' }} AWS_REGION: us-east-1 diff --git a/macros/create_sps.sql b/macros/create_sps.sql index ea91fdd..bbf917e 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,8 +1,5 @@ -{% macro create_sps() %} +{# {% macro create_sps() %} {% if var("UPDATE_UDFS_AND_SPS") %} - {% if target.database == 'REFERENCE' %} - CREATE schema IF NOT EXISTS _internal; - {{ sp_create_prod_clone('_internal') }}; {% endif %} {% endif %} -{% endmacro %} +{% endmacro %} #} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index b6af668..77c8adc 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,13 +1,13 @@ {% macro create_udfs(drop_=False,schema="utils") %} {% if var("UPDATE_UDFS_AND_SPS") %} {% set name %} - {{- reference_models.udf_configs(schema) -}} + {{- fsc_utils.udf_configs(schema) -}} {% endset %} {% set udfs = fromyaml(name) %} {% set sql %} CREATE schema if NOT EXISTS {{ schema }}; {%- for udf in udfs -%} - {{- reference_models.create_or_drop_function_from_config(udf, drop_=drop_) -}} + {{- fsc_utils.create_or_drop_function_from_config(udf, drop_=drop_) -}} {% endfor %} {% endset %} {% do run_query(sql) %} diff --git a/macros/hex_to_str_udf.sql b/macros/hex_to_str_udf.sql deleted file mode 100644 index 7153cc0..0000000 --- a/macros/hex_to_str_udf.sql +++ /dev/null @@ -1,5 +0,0 @@ -{% macro udf_hex_to_string(hex_string) %} - LTRIM(regexp_replace( - try_hex_decode_string({{ hex_string }}), - '[\x00-\x1F\x7F-\x9F\xAD]', '', 1)) -{% endmacro %} \ No newline at end of file diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql deleted file mode 100644 index 19d9e7e..0000000 --- a/macros/run_sp_create_prod_clone.sql +++ /dev/null @@ -1,10 +0,0 @@ -{% macro run_sp_create_prod_clone() %} - {% set clone_query %} - call reference._internal.create_prod_clone( - 'reference', - 'reference_dev', - 'reference_dev_owner' - ); -{% endset %} - {% do run_query(clone_query) %} -{% endmacro %} diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql deleted file mode 100644 index 20ee897..0000000 --- a/macros/sp_create_prod_clone.sql +++ /dev/null @@ -1,44 +0,0 @@ -{% macro sp_create_prod_clone(target_schema) -%} - -create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string) -returns boolean -language javascript -execute as caller -as -$$ - snowflake.execute({sqlText: `BEGIN TRANSACTION;`}); - try { - snowflake.execute({sqlText: `CREATE OR REPLACE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`}); - snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */ - - snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); - snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); - - snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}) - - var existing_tags = snowflake.execute({sqlText: `SHOW TAGS IN DATABASE ${DESTINATION_DB_NAME};`}); - while (existing_tags.next()) { - var schema = existing_tags.getColumnValue(4); - var tag_name = existing_tags.getColumnValue(2) - snowflake.execute({sqlText: `GRANT OWNERSHIP ON TAG ${DESTINATION_DB_NAME}.${schema}.${tag_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); - } - - snowflake.execute({sqlText: `COMMIT;`}); - } catch (err) { - snowflake.execute({sqlText: `ROLLBACK;`}); - throw(err); - } - - return true -$$ - -{%- endmacro %} \ No newline at end of file diff --git a/macros/streamline/configs.yaml.sql b/macros/streamline/configs.yaml.sql index 7395d08..7e09ec1 100644 --- a/macros/streamline/configs.yaml.sql +++ b/macros/streamline/configs.yaml.sql @@ -15,7 +15,7 @@ RUNTIME_VERSION = '3.8' HANDLER = 'hex_to_int' sql: | - {{ reference_models.python_hex_to_int() | indent(4) }} + {{ fsc_utils.python_hex_to_int() | indent(4) }} - name: {{ schema }}.udf_hex_to_int signature: - [encoding, STRING] @@ -28,7 +28,7 @@ RUNTIME_VERSION = '3.8' HANDLER = 'hex_to_int' sql: | - {{ reference_models.python_udf_hex_to_int_with_encoding() | indent(4) }} + {{ fsc_utils.python_udf_hex_to_int_with_encoding() | indent(4) }} - name: {{ schema }}.udf_hex_to_string signature: diff --git a/macros/streamline/utils.sql b/macros/streamline/utils.sql index 27acc26..f67b6dd 100644 --- a/macros/streamline/utils.sql +++ b/macros/streamline/utils.sql @@ -2,7 +2,7 @@ func_name, signature ) %} - DROP FUNCTION IF EXISTS {{ func_name }}({{ reference_models.compile_signature(signature, drop_ = True) }}); + DROP FUNCTION IF EXISTS {{ func_name }}({{ fsc_utils.compile_signature(signature, drop_ = True) }}); {% endmacro %} {%- macro construct_api_route(route) -%} @@ -35,7 +35,7 @@ func_type = none ) %} CREATE OR REPLACE {{ func_type }} FUNCTION {{ name_ }}( - {{- reference_models.compile_signature(signature) }} + {{- fsc_utils.compile_signature(signature) }} ) COPY GRANTS RETURNS {{ return_type }} @@ -44,7 +44,7 @@ {% endif %} {%- if api_integration -%} api_integration = {{ api_integration }} - AS {{ reference_models.construct_api_route(sql_) ~ ";" }} + AS {{ fsc_utils.construct_api_route(sql_) ~ ";" }} {% else -%} AS $$ @@ -66,7 +66,7 @@ {% set func_type = config ["func_type"] %} {% if not drop_ -%} - {{ reference_models.create_sql_function( + {{ fsc_utils.create_sql_function( name_ = name_, signature = signature, return_type = return_type, @@ -76,7 +76,7 @@ func_type = func_type ) }} {%- else -%} - {{ reference_models.drop_function( + {{ fsc_utils.drop_function( name_, signature = signature, ) }} diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql deleted file mode 100644 index 7cdc4d2..0000000 --- a/macros/tags/add_database_or_schema_tags.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro add_database_or_schema_tags() %} - {{ set_database_tag_value('BLOCKCHAIN_NAME','LIVEQUERY') }} -{% endmacro %} \ No newline at end of file diff --git a/macros/tags/snowflake_tagging.sql b/macros/tags/snowflake_tagging.sql deleted file mode 100644 index bc25e69..0000000 --- a/macros/tags/snowflake_tagging.sql +++ /dev/null @@ -1,127 +0,0 @@ -{% macro apply_meta_as_tags(results) %} - {% if var("UPDATE_SNOWFLAKE_TAGS") %} - {{ log('apply_meta_as_tags', info=False) }} - {{ log(results, info=False) }} - {% if execute %} - - {%- set tags_by_schema = {} -%} - {% for res in results -%} - {% if res.node.meta.database_tags %} - - {%- set model_database = res.node.database -%} - {%- set model_schema = res.node.schema -%} - {%- set model_schema_full = model_database+'.'+model_schema -%} - {%- set model_alias = res.node.alias -%} - - {% if model_schema_full not in tags_by_schema.keys() %} - {{ log('need to fetch tags for schema '+model_schema_full, info=False) }} - {%- call statement('main', fetch_result=True) -%} - show tags in {{model_database}}.{{model_schema}} - {%- endcall -%} - {%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%} - {{ log('Added tags to cache', info=False) }} - {% else %} - {{ log('already have tag info for schema', info=False) }} - {% endif %} - - {%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%} - {{ log('current_tags_in_schema:', info=False) }} - {{ log(current_tags_in_schema, info=False) }} - {{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }} - - {% set line -%} - node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }}) - node full: {{ res.node}} - meta: {{ res.node.meta}} - materialized: {{ res.node.config.materialized }} - {%- endset %} - {{ log(line, info=False) }} - - {%- call statement('main', fetch_result=True) -%} - select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table')) - {%- endcall -%} - {%- set existing_tags_for_table = load_result('main')['data'] -%} - {{ log('Existing tags for table:', info=False) }} - {{ log(existing_tags_for_table, info=False) }} - - {{ log('--', info=False) }} - {% for table_tag in res.node.meta.database_tags.table %} - - {{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }} - {% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %} - - {{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}} - {% endfor %} - {{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }} - {% endif %} - {% endfor %} - {% endif %} - {% endif %} -{% endmacro %} - - -{% macro create_tag_if_missing(all_tag_names,table_tag) %} - {% if table_tag not in all_tag_names %} - {{ log('Creating missing tag '+table_tag, info=False) }} - {%- call statement('main', fetch_result=True) -%} - create tag if not exists silver.{{table_tag}} - {%- endcall -%} - {{ log(load_result('main').data, info=False) }} - {% else %} - {{ log('Tag already exists: '+table_tag, info=False) }} - {% endif %} -{% endmacro %} - -{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %} - {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }} - {%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%} - {{ log('Filtered tags for table:', info=False) }} - {{ log(existing_tag_for_table[0], info=False) }} - {% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %} - {{ log('Correct tag value already exists', info=False) }} - {% else %} - {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} - {%- call statement('main', fetch_result=True) -%} - alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}' - {%- endcall -%} - {{ log(load_result('main').data, info=False) }} - {% endif %} -{% endmacro %} - -{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %} - {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }} - {%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%} - {{ log('Filtered tags for column:', info=False) }} - {{ log(existing_tag_for_column[0], info=False) }} - {% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %} - {{ log('Correct tag value already exists', info=False) }} - {% else %} - {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} - {%- call statement('main', fetch_result=True) -%} - alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}' - {%- endcall -%} - {{ log(load_result('main').data, info=False) }} - {% endif %} -{% endmacro %} - -{% macro set_database_tag_value(tag_name,tag_value) %} - {% set query %} - create tag if not exists silver.{{tag_name}} - {% endset %} - {% do run_query(query) %} - {% set query %} - alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' - {% endset %} - {% do run_query(query) %} -{% endmacro %} - -{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %} - {% set query %} - create tag if not exists silver.{{tag_name}} - {% endset %} - {% do run_query(query) %} - {% set query %} - alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' - {% endset %} - {% do run_query(query) %} -{% endmacro %} \ No newline at end of file diff --git a/models/doc_descriptions/general/__overview__.md b/models/doc_descriptions/general/__overview__.md index a7214af..e0ddda4 100644 --- a/models/doc_descriptions/general/__overview__.md +++ b/models/doc_descriptions/general/__overview__.md @@ -1,9 +1,9 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto Reference Models Documentation! +# Welcome to the Flipside Crypto Utility Functions Documentation! ## **What does this documentation cover?** -The documentation included here details the design of the Reference functions available via [Flipside Crypto](https://flipsidecrypto.xyz/). For more information on how these functions are built, please see [the github repository.](https://github.com/FlipsideCrypto/reference-models) +The documentation included here details the design of the utility functions available via [Flipside Crypto](https://flipsidecrypto.xyz/). For more information on how these functions are built, please see [the github repository.](https://github.com/FlipsideCrypto/fsc-utils) ### **Overview of Available Functions** @@ -64,7 +64,7 @@ Note that you can also right-click on models to interactively filter and explore ### **More information** - [Flipside](https://flipsidecrypto.xyz/) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/reference-models) +- [Github](https://github.com/FlipsideCrypto/fsc-utils) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index a75e6e5..4888674 100644 --- a/profiles.yml +++ b/profiles.yml @@ -13,7 +13,7 @@ reference: schema: SILVER threads: 4 client_session_keep_alive: False - query_tag: reference_curator + query_tag: fsc_utils_curator prod: type: snowflake account: "{{ env_var('ACCOUNT') }}" @@ -26,4 +26,4 @@ reference: schema: SILVER threads: 4 client_session_keep_alive: False - query_tag: reference_curator \ No newline at end of file + query_tag: fsc_utils_curator \ No newline at end of file