From 5bca1da0744d2247ff6edbd63b59e303206c3b36 Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 1 Oct 2024 23:54:37 -0400 Subject: [PATCH] clean --- .../dbt_run_streamline_chainhead.yml | 2 +- README.md | 91 ++++++++++++------- dbt_project.yml | 1 + macros/create_sps.sql | 12 ++- macros/create_udfs.sql | 2 +- macros/run_sp_create_prod_clone.sql | 16 ++-- macros/tags/add_database_or_schema_tags.sql | 3 +- models/sources.yml | 8 +- profiles.yml | 31 ------- 9 files changed, 84 insertions(+), 82 deletions(-) delete mode 100644 profiles.yml diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 40109c4..6c2d6a7 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "core_models,tag:streamline_testnet_core_complete" "core_models,tag:streamline_testnet_realtime" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m \ No newline at end of file diff --git a/README.md b/README.md index 6941edf..8ce1190 100644 --- a/README.md +++ b/README.md @@ -4,48 +4,81 @@ ---- ```yml -core: +{{ chain }}: -- see dbt_project.yml for the project name, replace {{ chain }}/{{ CHAIN }} with the project name, remove this comment target: dev outputs: dev: type: snowflake account: - role: + role: INTERNAL_DEV user: - password: - region: - database: CORE_DEV - warehouse: + authenticator: externalbrowser + region: us-east-1 + database: {{ CHAIN }}_DEV + warehouse: DBT schema: silver - threads: 12 + threads: 4 client_session_keep_alive: False - query_tag: + query_tag: dbt__dev + prod: type: snowflake account: - role: + role: DBT_CLOUD_{{ CHAIN }} user: - password: - region: - database: CORE - warehouse: + authenticator: externalbrowser + region: us-east-1 + database: {{ CHAIN }} + warehouse: DBT_CLOUD_{{ CHAIN }} schema: silver - threads: 12 + threads: 4 client_session_keep_alive: False - query_tag: + query_tag: dbt__dev ``` -### Variables +### Common DBT Run Variables -To control the creation of UDF or SP macros with dbt run: -* UPDATE_UDFS_AND_SPS -When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal -When False, none of the on-run-start macros are executed on model run +The following variables can be used to control various aspects of the dbt run. Use them with the `--vars` flag when running dbt commands. -Default values are False +| Variable | Description | Example Usage | +|----------|-------------|---------------| +| `UPDATE_UDFS_AND_SPS` | Update User Defined Functions and Stored Procedures. By default, this is set to False | `--vars '{"UPDATE_UDFS_AND_SPS":true}'` | +| `STREAMLINE_INVOKE_STREAMS` | Invoke Streamline processes. By default, this is set to False | `--vars '{"STREAMLINE_INVOKE_STREAMS":true}'` | +| `STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES` | Use development environment for external tables. By default, this is set to False | `--vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}'` | +| `HEAL_CURATED_MODEL` | Heal specific curated models. By default, this is set to an empty array []. See more below. | `--vars '{"HEAL_CURATED_MODEL":["axelar","across","celer_cbridge"]}'` | +| `UPDATE_SNOWFLAKE_TAGS` | Control updating of Snowflake tags. By default, this is set to False | `--vars '{"UPDATE_SNOWFLAKE_TAGS":false}'` | +| `START_GHA_TASKS` | Start GitHub Actions tasks. By default, this is set to False | `--vars '{"START_GHA_TASKS":true}'` | -* Usage: -dbt run --vars '{"UPDATE_UDFS_AND_SPS":True}' -m ... +#### Example Commands + +1. Update UDFs and SPs: + ``` + dbt run --vars '{"UPDATE_UDFS_AND_SPS":true}' -m ... + ``` + +2. Invoke Streamline and use dev for external tables: + ``` + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":true,"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -m ... + ``` + +3. Heal specific curated models: + ``` + dbt run --vars '{"HEAL_CURATED_MODEL":["axelar","across","celer_cbridge"]}' -m ... + ``` + +4. Update Snowflake tags for a specific model: + ``` + dbt run --vars '{"UPDATE_SNOWFLAKE_TAGS":true}' -s models/silver/utilities/silver__number_sequence.sql + ``` + +5. Start GHA tasks: + ``` + dbt seed -s github_actions__workflows && dbt run -m models/github_actions --full-refresh && dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' + ``` + +> Note: Replace `-m ...` with appropriate model selections or tags as needed for your project structure. + +### Healing Curated Models To reload records in a curated complete table without a full-refresh, such as `silver_bridge.complete_bridge_activity`: * HEAL_CURATED_MODEL @@ -58,14 +91,6 @@ Example set up: `{% if is_incremental() and 'axelar' not in var('HEAL_CURATED_MO Single CTE: dbt run --vars '{"HEAL_CURATED_MODEL":"axelar"}' -m ... Multiple CTEs: dbt run --vars '{"HEAL_CURATED_MODEL":["axelar","across","celer_cbridge"]}' -m ... - -### Resources: -- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) -- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers -- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support -- Find [dbt events](https://events.getdbt.com) near you -- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices - ## Applying Model Tags ### Database / Schema level tags @@ -98,12 +123,12 @@ To add/update a model's snowflake tags, add/modify the `meta` model property und By default, model tags are pushed to Snowflake on each load. You can disable this by setting the `UPDATE_SNOWFLAKE_TAGS` project variable to `False` during a run. ``` -dbt run --vars '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/testnet/testnet__fact_blocks.sql +dbt run --vars '{"UPDATE_SNOWFLAKE_TAGS":False}' -s models/silver/utilities/silver__number_sequence.sql ``` ### Querying for existing tags on a model in snowflake ``` select * -from table(core.information_schema.tag_references('core.testnet.fact_blocks', 'table')); +from table({{ chain }}.information_schema.tag_references('{{ chain }}.core.fact_blocks', 'table')); ``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 17d0ba5..3f9d151 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -100,6 +100,7 @@ vars: #### FSC_EVM BEGIN #### + PROD_DB_NAME: 'core' API_URL: '{Service}/{Authentication}' VAULT_SECRET_PATH: 'Vault/prod/core/ankr/mainnet' START_UP_BLOCK: 17962900 ### OPTIONAL, default is 0 diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 309209d..5acc252 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,8 +1,10 @@ {% macro create_sps() %} - {% if var("UPDATE_UDFS_AND_SPS") %} - {% if target.database == 'CORE' %} - CREATE schema IF NOT EXISTS _internal; - {{ sp_create_prod_clone('_internal') }}; + {% if var("UPDATE_UDFS_AND_SPS", false) %} + {% set current_profile = var('PROD_DB_NAME') | upper %} + {% if target.database | upper == current_profile and target.name == 'prod' %} + {% set schema_name = var("SPS_SCHEMA_NAME", '_internal') %} + CREATE SCHEMA IF NOT EXISTS {{ schema_name }}; + {{ sp_create_prod_clone(schema_name) }}; {% endif %} {% endif %} -{% endmacro %} +{% endmacro %} \ No newline at end of file diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 2477ba8..b045261 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -6,4 +6,4 @@ {% do run_query(sql) %} {{- fsc_utils.create_udfs() -}} {% endif %} -{% endmacro %} +{% endmacro %} \ No newline at end of file diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql index 76eb095..475a82a 100644 --- a/macros/run_sp_create_prod_clone.sql +++ b/macros/run_sp_create_prod_clone.sql @@ -1,10 +1,14 @@ {% macro run_sp_create_prod_clone() %} + {% set prod_db_name = var('PROD_DB_NAME') | upper %} + {% set dev_suffix = var('DEV_DATABASE_SUFFIX', '_DEV') %} + {% set clone_role = var('CLONE_ROLE', 'internal_dev') %} + {% set clone_query %} - call core._internal.create_prod_clone( - 'core', - 'core_dev', - 'internal_dev' + call {{ prod_db_name }}._internal.create_prod_clone( + '{{ prod_db_name }}', + '{{ prod_db_name }}{{ dev_suffix }}', + '{{ clone_role }}' ); -{% endset %} + {% endset %} {% do run_query(clone_query) %} -{% endmacro %} +{% endmacro %} \ No newline at end of file diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql index f0bf38b..23200db 100644 --- a/macros/tags/add_database_or_schema_tags.sql +++ b/macros/tags/add_database_or_schema_tags.sql @@ -1,4 +1,5 @@ {% macro add_database_or_schema_tags() %} - {{ set_database_tag_value('BLOCKCHAIN_NAME','BERACHAIN') }} + {% set current_profile = var('profile') | upper %} + {{ set_database_tag_value('BLOCKCHAIN_NAME', current_profile) }} {{ set_database_tag_value('BLOCKCHAIN_TYPE','EVM') }} {% endmacro %} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 8673ad5..342277a 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,15 +1,15 @@ -version: 2 +version: 2 sources: - name: github_actions - database: "{{ 'CORE' if target.database == 'CORE' else 'CORE_DEV' }}" + database: "{{ target.database }}" schema: github_actions tables: - name: workflows - name: bronze_streamline database: streamline - schema: | - {{ "CORE_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "CORE" }} + schema: >- + {{ var('PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }} tables: - name: blocks - name: transactions diff --git a/profiles.yml b/profiles.yml deleted file mode 100644 index 70ae0d9..0000000 --- a/profiles.yml +++ /dev/null @@ -1,31 +0,0 @@ -core: - target: prod - outputs: - dev: - type: snowflake - account: "{{ env_var('ACCOUNT') }}" - role: "{{ env_var('ROLE') }}" - user: "{{ env_var('USER') }}" - password: "{{ env_var('PASSWORD') }}" - region: "{{ env_var('REGION') }}" - database: "{{ env_var('DATABASE') }}" - warehouse: "{{ env_var('WAREHOUSE') }}" - schema: SILVER - threads: 4 - client_session_keep_alive: False - query_tag: core_curator - prod: - type: snowflake - account: "{{ env_var('ACCOUNT') }}" - role: "{{ env_var('ROLE') }}" - user: "{{ env_var('USER') }}" - password: "{{ env_var('PASSWORD') }}" - region: "{{ env_var('REGION') }}" - database: "{{ env_var('DATABASE') }}" - warehouse: "{{ env_var('WAREHOUSE') }}" - schema: SILVER - threads: 4 - client_session_keep_alive: False - query_tag: core_curator - config: - send_anonymous_usage_stats: False \ No newline at end of file