From b8ef32172ad5c1cb2b3ba05c796d685d29d3cd63 Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:23:27 -0700 Subject: [PATCH] template (#514) --- .github/workflows/dbt_docs_update.yml | 51 +++++++++++++++++-- .github/workflows/dbt_run_adhoc.yml | 2 +- .github/workflows/dbt_run_batch_backfill.yml | 2 +- .github/workflows/dbt_run_daily.yml | 2 +- .../workflows/dbt_run_decode_instructions.yml | 2 +- .../dbt_run_decode_instructions_backfill.yml | 2 +- ...t_run_decode_instructions_orchestrator.yml | 2 +- .../dbt_run_decode_instructions_pyth.yml | 2 +- .github/workflows/dbt_run_deploy_seeds.yml | 2 +- .../workflows/dbt_run_deploy_udfs_and_sps.yml | 2 +- .github/workflows/dbt_run_deployment.yml | 37 +++++++++++--- .github/workflows/dbt_run_dev_refresh.yml | 2 +- .../workflows/dbt_run_full_observability.yml | 2 +- .github/workflows/dbt_run_helius_metadata.yml | 2 +- .github/workflows/dbt_run_idls_history.yml | 2 +- .github/workflows/dbt_run_incremental.yml | 2 +- .../dbt_run_incremental_non_core.yml | 2 +- .../dbt_run_macro_get_block_production.yml | 2 +- .../dbt_run_macro_get_compressed_nft.yml | 2 +- .../dbt_run_macro_get_token_data.yml | 2 +- .../dbt_run_nft_compressed_mints_backfill.yml | 2 +- .../dbt_run_nft_compressed_mints_realtime.yml | 2 +- .github/workflows/dbt_run_solscan_blocks.yml | 2 +- .github/workflows/dbt_run_weekly.yml | 2 +- .github/workflows/dbt_test.yml | 2 +- .github/workflows/dbt_test_hourly.yml | 2 +- .github/workflows/dbt_test_intraday.yml | 2 +- .github/workflows/dbt_test_weekly.yml | 2 +- 28 files changed, 103 insertions(+), 37 deletions(-) diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 04531843..2c5bce9b 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -23,6 +23,51 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7 - secrets: inherit \ No newline at end of file +run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: refresh ddl for datashare + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + - name: checkout docs branch + run: | + git checkout -B docs origin/main + - name: generate dbt docs + run: dbt docs generate -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 2ede01f1..a7bc8c5d 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -59,7 +59,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_batch_backfill.yml b/.github/workflows/dbt_run_batch_backfill.yml index ae74bfac..902160df 100644 --- a/.github/workflows/dbt_run_batch_backfill.yml +++ b/.github/workflows/dbt_run_batch_backfill.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 726685b6..d81562b3 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_decode_instructions.yml b/.github/workflows/dbt_run_decode_instructions.yml index 0a46867b..8b3f1da8 100644 --- a/.github/workflows/dbt_run_decode_instructions.yml +++ b/.github/workflows/dbt_run_decode_instructions.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_decode_instructions_backfill.yml b/.github/workflows/dbt_run_decode_instructions_backfill.yml index ca6480d9..a851bd8a 100644 --- a/.github/workflows/dbt_run_decode_instructions_backfill.yml +++ b/.github/workflows/dbt_run_decode_instructions_backfill.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_decode_instructions_orchestrator.yml b/.github/workflows/dbt_run_decode_instructions_orchestrator.yml index 1b61a6b8..e1ed4c29 100644 --- a/.github/workflows/dbt_run_decode_instructions_orchestrator.yml +++ b/.github/workflows/dbt_run_decode_instructions_orchestrator.yml @@ -48,7 +48,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs diff --git a/.github/workflows/dbt_run_decode_instructions_pyth.yml b/.github/workflows/dbt_run_decode_instructions_pyth.yml index c9aa744e..c0446eb0 100644 --- a/.github/workflows/dbt_run_decode_instructions_pyth.yml +++ b/.github/workflows/dbt_run_decode_instructions_pyth.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_deploy_seeds.yml b/.github/workflows/dbt_run_deploy_seeds.yml index f76e5d8b..b014f986 100644 --- a/.github/workflows/dbt_run_deploy_seeds.yml +++ b/.github/workflows/dbt_run_deploy_seeds.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs diff --git a/.github/workflows/dbt_run_deploy_udfs_and_sps.yml b/.github/workflows/dbt_run_deploy_udfs_and_sps.yml index 78ad0ba3..3e8b2ba6 100644 --- a/.github/workflows/dbt_run_deploy_udfs_and_sps.yml +++ b/.github/workflows/dbt_run_deploy_udfs_and_sps.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index f69e52b5..457646e8 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -37,11 +37,32 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: | - ${{ inputs.dbt_command }} - environment: workflow_prod - warehouse: ${{ inputs.WAREHOUSE }} - secrets: inherit \ No newline at end of file + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} + - name: Run datashare model + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}" + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 5d892ee3..c547fbd7 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index 16ed3f75..5acbf743 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_helius_metadata.yml b/.github/workflows/dbt_run_helius_metadata.yml index 5d405647..08f886fb 100644 --- a/.github/workflows/dbt_run_helius_metadata.yml +++ b/.github/workflows/dbt_run_helius_metadata.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_idls_history.yml b/.github/workflows/dbt_run_idls_history.yml index c760209a..6db1e423 100644 --- a/.github/workflows/dbt_run_idls_history.yml +++ b/.github/workflows/dbt_run_idls_history.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 8cadfd74..1ddcd2ac 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml index 67768658..e4683183 100644 --- a/.github/workflows/dbt_run_incremental_non_core.yml +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_macro_get_block_production.yml b/.github/workflows/dbt_run_macro_get_block_production.yml index 4194c49c..cad1d155 100644 --- a/.github/workflows/dbt_run_macro_get_block_production.yml +++ b/.github/workflows/dbt_run_macro_get_block_production.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_macro_get_compressed_nft.yml b/.github/workflows/dbt_run_macro_get_compressed_nft.yml index 1ef9e9f5..fd54772e 100644 --- a/.github/workflows/dbt_run_macro_get_compressed_nft.yml +++ b/.github/workflows/dbt_run_macro_get_compressed_nft.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_macro_get_token_data.yml b/.github/workflows/dbt_run_macro_get_token_data.yml index 66b1d524..20586b19 100644 --- a/.github/workflows/dbt_run_macro_get_token_data.yml +++ b/.github/workflows/dbt_run_macro_get_token_data.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml b/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml index e6f2f414..6b395793 100644 --- a/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml +++ b/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml b/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml index 35d0840a..9f1f2f3d 100644 --- a/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml +++ b/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_solscan_blocks.yml b/.github/workflows/dbt_run_solscan_blocks.yml index f4147353..70f63d08 100644 --- a/.github/workflows/dbt_run_solscan_blocks.yml +++ b/.github/workflows/dbt_run_solscan_blocks.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_weekly.yml b/.github/workflows/dbt_run_weekly.yml index a35780d6..0db2c96b 100644 --- a/.github/workflows/dbt_run_weekly.yml +++ b/.github/workflows/dbt_run_weekly.yml @@ -37,7 +37,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 29ebbc0e..e20b973e 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test_hourly.yml b/.github/workflows/dbt_test_hourly.yml index f2df39a3..f85fa983 100644 --- a/.github/workflows/dbt_test_hourly.yml +++ b/.github/workflows/dbt_test_hourly.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index fe0e4282..eded34f4 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test_weekly.yml b/.github/workflows/dbt_test_weekly.yml index 03e323df..bff3bd6d 100644 --- a/.github/workflows/dbt_test_weekly.yml +++ b/.github/workflows/dbt_test_weekly.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: |