From d7cbc2b78c0b70f53223e061539a6f5268472e96 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 13 Mar 2024 16:01:58 -0400 Subject: [PATCH] reqs file --- .github/workflows/dbt_docs_update.yml | 53 +++++++++++++++++-- .github/workflows/dbt_run_adhoc.yml | 2 +- .github/workflows/dbt_run_daily.yml | 2 +- .github/workflows/dbt_run_dev_refresh.yml | 2 +- .../workflows/dbt_run_incremental_core.yml | 2 +- .../dbt_run_incremental_non_core.yml | 2 +- .github/workflows/dbt_run_observability.yml | 2 +- .../dbt_run_observability_monthly.yml | 2 +- .../workflows/dbt_run_streamline_balances.yml | 2 +- ...run_streamline_blocks_txcount_realtime.yml | 2 +- .../workflows/dbt_run_streamline_history.yml | 2 +- ...t_run_streamline_transactions_realtime.yml | 2 +- .github/workflows/dbt_run_udf_blockchain.yml | 2 +- .github/workflows/dbt_test.yml | 2 +- requirements.txt | 3 +- 15 files changed, 64 insertions(+), 18 deletions(-) diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index c9decb3..d70a604 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -1,4 +1,4 @@ -name: dbt_docs_update +name: docs_update on: push: @@ -22,6 +22,51 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7 - secrets: inherit \ No newline at end of file + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: refresh ddl for datashare + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + - name: checkout docs branch + run: | + git checkout -B docs origin/main + - name: generate dbt docs + run: dbt docs generate -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index aa726ad..f0a1944 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -60,7 +60,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index ed0dd58..04b4cbd 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index b6c832f..da08bc1 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_incremental_core.yml b/.github/workflows/dbt_run_incremental_core.yml index d64f567..be1350f 100644 --- a/.github/workflows/dbt_run_incremental_core.yml +++ b/.github/workflows/dbt_run_incremental_core.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml index aa4d016..a7a6775 100644 --- a/.github/workflows/dbt_run_incremental_non_core.yml +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index dddd0d7..5d22eef 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_observability_monthly.yml b/.github/workflows/dbt_run_observability_monthly.yml index 539dd2c..de88203 100644 --- a/.github/workflows/dbt_run_observability_monthly.yml +++ b/.github/workflows/dbt_run_observability_monthly.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_streamline_balances.yml b/.github/workflows/dbt_run_streamline_balances.yml index 650784a..bba5e8d 100644 --- a/.github/workflows/dbt_run_streamline_balances.yml +++ b/.github/workflows/dbt_run_streamline_balances.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml index dff9ad1..04db99f 100644 --- a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml +++ b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index c2e29af..770a222 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_streamline_transactions_realtime.yml b/.github/workflows/dbt_run_streamline_transactions_realtime.yml index 4be24ec..388a525 100644 --- a/.github/workflows/dbt_run_streamline_transactions_realtime.yml +++ b/.github/workflows/dbt_run_streamline_transactions_realtime.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_udf_blockchain.yml b/.github/workflows/dbt_run_udf_blockchain.yml index 915c97b..66148d7 100644 --- a/.github/workflows/dbt_run_udf_blockchain.yml +++ b/.github/workflows/dbt_run_udf_blockchain.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 78b2376..a6eadda 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -38,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click workstream-dbt-core + pip install -r requirements.txt workstream-dbt-core dbt deps - name: Run DBT Jobs run: | diff --git a/requirements.txt b/requirements.txt index 5921743..39b82bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -dbt-snowflake>=1.7,<1.8 \ No newline at end of file +dbt-snowflake>=1.7,<1.8 +protobuf==4.25.3 \ No newline at end of file