diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index b6b463d..4562a1b 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -22,6 +22,54 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7 - secrets: inherit + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: refresh ddl for datashare + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + + - name: checkout docs branch + run: | + git checkout -B docs origin/main + + - name: generate dbt docs + run: dbt docs generate -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index ed413ac..b0f1f9e 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -58,7 +58,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs diff --git a/.github/workflows/dbt_run_daily_dev_refresh.yml b/.github/workflows/dbt_run_daily_dev_refresh.yml index 655604f..d68a772 100644 --- a/.github/workflows/dbt_run_daily_dev_refresh.yml +++ b/.github/workflows/dbt_run_daily_dev_refresh.yml @@ -24,11 +24,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run-operation run_sp_create_prod_clone - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index b3921e1..79325cc 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -37,11 +37,36 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: | - ${{ inputs.dbt_command }} - warehouse: ${{ inputs.WAREHOUSE }} - environment: workflow_prod - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} + + - name: Run datashare model + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}" + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target + \ No newline at end of file diff --git a/.github/workflows/dbt_run_history.yml b/.github/workflows/dbt_run_history.yml index a733ad9..2ff9053 100644 --- a/.github/workflows/dbt_run_history.yml +++ b/.github/workflows/dbt_run_history.yml @@ -24,21 +24,40 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run -s - 2+streamline__get_transactions_history_mainnet_18 - 2+streamline__get_transactions_history_mainnet_19 - 2+streamline__get_transaction_results_history_mainnet_14 - 2+streamline__get_transaction_results_history_mainnet_15 - 2+streamline__get_transaction_results_history_mainnet_17 - 2+streamline__get_transaction_results_history_mainnet_16 - 2+streamline__get_transaction_results_history_mainnet_18 - 2+streamline__get_transaction_results_history_mainnet_19 - 2+streamline__get_transaction_results_history_mainnet_22 - --vars '{"STREAMLINE_INVOKE_STREAMS": True}' - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: > + dbt run -s + 2+streamline__get_transactions_history_mainnet_18 + 2+streamline__get_transactions_history_mainnet_19 + 2+streamline__get_transaction_results_history_mainnet_14 + 2+streamline__get_transaction_results_history_mainnet_15 + 2+streamline__get_transaction_results_history_mainnet_17 + 2+streamline__get_transaction_results_history_mainnet_16 + 2+streamline__get_transaction_results_history_mainnet_18 + 2+streamline__get_transaction_results_history_mainnet_19 + 2+streamline__get_transaction_results_history_mainnet_22 + --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_moments_metadata.yml b/.github/workflows/dbt_run_moments_metadata.yml index 15568e9..2ea8d97 100644 --- a/.github/workflows/dbt_run_moments_metadata.yml +++ b/.github/workflows/dbt_run_moments_metadata.yml @@ -22,16 +22,35 @@ env: concurrency: group: ${{ github.workflow }} - + jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run -m silver__allday_moments_metadata_needed_s.sql; - dbt run-operation get_allday_metadata; - dbt run -m silver__nft_allday_metadata_s; - dbt run -s tag:topshot; - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: > + dbt run -m silver__allday_moments_metadata_needed_s.sql; + dbt run-operation get_allday_metadata; + dbt run -m silver__nft_allday_metadata_s; + dbt run -s tag:topshot; + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index 78b7663..f09da4a 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -24,15 +24,34 @@ env: concurrency: group: ${{ github.workflow }} - + jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; - dbt seed; - dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: > + dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; + dbt seed; + dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml index d5d132e..7ba4681 100644 --- a/.github/workflows/dbt_run_scheduled_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -22,15 +22,35 @@ env: concurrency: group: ${{ github.workflow }} - + jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 - with: - dbt_command: > - dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; - dbt seed; - dbt run -s tag:scheduled_non_core --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: > + dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; + dbt seed; + dbt run -s tag:scheduled_non_core --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target + \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 9c7307d..5be6780 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -36,10 +36,11 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.10" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs @@ -50,3 +51,12 @@ jobs: - name: Log test results run: | python python/dbt_test_alert.py + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target + \ No newline at end of file