diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index f9bca76b..72ebf14f 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -73,3 +73,10 @@ jobs: - name: push changes to docs run: | git push -f --set-upstream origin docs + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index a7bc8c5d..f5c95080 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -64,4 +64,10 @@ jobs: - name: Run DBT Jobs run: | ${{ matrix.command }} - \ No newline at end of file + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_backfill_transfers_mints.yml b/.github/workflows/dbt_run_backfill_transfers_mints.yml deleted file mode 100644 index 46665fa9..00000000 --- a/.github/workflows/dbt_run_backfill_transfers_mints.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_backfill_transfers_mints -run-name: dbt_run_backfill_transfers_mints - -on: - workflow_dispatch: - branches: - - "tmp-backfill-transfers-missing-mints" - schedule: - - cron: "*/3 * * * *" - -env: - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s silver__transfers_null_mints_fix -t dev diff --git a/.github/workflows/dbt_run_batch_backfill_2.yml b/.github/workflows/dbt_run_batch_backfill_2.yml deleted file mode 100644 index 75af5f77..00000000 --- a/.github/workflows/dbt_run_batch_backfill_2.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_batch_backfill_2 -run-name: dbt_run_batch_backfill_2 - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: DBT_EMERGENCY - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s "solana_models,tag:events_inner_backfill" \ No newline at end of file diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index f250d4a2..eb49f9ed 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -49,3 +49,9 @@ jobs: dbt run -s "solana_models,tag:nft_api" dbt run -s "solana_models,tag:daily" + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_decode_instructions.yml b/.github/workflows/dbt_run_decode_instructions.yml index b4909b6f..29931e2a 100644 --- a/.github/workflows/dbt_run_decode_instructions.yml +++ b/.github/workflows/dbt_run_decode_instructions.yml @@ -41,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s models/streamline/decode_instructions/streamline__decode_instructions_3_realtime.sql \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s models/streamline/decode_instructions/streamline__decode_instructions_3_realtime.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_decode_instructions_backfill.yml b/.github/workflows/dbt_run_decode_instructions_backfill.yml index a851bd8a..83d2876d 100644 --- a/.github/workflows/dbt_run_decode_instructions_backfill.yml +++ b/.github/workflows/dbt_run_decode_instructions_backfill.yml @@ -42,4 +42,11 @@ jobs: - name: Run DBT Jobs run: | dbt run-operation decoded_instructions_backill_cleanup_views - dbt run-operation decoded_instructions_backfill_calls --vars '{"STREAMLINE_INVOKE_STREAMS": True}' \ No newline at end of file + dbt run-operation decoded_instructions_backfill_calls --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_decode_instructions_orchestrator.yml b/.github/workflows/dbt_run_decode_instructions_orchestrator.yml index 1e474a07..e055d00e 100644 --- a/.github/workflows/dbt_run_decode_instructions_orchestrator.yml +++ b/.github/workflows/dbt_run_decode_instructions_orchestrator.yml @@ -69,3 +69,10 @@ jobs: run: | dbt run-operation dispatch_github_workflow --args "{'workflow_name': 'dbt_run_decode_instructions_backfill'}" if: ${{ github.event_name == 'workflow_dispatch' && contains(fromJSON('["50"]'), steps.capture_minute.outputs.formatted_minute) }} + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_decode_instructions_pyth.yml b/.github/workflows/dbt_run_decode_instructions_pyth.yml index a6989a66..c3b3ba91 100644 --- a/.github/workflows/dbt_run_decode_instructions_pyth.yml +++ b/.github/workflows/dbt_run_decode_instructions_pyth.yml @@ -42,3 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s models/streamline/decode_instructions/streamline__decode_instructions_3_pyth_realtime.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_decode_logs.yml b/.github/workflows/dbt_run_decode_logs.yml index f410434d..a5792e3d 100644 --- a/.github/workflows/dbt_run_decode_logs.yml +++ b/.github/workflows/dbt_run_decode_logs.yml @@ -42,3 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s models/streamline/decode_logs/streamline__decode_logs_2_realtime.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_decode_logs_backfill.yml b/.github/workflows/dbt_run_decode_logs_backfill.yml index 260536e2..5698916a 100644 --- a/.github/workflows/dbt_run_decode_logs_backfill.yml +++ b/.github/workflows/dbt_run_decode_logs_backfill.yml @@ -42,4 +42,11 @@ jobs: - name: Run DBT Jobs run: | dbt run-operation decoded_logs_backill_cleanup_views - dbt run-operation decoded_logs_backfill_calls --vars '{"STREAMLINE_INVOKE_STREAMS": True}' \ No newline at end of file + dbt run-operation decoded_logs_backfill_calls --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_decode_logs_orchestrator.yml b/.github/workflows/dbt_run_decode_logs_orchestrator.yml index 355028a9..2e5f73c4 100644 --- a/.github/workflows/dbt_run_decode_logs_orchestrator.yml +++ b/.github/workflows/dbt_run_decode_logs_orchestrator.yml @@ -63,4 +63,11 @@ jobs: - name: Run Backfill on minutes 45 every hour run: | dbt run-operation dispatch_github_workflow --args "{'workflow_name': 'dbt_run_decode_logs_backfill'}" - if: ${{ github.event_name == 'workflow_dispatch' && contains(fromJSON('["45"]'), steps.capture_minute.outputs.formatted_minute) }} \ No newline at end of file + if: ${{ github.event_name == 'workflow_dispatch' && contains(fromJSON('["45"]'), steps.capture_minute.outputs.formatted_minute) }} + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_deploy_seeds.yml b/.github/workflows/dbt_run_deploy_seeds.yml index 59c85603..e136a6cc 100644 --- a/.github/workflows/dbt_run_deploy_seeds.yml +++ b/.github/workflows/dbt_run_deploy_seeds.yml @@ -45,4 +45,11 @@ jobs: - name: Run DBT Jobs run: | dbt seed -s "solana_models,./data" --full-refresh + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index 1aa464a1..47c474ff 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -42,4 +42,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"OBSERV_FULL_TEST":True}' -m "solana_models,models/silver/_observability" \ No newline at end of file + dbt run --vars '{"OBSERV_FULL_TEST":True}' -m "solana_models,models/silver/_observability" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_helius_metadata.yml b/.github/workflows/dbt_run_helius_metadata.yml index 08f886fb..bf2798b9 100644 --- a/.github/workflows/dbt_run_helius_metadata.yml +++ b/.github/workflows/dbt_run_helius_metadata.yml @@ -43,5 +43,12 @@ jobs: - name: Run DBT Jobs run: | dbt run -s models/silver/metadata/helius/silver__helius_nft_requests.sql models/bronze/bronze_api/bronze_api__helius_nft_metadata.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_idls_history.yml b/.github/workflows/dbt_run_idls_history.yml index 6db1e423..b1930044 100644 --- a/.github/workflows/dbt_run_idls_history.yml +++ b/.github/workflows/dbt_run_idls_history.yml @@ -41,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -s models/streamline/parser/history/streamline__idls_history.sql models/streamline/parser/history/streamline__complete_decoded_history.sql \ No newline at end of file + dbt run -s models/streamline/parser/history/streamline__idls_history.sql models/streamline/parser/history/streamline__complete_decoded_history.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index ee14a51f..14b565d6 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -43,4 +43,11 @@ jobs: run: | dbt run-operation run_sp_refresh_external_tables_full dbt run -s "solana_models,tag:scheduled_core" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_incremental_intermediate.yml b/.github/workflows/dbt_run_incremental_intermediate.yml index 208b5bf6..00175a21 100644 --- a/.github/workflows/dbt_run_incremental_intermediate.yml +++ b/.github/workflows/dbt_run_incremental_intermediate.yml @@ -43,4 +43,11 @@ jobs: run: | dbt run-operation run_sp_refresh_external_tables_full dbt run -s "solana_models,tag:scheduled_core" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml index e4683183..7dc1ed3b 100644 --- a/.github/workflows/dbt_run_incremental_non_core.yml +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -44,4 +44,10 @@ jobs: run: | dbt run-operation run_sp_refresh_external_tables_full dbt run -s "solana_models,tag:scheduled_core" "solana_models,tag:scheduled_non_core" - + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_incremental_non_core_hourly.yml b/.github/workflows/dbt_run_incremental_non_core_hourly.yml index e36f2034..a4f91a95 100644 --- a/.github/workflows/dbt_run_incremental_non_core_hourly.yml +++ b/.github/workflows/dbt_run_incremental_non_core_hourly.yml @@ -39,4 +39,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -s "solana_models,tag:scheduled_non_core_hourly" \ No newline at end of file + dbt run -s "solana_models,tag:scheduled_non_core_hourly" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_macro_get_block_production.yml b/.github/workflows/dbt_run_macro_get_block_production.yml index 7a9a1a0d..081ce480 100644 --- a/.github/workflows/dbt_run_macro_get_block_production.yml +++ b/.github/workflows/dbt_run_macro_get_block_production.yml @@ -45,3 +45,10 @@ jobs: dbt run-operation get_block_production dbt run -s models/silver/validator/silver__snapshot_block_production_2.sql + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + diff --git a/.github/workflows/dbt_run_macro_get_compressed_nft.yml b/.github/workflows/dbt_run_macro_get_compressed_nft.yml index fd54772e..874e1138 100644 --- a/.github/workflows/dbt_run_macro_get_compressed_nft.yml +++ b/.github/workflows/dbt_run_macro_get_compressed_nft.yml @@ -43,3 +43,10 @@ jobs: run: | dbt run -s models/silver/nfts/silver__nft_compressed_mints_onchain.sql dbt run -s models/silver/nfts/silver__nft_compressed_mints.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml b/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml index 9ccbba86..caad4bbc 100644 --- a/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml +++ b/.github/workflows/dbt_run_nft_compressed_mints_backfill.yml @@ -43,4 +43,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run-operation nft_compressed_mints_backfill_make_call \ No newline at end of file + dbt run-operation nft_compressed_mints_backfill_make_call + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml b/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml index 7a392e3f..84ea149f 100644 --- a/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml +++ b/.github/workflows/dbt_run_nft_compressed_mints_realtime.yml @@ -47,4 +47,11 @@ jobs: dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql - dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql \ No newline at end of file + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_nft_compressed_mints_sales_backfill.yml b/.github/workflows/dbt_run_nft_compressed_mints_sales_backfill.yml index 0f5668b4..9684daa7 100644 --- a/.github/workflows/dbt_run_nft_compressed_mints_sales_backfill.yml +++ b/.github/workflows/dbt_run_nft_compressed_mints_sales_backfill.yml @@ -41,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_magic_eden.sql \ No newline at end of file + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_magic_eden.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml b/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml index f73f782a..83ee8a8c 100644 --- a/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml +++ b/.github/workflows/dbt_run_nft_compressed_mints_sales_realtime.yml @@ -41,4 +41,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_magic_eden.sql \ No newline at end of file + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_sales_magic_eden.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_snapshot_validators.yml b/.github/workflows/dbt_run_snapshot_validators.yml deleted file mode 100644 index 493fcd74..00000000 --- a/.github/workflows/dbt_run_snapshot_validators.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: dbt_run_snapshot_validators -run-name: dbt_run_snapshot_validators - -on: - workflow_dispatch: - schedule: - # Runs 02:22 daily (see https://crontab.guru) - - cron: '22 2 * * *' - -env: - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s "solana_models,tag:validator_2" diff --git a/.github/workflows/dbt_run_stage_block_txs.yml b/.github/workflows/dbt_run_stage_block_txs.yml index f888eba6..4fc2826c 100644 --- a/.github/workflows/dbt_run_stage_block_txs.yml +++ b/.github/workflows/dbt_run_stage_block_txs.yml @@ -42,3 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run -s bronze__stage_block_txs_2 + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_block_rewards.yml b/.github/workflows/dbt_run_streamline_block_rewards.yml index 322be82c..89c7d6c9 100644 --- a/.github/workflows/dbt_run_streamline_block_rewards.yml +++ b/.github/workflows/dbt_run_streamline_block_rewards.yml @@ -42,4 +42,11 @@ jobs: - name: Run DBT Jobs run: | dbt run -s models/streamline/core/complete/streamline__complete_block_rewards_2.sql - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s streamline__block_rewards_2 \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s streamline__block_rewards_2 + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_block_txs.yml b/.github/workflows/dbt_run_streamline_block_txs.yml index 54c9acc9..a55345a6 100644 --- a/.github/workflows/dbt_run_streamline_block_txs.yml +++ b/.github/workflows/dbt_run_streamline_block_txs.yml @@ -42,3 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s streamline__complete_block_txs_2 streamline__block_txs_2 + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_blocks.yml b/.github/workflows/dbt_run_streamline_blocks.yml index 4f7d199e..929a8078 100644 --- a/.github/workflows/dbt_run_streamline_blocks.yml +++ b/.github/workflows/dbt_run_streamline_blocks.yml @@ -42,3 +42,10 @@ jobs: - name: Run DBT Jobs run: | dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s streamline__complete_blocks_2 streamline__blocks_2 + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_helius_cnft_metadata.yml b/.github/workflows/dbt_run_streamline_helius_cnft_metadata.yml index 932fa99e..c409032f 100644 --- a/.github/workflows/dbt_run_streamline_helius_cnft_metadata.yml +++ b/.github/workflows/dbt_run_streamline_helius_cnft_metadata.yml @@ -42,4 +42,11 @@ jobs: - name: Run DBT Jobs run: | dbt run -s streamline__complete_helius_cnft_metadata_requests - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s streamline__helius_cnft_metadata_requests \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s streamline__helius_cnft_metadata_requests + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_solscan_blocks.yml b/.github/workflows/dbt_run_streamline_solscan_blocks.yml index ea31ff38..77c35336 100644 --- a/.github/workflows/dbt_run_streamline_solscan_blocks.yml +++ b/.github/workflows/dbt_run_streamline_solscan_blocks.yml @@ -43,3 +43,10 @@ jobs: run: | dbt run -s silver___blocks_tx_count dbt run --vars '{"STREAMLINE_INVOKE_STREAMS": True}' -s streamline__solscan_blocks + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_solscan_token_list.yml b/.github/workflows/dbt_run_streamline_solscan_token_list.yml index f9b19102..4568e653 100644 --- a/.github/workflows/dbt_run_streamline_solscan_token_list.yml +++ b/.github/workflows/dbt_run_streamline_solscan_token_list.yml @@ -44,3 +44,10 @@ jobs: run: | dbt run -s "solana_models,streamline__solscan_token_list_by_market_cap" --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + diff --git a/.github/workflows/dbt_run_streamline_stake_accounts_snapshot.yml b/.github/workflows/dbt_run_streamline_stake_accounts_snapshot.yml index 52c04514..2ff73718 100644 --- a/.github/workflows/dbt_run_streamline_stake_accounts_snapshot.yml +++ b/.github/workflows/dbt_run_streamline_stake_accounts_snapshot.yml @@ -44,3 +44,10 @@ jobs: dbt build -s "solana_models,tag:streamline_stake_accounts" dbt run -s "solana_models,streamline__stake_program_accounts_2" --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + diff --git a/.github/workflows/dbt_run_streamline_validator_metadata_snapshot.yml b/.github/workflows/dbt_run_streamline_validator_metadata_snapshot.yml index b5e2b710..c9c66bb6 100644 --- a/.github/workflows/dbt_run_streamline_validator_metadata_snapshot.yml +++ b/.github/workflows/dbt_run_streamline_validator_metadata_snapshot.yml @@ -44,3 +44,9 @@ jobs: run: | dbt run -s "solana_models,streamline__validator_metadata_2" --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_validator_vote_accounts_snapshot.yml b/.github/workflows/dbt_run_streamline_validator_vote_accounts_snapshot.yml index 276aff03..41a66b32 100644 --- a/.github/workflows/dbt_run_streamline_validator_vote_accounts_snapshot.yml +++ b/.github/workflows/dbt_run_streamline_validator_vote_accounts_snapshot.yml @@ -44,3 +44,9 @@ jobs: run: | dbt run -s "solana_models,streamline__validator_vote_accounts_2" --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_validator_vote_program_accounts_snapshot.yml b/.github/workflows/dbt_run_streamline_validator_vote_program_accounts_snapshot.yml index 7f9252b2..3d1dbd08 100644 --- a/.github/workflows/dbt_run_streamline_validator_vote_program_accounts_snapshot.yml +++ b/.github/workflows/dbt_run_streamline_validator_vote_program_accounts_snapshot.yml @@ -44,3 +44,9 @@ jobs: run: | dbt run -s "solana_models,streamline__validator_vote_program_accounts_2" --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_validators_list_snapshot.yml b/.github/workflows/dbt_run_streamline_validators_list_snapshot.yml index bb198669..9e55cde0 100644 --- a/.github/workflows/dbt_run_streamline_validators_list_snapshot.yml +++ b/.github/workflows/dbt_run_streamline_validators_list_snapshot.yml @@ -44,3 +44,9 @@ jobs: run: | dbt run -s "solana_models,streamline__validators_list_2" --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_transactions_backfill.yml b/.github/workflows/dbt_run_transactions_backfill.yml deleted file mode 100644 index cc126f37..00000000 --- a/.github/workflows/dbt_run_transactions_backfill.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_transactions_backfill -run-name: dbt_run_transactions_backfill - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s "solana_models,tag:transactions_votes_with_non_votes_backfill" - dbt run -s "solana_models,tag:transactions_votes_with_non_votes_backfill" - dbt run -s "solana_models,tag:transactions_votes_with_non_votes_backfill" \ No newline at end of file diff --git a/.github/workflows/dbt_run_transactions_units_consumed_backfill.yml b/.github/workflows/dbt_run_transactions_units_consumed_backfill.yml deleted file mode 100644 index 01f87802..00000000 --- a/.github/workflows/dbt_run_transactions_units_consumed_backfill.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: dbt_run_transactions_units_consumed_backfill -run-name: dbt_run_transactions_units_consumed_backfill - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ vars.PYTHON_VERSION }}" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s "solana_models,tag:units_consumed_backfill" - dbt run -s "solana_models,tag:units_consumed_backfill" \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 93da9ef5..77721f69 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -43,3 +43,10 @@ jobs: - name: Run DBT Jobs run: | dbt test -s "solana_models,./models" "solana_models,tag:test_daily" --exclude tag:test_weekly tag:test_hourly tag:exclude_test_daily + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_test_hourly.yml b/.github/workflows/dbt_test_hourly.yml index 52e42a81..e1f9f6a9 100644 --- a/.github/workflows/dbt_test_hourly.yml +++ b/.github/workflows/dbt_test_hourly.yml @@ -52,3 +52,10 @@ jobs: if: success() || failure() run: | dbt run -s streamline__transactions_and_votes_missing_7_days + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index 9902fe08..a1ae5c54 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -42,4 +42,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "solana_models,models/silver/_observability" \ No newline at end of file + dbt run -m "solana_models,models/silver/_observability" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_test_weekly.yml b/.github/workflows/dbt_test_weekly.yml index f8dec8ae..b58644f9 100644 --- a/.github/workflows/dbt_test_weekly.yml +++ b/.github/workflows/dbt_test_weekly.yml @@ -42,4 +42,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test -s "solana_models,tag:test_weekly" \ No newline at end of file + dbt test -s "solana_models,tag:test_weekly" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/slack_notify.yml b/.github/workflows/slack_notify.yml new file mode 100644 index 00000000..1866ba2a --- /dev/null +++ b/.github/workflows/slack_notify.yml @@ -0,0 +1,27 @@ +name: Slack Notification +on: + workflow_call: + secrets: + SLACK_WEBHOOK_URL: + required: true + +jobs: + notify: + runs-on: ubuntu-latest + environment: workflow_prod + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: pip install requests + + - name: Send Slack notification + run: python python/slack_alert.py + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/python/slack_alert.py b/python/slack_alert.py new file mode 100644 index 00000000..11ad94ad --- /dev/null +++ b/python/slack_alert.py @@ -0,0 +1,74 @@ +import requests +import os +import sys + +def create_message(): + """Creates a simple failure notification message with repo, workflow name, and URL""" + + # Get GitHub environment variables + repository = os.environ.get('GITHUB_REPOSITORY', 'Unknown repository') + repo_name = repository.split('/')[-1] if '/' in repository else repository + workflow_name = os.environ.get('GITHUB_WORKFLOW', 'Unknown workflow') + run_id = os.environ.get('GITHUB_RUN_ID', '') + server_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') + + # Build the workflow URL + workflow_url = f"{server_url}/{repository}/actions/runs/{run_id}" + + message_body = { + "text": f"Failure in {repo_name}", + "attachments": [ + { + "color": "#f44336", # Red color for failures + "fields": [ + { + "title": "Repository", + "value": repository, + "short": True + }, + { + "title": "Workflow", + "value": workflow_name, + "short": True + } + ], + "actions": [ + { + "type": "button", + "text": "View Workflow Run", + "style": "primary", + "url": workflow_url + } + ], + "footer": "GitHub Actions" + } + ] + } + + return message_body + +def send_alert(webhook_url): + """Sends a failure notification to Slack""" + + message = create_message() + + try: + response = requests.post(webhook_url, json=message) + + if response.status_code == 200: + print("Successfully sent Slack notification") + else: + print(f"Failed to send Slack notification: {response.status_code} {response.text}") + sys.exit(1) + except Exception as e: + print(f"Error sending Slack notification: {str(e)}") + sys.exit(1) + +if __name__ == '__main__': + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + + if not webhook_url: + print("ERROR: SLACK_WEBHOOK_URL environment variable is required") + sys.exit(1) + + send_alert(webhook_url) \ No newline at end of file