diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 4e6740d..40e9621 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -25,7 +25,7 @@ concurrency: jobs: - crun_dbt_jobs: + run_dbt_jobs: runs-on: ubuntu-latest environment: name: workflow_prod @@ -72,4 +72,11 @@ jobs: git commit -am "Auto-update docs" - name: push changes to docs run: | - git push -f --set-upstream origin docs \ No newline at end of file + git push -f --set-upstream origin docs + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index c163150..bcdbdc7 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -68,3 +68,10 @@ jobs: - name: Run DBT Jobs run: | ${{ inputs.dbt_command }} + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_incremental_core.yml b/.github/workflows/dbt_run_incremental_core.yml index d2136e6..3ef5db6 100644 --- a/.github/workflows/dbt_run_incremental_core.yml +++ b/.github/workflows/dbt_run_incremental_core.yml @@ -46,4 +46,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m models/bronze models/silver/core models/gold/core --exclude models/bronze/genesis_backfill models/silver/core/genesis_backfill \ No newline at end of file + dbt run -m models/bronze models/silver/core models/gold/core --exclude models/bronze/genesis_backfill models/silver/core/genesis_backfill + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml index f67ef45..cdb7a88 100644 --- a/.github/workflows/dbt_run_incremental_non_core.yml +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -46,4 +46,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/gov models/gold/gov 1+models/gold/stats/stats__ez_core_metrics_hourly.sql \ No newline at end of file + dbt run -m models/silver/gov models/gold/gov 1+models/gold/stats/stats__ez_core_metrics_hourly.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index bf90e7c..6e71e29 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -47,3 +47,9 @@ jobs: run: | dbt run -m models/silver/_observability/silver_observability* + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_observability_monthly.yml b/.github/workflows/dbt_run_observability_monthly.yml index 1d8b83a..648f74f 100644 --- a/.github/workflows/dbt_run_observability_monthly.yml +++ b/.github/workflows/dbt_run_observability_monthly.yml @@ -45,4 +45,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/_observability/silver_observability* --vars "OBSERV_FULL_TEST: true" \ No newline at end of file + dbt run -m models/silver/_observability/silver_observability* --vars "OBSERV_FULL_TEST: true" + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_blocks_tx_counts.yml b/.github/workflows/dbt_run_streamline_blocks_tx_counts.yml index 81914b5..0409713 100644 --- a/.github/workflows/dbt_run_streamline_blocks_tx_counts.yml +++ b/.github/workflows/dbt_run_streamline_blocks_tx_counts.yml @@ -47,3 +47,10 @@ jobs: - name: Run DBT Jobs run: | dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 2+models/streamline/silver/realtime/streamline__blocks_realtime.sql 1+models/streamline/silver/realtime/streamline__tx_counts_realtime.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_transactions.yml b/.github/workflows/dbt_run_streamline_transactions.yml index d9b73fd..7c96189 100644 --- a/.github/workflows/dbt_run_streamline_transactions.yml +++ b/.github/workflows/dbt_run_streamline_transactions.yml @@ -46,4 +46,11 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/silver/realtime/streamline__transactions_realtime.sql \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/silver/realtime/streamline__transactions_realtime.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_udf_blockchain.yml b/.github/workflows/dbt_run_udf_blockchain.yml index 6cd5e58..4b94145 100644 --- a/.github/workflows/dbt_run_udf_blockchain.yml +++ b/.github/workflows/dbt_run_udf_blockchain.yml @@ -60,4 +60,11 @@ jobs: dbt run-operation get_blockchain_api dbt run -m models/silver/core/silver__blockchain.sql dbt run-operation get_blockchain_api - dbt run -m models/silver/core/silver__blockchain.sql \ No newline at end of file + dbt run -m models/silver/core/silver__blockchain.sql + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index eee4fe3..dd05fff 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -46,3 +46,10 @@ jobs: - name: Run DBT Jobs run: | dbt test -m models/bronze models/silver models/gold --exclude models/bronze/genesis_backfill models/silver/core/genesis_backfill + + notify-failure: + needs: [run_dbt_jobs] + if: failure() + uses: ./.github/workflows/slack_notify.yml + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/slack_notify.yml b/.github/workflows/slack_notify.yml new file mode 100644 index 0000000..1866ba2 --- /dev/null +++ b/.github/workflows/slack_notify.yml @@ -0,0 +1,27 @@ +name: Slack Notification +on: + workflow_call: + secrets: + SLACK_WEBHOOK_URL: + required: true + +jobs: + notify: + runs-on: ubuntu-latest + environment: workflow_prod + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install dependencies + run: pip install requests + + - name: Send Slack notification + run: python python/slack_alert.py + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/models/streamline/silver/realtime/streamline__transactions_single_realtime.sql b/models/streamline/silver/realtime/streamline__transactions_single_realtime.sql new file mode 100644 index 0000000..83584ec --- /dev/null +++ b/models/streamline/silver/realtime/streamline__transactions_single_realtime.sql @@ -0,0 +1,101 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'streamline.udf_bulk_rest_api_v2', + target = "{{this.schema}}.{{this.identifier}}", + params ={ "external_table" :"transactions_v2", + "sql_limit" :"10", + "producer_batch_size" :"5", + "worker_batch_size" :"1", + "exploded_key": "[\"result.txs\"]", + "sql_source" :"{{this.identifier}}" } + ) +) }} +-- depends_on: {{ ref('streamline__complete_transactions') }} +-- depends_on: {{ ref('streamline__complete_tx_counts') }} +WITH blocks AS ( + + SELECT + A.block_number, + tx_count + FROM + {{ ref("streamline__complete_tx_counts") }} A + WHERE + block_number IN ( + 20745901, + 20746801, + 20748924, + 20759934, + 20778140, + 20784319, + 20818327, + 20826784, + 20843581, + 20857395, + 20861200, + 20864260 + ) +), +numbers AS ( + SELECT + _id AS page_number + FROM + {{ source( + 'crosschain_silver', + 'number_sequence' + ) }} + WHERE + _id < 2000 +), +blocks_with_page_numbers AS ( + SELECT + b.block_number :: INT AS block_number, + n.page_number + FROM + numbers n + JOIN blocks b + ON n.page_number <= b.tx_count + EXCEPT + SELECT + block_number, + page_number + FROM + {{ ref("streamline__complete_transactions") }} +) +SELECT + ROUND( + block_number, + -3 + ) :: INT AS partition_key, + live.udf_api( + 'POST', + '{Service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'id', + block_number, + 'jsonrpc', + '2.0', + 'method', + 'tx_search', + 'params', + ARRAY_CONSTRUCT( + 'tx.height=' || block_number :: STRING, + TRUE, + page_number :: STRING, + '1', + 'asc' + ) + ), + 'vault/prod/cosmos/quicknode/mainnet' + ) AS request, + page_number, + block_number AS block_number_requested +FROM + blocks_with_page_numbers +ORDER BY + block_number, + page_number diff --git a/python/slack_alert.py b/python/slack_alert.py new file mode 100644 index 0000000..4e624f7 --- /dev/null +++ b/python/slack_alert.py @@ -0,0 +1,74 @@ +import requests +import os +import sys + +def create_message(): + """Creates a simple failure notification message with repo, workflow name, and URL""" + + # Get GitHub environment variables + repository = os.environ.get('GITHUB_REPOSITORY', 'Unknown repository') + repo_name = repository.split('/')[-1] if '/' in repository else repository + workflow_name = os.environ.get('GITHUB_WORKFLOW', 'Unknown workflow') + run_id = os.environ.get('GITHUB_RUN_ID', '') + server_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com') + + # Build the workflow URL + workflow_url = f"{server_url}/{repository}/actions/runs/{run_id}" + + message_body = { + "text": f"Failure in {repo_name}", + "attachments": [ + { + "color": "#f44336", # Red color for failures + "fields": [ + { + "title": "Repository", + "value": repository, + "short": True + }, + { + "title": "Workflow", + "value": workflow_name, + "short": True + } + ], + "actions": [ + { + "type": "button", + "text": "View Workflow Run", + "style": "primary", + "url": workflow_url + } + ], + "footer": "GitHub Actions" + } + ] + } + + return message_body + +def send_alert(webhook_url): + """Sends a failure notification to Slack""" + + message = create_message() + + try: + response = requests.post(webhook_url, json=message) + + if response.status_code == 200: + print("Successfully sent Slack notification") + else: + print(f"Failed to send Slack notification: {response.status_code} {response.text}") + sys.exit(1) + except Exception as e: + print(f"Error sending Slack notification: {str(e)}") + sys.exit(1) + +if __name__ == '__main__': + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + + if not webhook_url: + print("ERROR: SLACK_WEBHOOK_URL environment variable is required") + sys.exit(1) + + send_alert(webhook_url) \ No newline at end of file