From c8ee02de19c71dd89f722060f6a7d90d0c619378 Mon Sep 17 00:00:00 2001
From: Mark Story <mark@mark-story.com>
Date: Wed, 11 Jun 2025 10:57:24 -0400
Subject: [PATCH] feat: Add taskbroker + worker + scheduler (#3738)

---
 .env                               |  1 +
 .github/ISSUE_TEMPLATE/release.yml |  1 +
 docker-compose.yml                 | 22 ++++++++++++++++++++++
 scripts/bump-version.sh            |  2 +-
 4 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/.env b/.env
index 5688114..a73d4dd 100644
--- a/.env
+++ b/.env
@@ -13,6 +13,7 @@ SENTRY_IMAGE=getsentry/sentry:nightly
 SNUBA_IMAGE=getsentry/snuba:nightly
 RELAY_IMAGE=getsentry/relay:nightly
 SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly
+TASKBROKER_IMAGE=getsentry/taskbroker:nightly
 VROOM_IMAGE=getsentry/vroom:nightly
 HEALTHCHECK_INTERVAL=30s
 HEALTHCHECK_TIMEOUT=1m30s
diff --git a/.github/ISSUE_TEMPLATE/release.yml b/.github/ISSUE_TEMPLATE/release.yml
index 48691f3..545e733 100644
--- a/.github/ISSUE_TEMPLATE/release.yml
+++ b/.github/ISSUE_TEMPLATE/release.yml
@@ -15,6 +15,7 @@ body:
           - [ ] [`snuba`](https://github.com/getsentry/snuba/actions/workflows/release.yml)
           - [ ] [`symbolicator`](https://github.com/getsentry/symbolicator/actions/workflows/release.yml)
           - [ ] [`vroom`](https://github.com/getsentry/vroom/actions/workflows/release.yaml)
+          - [ ] [`taskbroker`](https://github.com/getsentry/taskbroker/actions/workflows/release.yml)
         - [ ] Release self-hosted.
           - [ ] [Prepare the `self-hosted` release](https://github.com/getsentry/self-hosted/actions/workflows/release.yml) (_replace with publish issue repo link_).
           - [ ] Check to make sure the new release branch in self-hosted includes the appropriate CalVer images.
diff --git a/docker-compose.yml b/docker-compose.yml
index ae520f7..b1805dd 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -487,6 +487,24 @@ services:
         <<: *depends_on-healthy
       web:
         <<: *depends_on-healthy
+  taskbroker:
+    <<: *restart_policy
+    image: "$TASKBROKER_IMAGE"
+    environment:
+      TASKBROKER_KAFKA_CLUSTER: "kafka:9092"
+      TASKBROKER_KAFKA_DEADLETTER_CLUSTER: "kafka:9092"
+      TASKBROKER_DB_PATH: "/opt/sqlite/taskbroker-activations.sqlite"
+    volumes:
+      - sentry-taskbroker:/opt/sqlite
+    depends_on:
+      kafka:
+        <<: *depends_on-healthy
+  taskscheduler:
+    <<: *sentry_defaults
+    command: run taskworker-scheduler
+  taskworker:
+    <<: *sentry_defaults
+    command: run taskworker --concurrency=4 --rpc-host=taskbroker:50051 --num-brokers=1
   vroom:
     <<: *restart_policy
     image: "$VROOM_IMAGE"
@@ -541,6 +559,10 @@ volumes:
   # Not being external will still persist data across restarts.
   # It won't persist if someone does a docker compose down -v.
   sentry-vroom:
+  # This volume stores task data that is inflight
+  # It should persist across restarts. If this volume is
+  # deleted, up to ~2048 tasks will be lost.
+  sentry-taskbroker:
   # These store ephemeral data that needn't persist across restarts.
   # That said, volumes will be persisted across restarts until they are deleted.
   sentry-secrets:
diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh
index 808df09..5e53ec8 100755
--- a/scripts/bump-version.sh
+++ b/scripts/bump-version.sh
@@ -4,7 +4,7 @@ set -eu
 OLD_VERSION="$1"
 NEW_VERSION="$2"
 
-sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR\|VROOM\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env
+sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR\|TASKBROKER\|VROOM\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env
 sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md
 
 echo "New version: $NEW_VERSION"