feat(build-tracker): Adds a build-tracking service to provide better notifications about build failures (#39355)

* notify on failed step * add healthcheck and fix docker build * slack msg fixes * improve slack message * review comments * build all the things inside the docker container * review comments Co-authored-by: Jean-Hadrien Chabran <jh@chabran.fr>
2026-02-06 15:12:02 +00:00 · 2022-08-02 17:25:39 +00:00 · 2022-08-02 17:25:39 +00:00 · ea12ab8dc0
commit ea12ab8dc0
parent 47eea306c9
11 changed files with 877 additions and 1 deletions
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@ -14,7 +14,7 @@ if [ "$BUILDKITE_BRANCH" == "main" ]; then
  # It's possible for the exit status to be unset, in the case of an earlier hook failed, so we need to
  # account for that.
  if [ -n "$BUILDKITE_COMMAND_EXIT_STATUS" ] && [ "$BUILDKITE_COMMAND_EXIT_STATUS" -eq "0" ]; then
-    # If the job exit code is either 0 or a soft failt exit code defined by that step, do nothing.
+    # If the job exit code is either 0 or a soft failed exit code defined by that step, do nothing.
    exit 0
  fi

--- a/dev/build-tracker/.gitignore
+++ b/dev/build-tracker/.gitignore
@ -0,0 +1 @@
+env
--- a/dev/build-tracker/Dockerfile
+++ b/dev/build-tracker/Dockerfile
@ -0,0 +1,17 @@
+FROM golang:1.18.1-alpine@sha256:42d35674864fbb577594b60b84ddfba1be52b4d4298c961b46ba95e9fb4712e8 AS build-tracker-build
+
+ENV GO111MODULE on
+ENV GOARCH amd64
+ENV GOOS linux
+
+COPY . /repo
+
+WORKDIR /repo/dev/build-tracker
+
+RUN go build -o /build-tracker .
+
+FROM sourcegraph/alpine-3.14:154143_2022-06-13_1eababf8817e@sha256:f1c4ac9ca1a36257c1eb699d0acf489d83dd86e067b1fc3ea4a563231a047e05 AS build-tracker
+
+RUN apk --no-cache add tzdata
+COPY --from=build-tracker-build /build-tracker /usr/local/bin/build-tracker
+ENTRYPOINT ["build-tracker"]
--- a/dev/build-tracker/README.md
+++ b/dev/build-tracker/README.md
@ -0,0 +1,26 @@
+# BUILD TRACKER
+
+Build Tracker is a server that listens for build events from Buildkite and stores them in memory and sends notifications about builds if they've failed.
+
+The server currently listens for two events:
+
+- `build.finished`
+- `job.finished`
+
+For each `job.finished` event that is received, the corresponding `build` is updated with the job that has finished. On receipt of a `build.finished` event, the server will determine if the build has failed by going through all the contained jobs of the build. If one or more jobs have indeed failed, a notification will be sent over slack.
+
+## Deployment infrastructure
+
+Build Tracker is deployed in the Buildkite kubernetes cluster of the Sourcegraph CI project on GCP. For more information on the deployment see [infrastructure](https://github.com/sourcegraph/infrastructure/tree/main/buildkite/kubernetes)
+
+## Build
+
+Execute the `build.sh` script which will build the docker container and push it to correct GCP registry. Once the image has been pushed the pod needs to be restarted so that it can pick up the new image!
+
+## Test
+
+To run the tests execute `go test .`
+
+### Notification testing
+
+To test the notifications that get sent over slack you can pass the flag `-RunIntegrationTest` as part of your test invocation i.e. `SLACK_TOKEN='my valid token' go test . -RunIntegrationTest`. In addition to the flag, you also need a valid slack token defined in your environment variables as `SLACK_TOKEN`.
--- a/dev/build-tracker/build.go
+++ b/dev/build-tracker/build.go
@ -0,0 +1,191 @@
+package main
+
+import (
+	"sync"
+
+	"github.com/buildkite/go-buildkite/v3/buildkite"
+	"github.com/sourcegraph/log"
+)
+
+// Build keeps track of a buildkite.Build and it's associated jobs and pipeline.
+// See BuildStore for where jobs are added to the build.
+type Build struct {
+	buildkite.Build
+	Pipeline *Pipeline
+	Jobs     map[string]Job
+}
+
+func (b *Build) hasFailed() bool {
+	return b.state() == "failed"
+}
+
+func (b *Build) isFinished() bool {
+	switch b.state() {
+	case "passed", "failed", "blocked", "canceled":
+		return true
+	default:
+		return false
+	}
+}
+
+func (b *Build) authorName() string {
+	if b.Author == nil {
+		return ""
+	}
+
+	return b.Author.Name
+}
+
+func (b *Build) state() string {
+	return strp(b.State)
+}
+
+func (b *Build) commit() string {
+	return strp(b.Commit)
+}
+
+func (b *Build) number() int {
+	return intp(b.Number)
+}
+
+func (b *Build) branch() string {
+	return strp(b.Branch)
+}
+
+func (b *Build) message() string {
+	return strp(b.Message)
+}
+
+type Job struct {
+	buildkite.Job
+}
+
+func (j *Job) name() string {
+	return strp(j.Name)
+}
+
+func (j *Job) exitStatus() int {
+	return intp(j.ExitStatus)
+}
+
+func (j *Job) failed() bool {
+	return !j.SoftFailed && j.exitStatus() > 0
+}
+
+// Pipeline wraps a buildkite.Pipeline and provides convenience functions to access values of the wrapped pipeline is a safe maner
+type Pipeline struct {
+	buildkite.Pipeline
+}
+
+func (p *Pipeline) name() string {
+	return strp(p.Name)
+}
+
+// Event contains information about a buildkite event. Each event contains the build, pipeline, and job. Note that when the event
+// is `build.*` then Job will be empty.
+type Event struct {
+	Name     string             `json:"event"`
+	Build    buildkite.Build    `json:"build,omitempty"`
+	Pipeline buildkite.Pipeline `json:"pipeline,omitempty"`
+	Job      buildkite.Job      `json:"job,omitempty"`
+}
+
+func (b *Event) build() *Build {
+	return &Build{
+		Build:    b.Build,
+		Pipeline: b.pipeline(),
+		Jobs:     make(map[string]Job),
+	}
+}
+
+func (b *Event) job() *Job {
+	return &Job{Job: b.Job}
+}
+
+func (b *Event) pipeline() *Pipeline {
+	return &Pipeline{Pipeline: b.Pipeline}
+}
+
+func (b *Event) isBuildFinished() bool {
+	return b.Name == "build.finished"
+}
+
+func (b *Event) jobName() string {
+	return strp(b.Job.Name)
+}
+
+func (b *Event) buildNumber() int {
+	return intp(b.Build.Number)
+}
+
+// BuildStore is a thread safe store which keeps track of Builds described by buildkite build events.
+//
+// The store is backed by a map and the build number is used as the key.
+// When a build event is added the Buildkite Build, Pipeline and Job is extracted, if available. If the Build does not exist, Buildkite is wrapped
+// in a Build and added to the map. When the event contains a Job the corresponding job is retrieved from the map and added to the Job it is for.
+type BuildStore struct {
+	logger log.Logger
+	builds map[int]*Build
+	m      sync.RWMutex
+}
+
+func NewBuildStore(logger log.Logger) *BuildStore {
+	return &BuildStore{
+		logger: logger.Scoped("store", "stores all the buildkite builds"),
+		builds: make(map[int]*Build),
+		m:      sync.RWMutex{},
+	}
+}
+
+func (s *BuildStore) Add(event *Event) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	build, ok := s.builds[event.buildNumber()]
+	if !ok {
+		build = event.build()
+		s.builds[event.buildNumber()] = build
+	}
+	// if the build is finished replace the original build with the replaced one since it will be more up to date
+	build.Build = event.Build
+	build.Pipeline = event.pipeline()
+
+	wrappedJob := event.job()
+	if wrappedJob.name() != "" {
+		build.Jobs[wrappedJob.name()] = *wrappedJob
+	}
+
+	s.logger.Debug("job added", log.Int("buildNumber", event.buildNumber()), log.Int("totalJobs", len(build.Jobs)))
+}
+
+func (s *BuildStore) GetByBuildNumber(num int) *Build {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	return s.builds[num]
+}
+
+func (s *BuildStore) DelByBuildNumber(buildNumbers ...int) {
+	s.m.Lock()
+	defer s.m.Unlock()
+
+	for _, num := range buildNumbers {
+		delete(s.builds, num)
+	}
+	s.logger.Info("deleted builds", log.Int("totalBuilds", len(buildNumbers)))
+}
+
+func (s *BuildStore) FinishedBuilds() []*Build {
+	s.m.RLock()
+	defer s.m.RUnlock()
+
+	finished := make([]*Build, 0)
+	for _, b := range s.builds {
+		if b.isFinished() {
+			s.logger.Debug("build is finished", log.Int("buildNumber", b.number()), log.String("state", b.state()))
+			finished = append(finished, b)
+		}
+	}
+
+	return finished
+}
--- a/dev/build-tracker/build.sh
+++ b/dev/build-tracker/build.sh
@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# This script builds the build-tracker docker image.
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.."
+set -eu
+
+IMAGE="us-central1-docker.pkg.dev/sourcegraph-ci/build-tracker/build-tracker"
+
+
+echo "--- docker build build-tracker $(pwd)"
+docker build -f dev/build-tracker/Dockerfile -t "$IMAGE" "$(pwd)" \
+
+#docker push $IMAGE
--- a/dev/build-tracker/main.go
+++ b/dev/build-tracker/main.go
@ -0,0 +1,210 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/sourcegraph/log"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+var ErrInvalidToken = errors.New("buildkite token is invalid")
+var ErrInvalidHeader = errors.New("Header of request is invalid")
+var ErrUnwantedEvent = errors.New("Unwanted event received")
+
+var nowFunc func() time.Time = time.Now
+
+const DefaultChannel = "#william-buildchecker-webhook-test"
+
+// Server is the http server that listens for events from Buildkite. The server tracks builds and their associated jobs
+// with the use of a BuildStore. Once a build is finished and has failed, the server sends a notification.
+type Server struct {
+	logger       log.Logger
+	store        *BuildStore
+	bkToken      string
+	notifyClient *NotificationClient
+}
+
+type config struct {
+	BuildkiteToken string
+	SlackToken     string
+	GithubToken    string
+	SlackChannel   string
+}
+
+func configFromEnv() (*config, error) {
+	var c config
+
+	err := envVar("BUILDKITE_WEBHOOK_TOKEN", &c.BuildkiteToken)
+	if err != nil {
+		return nil, err
+	}
+	err = envVar("SLACK_TOKEN", &c.SlackToken)
+	if err != nil {
+		return nil, err
+	}
+	err = envVar("GITHUB_TOKEN", &c.GithubToken)
+	if err != nil {
+		return nil, err
+	}
+
+	err = envVar("SLACK_CHANNEL", &c.SlackChannel)
+	if err != nil {
+		c.SlackChannel = DefaultChannel
+	}
+
+	return &c, nil
+}
+
+// NewServer creatse a new server to listen for Buildkite webhook events.
+func NewServer(logger log.Logger, c config) *Server {
+	logger = logger.Scoped("server", "Server which tracks events received from Buildkite and sends notifications on failures")
+	return &Server{
+		logger:       logger,
+		store:        NewBuildStore(logger),
+		bkToken:      c.BuildkiteToken,
+		notifyClient: NewNotificationClient(logger, c.SlackToken, c.GithubToken, c.SlackChannel),
+	}
+}
+
+// handleEvent handles an event received from the http listener. A event is valid when:
+// - Has the correct headers from Buildkite
+// - On of the following events
+//   * job.finished
+//   * build.finished
+// - Has valid JSON
+// Note that if we received an unwanted event ie. the event is not "job.finished" or "build.finished" we respond with a 200 OK regardless.
+// Once all the conditions are met, the event is processed in a go routine with `processEvent`
+func (s *Server) handleEvent(w http.ResponseWriter, req *http.Request) {
+	h, ok := req.Header["X-Buildkite-Token"]
+	if !ok || len(h) == 0 {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	} else if h[0] != s.bkToken {
+		w.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+
+	h, ok = req.Header["X-Buildkite-Event"]
+	if !ok || len(h) == 0 {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	eventName := h[0]
+	s.logger.Debug("received event", log.String("eventName", eventName))
+
+	data, err := ioutil.ReadAll(req.Body)
+	defer req.Body.Close()
+	if err != nil {
+		s.logger.Error("failed to read request body", log.Error(err))
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	var event Event
+	err = json.Unmarshal(data, &event)
+	if err != nil {
+		s.logger.Error("failed to unmarshall request body", log.Error(err))
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	go s.processEvent(&event)
+	w.WriteHeader(http.StatusOK)
+}
+
+func (s *Server) handleHealthz(w http.ResponseWriter, req *http.Request) {
+	// do our super exhaustive check
+	w.WriteHeader(http.StatusOK)
+}
+
+// notifyIfFailed sends a notification over slack if the provided build has failed. If the build is successful not notifcation is sent
+func (s *Server) notifyIfFailed(build *Build) error {
+	if build.hasFailed() {
+		s.logger.Info("detected failed build - sending notification", log.Int("buildNumber", intp(build.Number)))
+		return s.notifyClient.send(build)
+	}
+
+	s.logger.Info("build has not failed", log.Int("buildNumber", intp(build.Number)))
+	return nil
+}
+
+func (s *Server) startOldBuildCleaner(every, window time.Duration) func() {
+	ticker := time.NewTicker(every)
+	done := make(chan interface{})
+
+	// We could technically remove  the builds immediately after we've sent a notification for or it, or the build has passed.
+	// But we keep builds a little longer and prediodically clean them out so that we can in future allow possibly querying
+	// of builds and other use cases, like retrying a build etc.
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				oldBuilds := make([]int, 0)
+				now := nowFunc()
+				for _, b := range s.store.FinishedBuilds() {
+					finishedAt := *b.FinishedAt
+					delta := now.Sub(finishedAt.Time)
+					if delta >= window {
+						s.logger.Debug("build past age window", log.Int("buildNumber", *b.Number), log.Time("FinishedAt", finishedAt.Time), log.Duration("window", window))
+						oldBuilds = append(oldBuilds, *b.Number)
+					}
+				}
+				s.logger.Info("deleting old builds", log.Int("oldBuildCount", len(oldBuilds)))
+				s.store.DelByBuildNumber(oldBuilds...)
+			case <-done:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+
+	return func() { done <- nil }
+}
+
+// processEvent processes a BuildEvent received from Buildkite. If the event is for a `build.finished` event we get the
+// full build which includes all recorded jobs for the build and send a notification.
+// processEvent delegates the decision to actually send a notifcation
+func (s *Server) processEvent(event *Event) {
+	s.logger.Info("processing event", log.String("eventName", event.Name), log.Int("buildNumber", event.buildNumber()), log.String("jobName", event.jobName()))
+	s.store.Add(event)
+	if event.isBuildFinished() {
+		build := s.store.GetByBuildNumber(event.buildNumber())
+		if err := s.notifyIfFailed(build); err != nil {
+			s.logger.Error("failed to send notification for build", log.Int("buildNumber", event.buildNumber()), log.Error(err))
+		}
+	}
+}
+
+// Serve starts the http server and listens for buildkite build events to be sent on the route "/buildkite"
+func (s *Server) Serve() error {
+	http.HandleFunc("/buildkite", s.handleEvent)
+	http.HandleFunc("/healthz", s.handleHealthz)
+	s.logger.Info("listening on :8080")
+	return http.ListenAndServe(":8080", nil)
+}
+
+func main() {
+	sync := log.Init(log.Resource{
+		Name:      "BuildTracker",
+		Namespace: "CI",
+	})
+	defer sync.Sync()
+
+	logger := log.Scoped("BuildTracker", "main entrypoint for Build Tracking Server")
+
+	serverConf, err := configFromEnv()
+	if err != nil {
+		logger.Fatal("failed to get config from env", log.Error(err))
+	}
+	server := NewServer(logger, *serverConf)
+
+	stopFn := server.startOldBuildCleaner(5*time.Minute, 24*time.Hour)
+	defer stopFn()
+	if err := server.Serve(); err != nil {
+		logger.Fatal("server exited with error", log.Error(err))
+	}
+}
--- a/dev/build-tracker/server_test.go
+++ b/dev/build-tracker/server_test.go
@ -0,0 +1,67 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/buildkite/go-buildkite/v3/buildkite"
+	"github.com/sourcegraph/log/logtest"
+)
+
+func TestOldBuildsGetDeleted(t *testing.T) {
+	logger := logtest.Scoped(t)
+
+	finishedBuild := func(num int, state string, finishedAt time.Time) *Build {
+		b := buildkite.Build{}
+		b.State = &state
+		b.Number = &num
+		b.FinishedAt = &buildkite.Timestamp{Time: finishedAt}
+
+		return &Build{Build: b}
+	}
+
+	t.Run("All old builds get removed", func(t *testing.T) {
+		server := NewServer(logger, config{})
+		b := finishedBuild(1, "passed", time.Now().AddDate(-1, 0, 0))
+		server.store.builds[*b.Number] = b
+
+		b = finishedBuild(2, "canceled", time.Now().AddDate(0, -1, 0))
+		server.store.builds[*b.Number] = b
+
+		b = finishedBuild(3, "failed", time.Now().AddDate(0, 0, -1))
+		server.store.builds[*b.Number] = b
+		builds := server.store.FinishedBuilds()
+
+		stopFunc := server.startOldBuildCleaner(10*time.Millisecond, 24*time.Hour)
+		time.Sleep(20 * time.Millisecond)
+		stopFunc()
+
+		builds = server.store.FinishedBuilds()
+
+		if len(builds) != 0 {
+			t.Errorf("Not all old builds removed. Got %d, wanted %d", len(builds), 0)
+		}
+	})
+	t.Run("1 build left after old builds are removed", func(t *testing.T) {
+		server := NewServer(logger, config{})
+		b := finishedBuild(1, "canceled", time.Now().AddDate(-1, 0, 0))
+		server.store.builds[*b.Number] = b
+
+		b = finishedBuild(2, "passed", time.Now().AddDate(0, -1, 0))
+		server.store.builds[*b.Number] = b
+
+		b = finishedBuild(3, "failed", time.Now())
+		server.store.builds[*b.Number] = b
+
+		stopFunc := server.startOldBuildCleaner(10*time.Millisecond, 24*time.Hour)
+		time.Sleep(20 * time.Millisecond)
+		stopFunc()
+
+		builds := server.store.FinishedBuilds()
+
+		if len(builds) != 1 {
+			t.Errorf("Expected one build to be left over. Got %d, wanted %d", len(builds), 1)
+		}
+	})
+
+}
--- a/dev/build-tracker/slack.go
+++ b/dev/build-tracker/slack.go
@ -0,0 +1,242 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/google/go-github/v41/github"
+	"github.com/slack-go/slack"
+	"github.com/sourcegraph/log"
+	"github.com/sourcegraph/sourcegraph/dev/team"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+type NotificationClient struct {
+	slack   slack.Client
+	team    team.TeammateResolver
+	logger  log.Logger
+	channel string
+}
+
+func NewNotificationClient(logger log.Logger, slackToken, githubToken, channel string) *NotificationClient {
+	slack := slack.New(slackToken)
+
+	httpClient := http.Client{
+		Timeout: 5 * time.Second,
+	}
+	githubClient := github.NewClient(&httpClient)
+	teamResolver := team.NewTeammateResolver(githubClient, slack)
+
+	return &NotificationClient{
+		logger:  logger.Scoped("notificationClient", "client which interacts with Slack and Github to send notifications"),
+		slack:   *slack,
+		team:    teamResolver,
+		channel: channel,
+	}
+}
+
+func (c *NotificationClient) getTeammateForBuild(build *Build) (*team.Teammate, error) {
+	if build.Author == nil {
+		return nil, errors.New("nil Author")
+	}
+	return c.team.ResolveByCommitAuthor(context.Background(), "sourcegraph", "sourcegraph", build.commit())
+}
+
+func (c *NotificationClient) send(build *Build) error {
+	logger := c.logger.With(log.Int("buildNumber", build.number()), log.String("channel", c.channel))
+	logger.Debug("creating slack json", log.Int("buildNumber", build.number()))
+
+	teammate, err := c.getTeammateForBuild(build)
+	if err != nil {
+		logger.Error("failed to find teammate", log.Error(err))
+	}
+
+	blocks, err := createMessageBlocks(logger, teammate, build)
+	if err != nil {
+		return err
+	}
+
+	logger.Debug("sending notification")
+	_, _, err = c.slack.PostMessage(c.channel, slack.MsgOptionBlocks(blocks...))
+	if err != nil {
+		logger.Error("failed to post message", log.Error(err))
+		return err
+	}
+
+	logger.Info("notification posted")
+	return nil
+}
+
+type GrafanaQuery struct {
+	RefId string `json:"refId"`
+	Expr  string `json:"expr"`
+}
+
+type GrafanaRange struct {
+	From string `json:"from"`
+	To   string `json:"to"`
+}
+
+type GrafanaPayload struct {
+	DataSource string         `json:"datasource"`
+	Queries    []GrafanaQuery `json:"queries"`
+	Range      GrafanaRange   `json:"range"`
+}
+
+func grafanaURLFor(build *Build) (string, error) {
+	queryData := struct {
+		Build int
+	}{
+		Build: intp(build.Number),
+	}
+	tmpl := template.Must(template.New("Expression").Parse(`{app="buildkite", build="{{.Build}}", state="failed"} |~ "(?i)failed|panic|error|FAIL \\|"`))
+
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, queryData); err != nil {
+		return "", err
+	}
+	var expression = buf.String()
+
+	begin := time.Now().Add(-(2 * time.Hour)).UnixMilli()
+	end := time.Now().Add(15 * time.Minute).UnixMilli()
+
+	data := GrafanaPayload{
+		DataSource: "grafanacloud-sourcegraph-logs",
+		Queries: []GrafanaQuery{
+			{
+				RefId: "A",
+				Expr:  expression,
+			},
+		},
+		Range: GrafanaRange{
+			From: fmt.Sprintf("%d", begin),
+			To:   fmt.Sprintf("%d", end),
+		},
+	}
+
+	result, err := json.Marshal(data)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to marshall GrafanaPayload")
+	}
+
+	query := url.PathEscape(string(result))
+	// default query escapes ":", which we  don't want since it is json. Query and Path escape
+	// escape a few characters incorrectly so we fix it with this replacer.
+	// Got the idea from https://sourcegraph.com/github.com/kubernetes/kubernetes/-/blob/vendor/github.com/PuerkitoBio/urlesc/urlesc.go?L115-121
+	replacer := strings.NewReplacer(
+		"+", "%20",
+		"%28", "(",
+		"%29", ")",
+		"=", "%3D",
+		"%2C", ",",
+	)
+	query = replacer.Replace(query)
+
+	return "https://sourcegraph.grafana.net/explore?orgId=1&left=" + query, nil
+}
+
+func commitLink(msg, commit string) string {
+	repo := "http://github.com/sourcegraph/sourcegraph"
+	sgURL := fmt.Sprintf("%s/commit/%s", repo, commit)
+	return fmt.Sprintf("<%s|%s>", sgURL, msg)
+}
+
+func slackMention(teammate *team.Teammate, build *Build) string {
+	if teammate == nil {
+		authorName := build.authorName()
+		if authorName == "" {
+			authorName = "N/A"
+		}
+		return fmt.Sprintf("Teammate *%s* not found. If this is you, ensure the github field is set in your profile <https://github.com/sourcegraph/handbook/blob/main/data/team.yml|here>", authorName)
+	}
+
+	return fmt.Sprintf("<@%s>", teammate.SlackID)
+}
+
+func createMessageBlocks(logger log.Logger, teammate *team.Teammate, build *Build) ([]slack.Block, error) {
+	msg, _, _ := strings.Cut(build.message(), "\n")
+	msg += fmt.Sprintf(" (%s)", build.commit()[:7])
+	failedSection := fmt.Sprintf("%s\n\n", commitLink(msg, build.commit()))
+	failedSection += "*Failed jobs*\n\n"
+	for _, j := range build.Jobs {
+		if j.ExitStatus != nil && *j.ExitStatus != 0 && !j.SoftFailed {
+			failedSection += fmt.Sprintf("• %s", *j.Name)
+			if j.WebURL != "" {
+				failedSection += fmt.Sprintf(" - <%s|logs>", j.WebURL)
+			}
+			failedSection += "\n"
+		}
+	}
+
+	author := slackMention(teammate, build)
+	grafanaURL, err := grafanaURLFor(build)
+	if err != nil {
+		return nil, err
+	}
+
+	blocks := []slack.Block{
+		slack.NewHeaderBlock(
+			slack.NewTextBlockObject(slack.PlainTextType, fmt.Sprintf(":red_circle: Build %d failed", build.number()), true, false),
+		),
+		slack.NewSectionBlock(
+			nil,
+			[]*slack.TextBlockObject{
+				{Type: slack.MarkdownType, Text: fmt.Sprintf("*Author:* %s", author)},
+				{Type: slack.MarkdownType, Text: fmt.Sprintf("*Pipeline:* %s", build.Pipeline.name())},
+			},
+			nil,
+		),
+		&slack.DividerBlock{
+			Type: slack.MBTDivider,
+		},
+		slack.NewSectionBlock(&slack.TextBlockObject{Type: slack.MarkdownType, Text: failedSection}, nil, nil),
+		&slack.DividerBlock{
+			Type: slack.MBTDivider,
+		},
+		slack.NewSectionBlock(
+			&slack.TextBlockObject{
+				Type: slack.MarkdownType,
+				Text: `:books: *More information on flakes*
+• *<https://docs.sourcegraph.com/dev/background-information/ci#flakes|How to disable flakey tests>*
+• *<https://docs.sourcegraph.com/dev/how-to/testing#assessing-flaky-client-steps|Recognizing flakey client steps and how to fix them>*
+
+_:sourcegraph: disable flakes on sight and save your fellow teammate some time!_`,
+			},
+			nil,
+			nil,
+		),
+		&slack.DividerBlock{
+			Type: slack.MBTDivider,
+		},
+		slack.NewActionBlock(
+			"",
+			[]slack.BlockElement{
+				&slack.ButtonBlockElement{
+					Type:  slack.METButton,
+					Style: slack.StylePrimary,
+					URL:   *build.WebURL,
+					Text:  &slack.TextBlockObject{Type: slack.PlainTextType, Text: "Go to build"},
+				},
+				&slack.ButtonBlockElement{
+					Type: slack.METButton,
+					URL:  grafanaURL,
+					Text: &slack.TextBlockObject{Type: slack.PlainTextType, Text: "View logs on Grafana"},
+				},
+				&slack.ButtonBlockElement{
+					Type: slack.METButton,
+					URL:  "https://www.loom.com/share/58cedf44d44c45a292f650ddd3547337",
+					Text: &slack.TextBlockObject{Type: slack.PlainTextType, Text: "Is this a flake ?"},
+				},
+			}...,
+		),
+	}
+
+	return blocks, nil
+}
--- a/dev/build-tracker/slack_test.go
+++ b/dev/build-tracker/slack_test.go
@ -0,0 +1,75 @@
+package main
+
+import (
+	"flag"
+	"testing"
+
+	"github.com/buildkite/go-buildkite/v3/buildkite"
+	"github.com/sourcegraph/log/logtest"
+)
+
+var RunIntegrationTest *bool = flag.Bool("RunIntegrationTest", false, "Run integrations tests")
+
+func newJob(name string, exit int) *Job {
+	return &Job{buildkite.Job{
+		Name:       &name,
+		ExitStatus: &exit,
+	}}
+}
+
+func TestSlack(t *testing.T) {
+	flag.Parse()
+	if !*RunIntegrationTest {
+		t.Skip("Integration test not enabled")
+	}
+	logger := logtest.NoOp(t)
+
+	config, err := configFromEnv()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := NewNotificationClient(logger, config.SlackToken, config.GithubToken, DefaultChannel)
+
+	num := 160000
+	url := "http://www.google.com"
+	commit := "78926a5b3b836a8a104a5d5adf891e5626b1e405"
+	pipelineID := "sourcegraph"
+	exit := 999
+	msg := "this is a test"
+	err = client.send(
+		&Build{
+			Build: buildkite.Build{
+				Message: &msg,
+				WebURL:  &url,
+				Creator: &buildkite.Creator{
+					AvatarURL: "https://www.gravatar.com/avatar/7d4f6781b10e48a94d1052c443d13149",
+				},
+				Pipeline: &buildkite.Pipeline{
+					ID:   &pipelineID,
+					Name: &pipelineID,
+				},
+				Author: &buildkite.Author{
+					Name:  "William Bezuidenhout",
+					Email: "william.bezuidenhout@sourcegraph.com",
+				},
+				Number: &num,
+				URL:    &url,
+				Commit: &commit,
+			},
+			Pipeline: &Pipeline{buildkite.Pipeline{
+				Name: &pipelineID,
+			}},
+			Jobs: map[string]Job{
+				":one: fake step":   *newJob(":one: fake step", exit),
+				":two: fake step":   *newJob(":two: fake step", exit),
+				":three: fake step": *newJob(":three: fake step", exit),
+				":four: fake step":  *newJob(":four: fake step", exit),
+			},
+		},
+	)
+
+	if err != nil {
+		t.Fatalf("failed to send slack notification: %v", err)
+	}
+}
--- a/dev/build-tracker/util.go
+++ b/dev/build-tracker/util.go
@ -0,0 +1,33 @@
+package main
+
+import (
+	"os"
+
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+func strp(v *string) string {
+	if v == nil {
+		return ""
+	}
+
+	return *v
+}
+
+func intp(v *int) int {
+	if v == nil {
+		return 0
+	}
+
+	return *v
+}
+
+func envVar(name string, target *string) error {
+	value, exists := os.LookupEnv(name)
+	if !exists {
+		return errors.Newf("%s not found in environment", name)
+	}
+
+	*target = value
+	return nil
+}