buildchecker: pipeline failure detection and branch locking (#28759)

2026-02-06 17:31:43 +00:00 · 2021-12-16 14:40:31 -08:00 · 2021-12-16 14:40:31 -08:00 · 167ad5b63b
commit 167ad5b63b
parent fec571f07e
16 changed files with 3080 additions and 1 deletions
--- a/.github/workflows/buildchecker.yml
+++ b/.github/workflows/buildchecker.yml
@ -0,0 +1,26 @@
+# See dev/buildchecker/README.md
+name: buildchecker
+on:
+  schedule:
+  - cron: '*/15 * * * *'
+  workflow_dispatch:
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+
+    # secrets for this workflow are configured in the 'autobuildsherrif' environment.
+    # 'autobuildsherrif' was the original name of the 'buildchecker' tool - GitHub does
+    # not provide a simple way to do a rename, so we leave it as is for now.
+    environment: autobuildsherrif
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with: { go-version: '1.17' }
+
+      - run: ./dev/buildchecker/run.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.AUTOBUILDSHERRIF_GITHUB_TOKEN }}
+          BUILDKITE_TOKEN: ${{ secrets.AUTOBUILDSHERRIF_BUILDKITE_TOKEN }}
+          SLACK_WEBHOOK: ${{ secrets.AUTOBUILDSHERRIF_SLACK_WEBHOOK }}
--- a/dev/bkstats/README.md
+++ b/dev/bkstats/README.md
@ -1,6 +1,7 @@
 # Bkstats

 A crude script to compute statistics from our Buildkite pipelines.
+Owned by the [DevX team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/enablement/dev-experience).

 ## Usage

--- a/dev/buildchecker/README.md
+++ b/dev/buildchecker/README.md
@ -0,0 +1,18 @@
+# buildchecker
+
+`buildchecker` is designed to respond to periods of consecutive build failures on a Buildkite pipeline.
+Owned by the [DevX team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/enablement/dev-experience).
+
+## Usage
+
+```sh
+go run ./dev/buildchecker/ # directly
+./dev/buildchecker/run.sh  # using wrapper script
+```
+
+Also see the [`buildchecker` GitHub Action workflow](../../.github/workflows/buildchecker.yml) where this program is run on an automated basis.
+
+## Development
+
+- `branch_test.go` contains integration tests against the GitHub API. Normally runs against recordings in `testdata` - to update `testdata`, run the tests with the `-update` flag.
+- All other tests are strictly unit tests.
--- a/dev/buildchecker/branch.go
+++ b/dev/buildchecker/branch.go
@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/google/go-github/v41/github"
+)
+
+type BranchLocker interface {
+	// Unlock returns a callback to execute the unlock if one is needed, otherwise returns nil.
+	Unlock(ctx context.Context) (unlock func() error, err error)
+	// Lock returns a callback to execute the lock if one is needed, otherwise returns nil.
+	Lock(ctx context.Context, commits []CommitInfo, fallbackTeam string) (lock func() error, err error)
+}
+
+type repoBranchLocker struct {
+	ghc    *github.Client
+	owner  string
+	repo   string
+	branch string
+}
+
+func NewBranchLocker(ghc *github.Client, owner, repo, branch string) BranchLocker {
+	return &repoBranchLocker{
+		ghc:    ghc,
+		owner:  owner,
+		repo:   repo,
+		branch: branch,
+	}
+}
+
+func (b *repoBranchLocker) Lock(ctx context.Context, commits []CommitInfo, fallbackTeam string) (func() error, error) {
+	protects, _, err := b.ghc.Repositories.GetBranchProtection(ctx, b.owner, b.repo, b.branch)
+	if err != nil {
+		return nil, fmt.Errorf("getBranchProtection: %+w", err)
+	}
+	if protects.Restrictions != nil {
+		// restrictions already in place, do not overwrite
+		return nil, nil
+	}
+
+	// Get the commit authors to determine who to exclude from branch lock
+	var failureAuthors []*github.User
+	for _, c := range commits {
+		commit, _, err := b.ghc.Repositories.GetCommit(ctx, b.owner, b.repo, c.Commit, &github.ListOptions{})
+		if err != nil {
+			return nil, err
+		}
+		failureAuthors = append(failureAuthors, commit.Author)
+	}
+
+	// Get authors that are in Sourcegraph org
+	allowAuthors := []string{}
+	for _, u := range failureAuthors {
+		membership, _, err := b.ghc.Organizations.GetOrgMembership(ctx, *u.Login, b.owner)
+		if err != nil {
+			return nil, fmt.Errorf("getOrgMembership: %+w", err)
+		}
+		if membership == nil || *membership.State != "active" {
+			continue // we don't want this user
+		}
+
+		allowAuthors = append(allowAuthors, *u.Login)
+	}
+
+	return func() error {
+		if _, _, err := b.ghc.Repositories.UpdateBranchProtection(ctx, b.owner, b.repo, b.branch, &github.ProtectionRequest{
+			// Restrict push access
+			Restrictions: &github.BranchRestrictionsRequest{
+				Users: allowAuthors,
+				Teams: []string{fallbackTeam},
+			},
+			// This is a replace operation, so we must set all the desired rules here as well
+			RequiredStatusChecks: protects.GetRequiredStatusChecks(),
+			RequireLinearHistory: github.Bool(true),
+			RequiredPullRequestReviews: &github.PullRequestReviewsEnforcementRequest{
+				RequiredApprovingReviewCount: 1,
+			},
+		}); err != nil {
+			return fmt.Errorf("unlock: %w", err)
+		}
+		return nil
+	}, nil
+}
+
+func (b *repoBranchLocker) Unlock(ctx context.Context) (func() error, error) {
+	protects, _, err := b.ghc.Repositories.GetBranchProtection(ctx, b.owner, b.repo, b.branch)
+	if err != nil {
+		return nil, fmt.Errorf("getBranchProtection: %+w", err)
+	}
+	if protects.Restrictions == nil {
+		// no restrictions in place, we are done
+		return nil, nil
+	}
+
+	req, err := b.ghc.NewRequest(http.MethodDelete,
+		fmt.Sprintf("/repos/%s/%s/branches/%s/protection/restrictions",
+			b.owner, b.repo, b.branch),
+		nil)
+	if err != nil {
+		return nil, fmt.Errorf("deleteRestrictions: %+w", err)
+	}
+
+	return func() error {
+		if _, err := b.ghc.Do(ctx, req, nil); err != nil {
+			return fmt.Errorf("unlock: %+w", err)
+		}
+		return nil
+	}, nil
+}
--- a/dev/buildchecker/branch_test.go
+++ b/dev/buildchecker/branch_test.go
@ -0,0 +1,146 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/dnaeon/go-vcr/cassette"
+	"github.com/google/go-github/v41/github"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/oauth2"
+
+	"github.com/sourcegraph/sourcegraph/internal/httptestutil"
+)
+
+var updateRecordings = flag.Bool("update", false, "update integration test")
+
+func newTestGitHubClient(ctx context.Context, t *testing.T) (ghc *github.Client, stop func() error) {
+	recording := filepath.Join("testdata", strings.ReplaceAll(t.Name(), " ", "-"))
+	recorder, err := httptestutil.NewRecorder(recording, *updateRecordings, func(i *cassette.Interaction) error {
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if *updateRecordings {
+		httpClient := oauth2.NewClient(ctx, oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")},
+		))
+		recorder.SetTransport(httpClient.Transport)
+	}
+	return github.NewClient(&http.Client{Transport: recorder}), recorder.Stop
+}
+
+func TestRepoBranchLocker(t *testing.T) {
+	ctx := context.Background()
+
+	const testBranch = "test-buildsherrif-branch"
+
+	validateDefaultProtections := func(t *testing.T, protects *github.Protection) {
+		// Require a pull request before merging
+		assert.NotNil(t, protects.RequiredPullRequestReviews)
+		assert.Equal(t, 1, protects.RequiredPullRequestReviews.RequiredApprovingReviewCount)
+		// Require status checks to pass before merging
+		assert.NotNil(t, protects.RequiredStatusChecks)
+		assert.Contains(t, protects.RequiredStatusChecks.Contexts, "buildkite/sourcegraph")
+		assert.False(t, protects.RequiredStatusChecks.Strict)
+		// Require linear history
+		assert.NotNil(t, protects.RequireLinearHistory)
+		assert.True(t, protects.RequireLinearHistory.Enabled)
+	}
+
+	t.Run("lock", func(t *testing.T) {
+		ghc, stop := newTestGitHubClient(ctx, t)
+		defer stop()
+		locker := NewBranchLocker(ghc, "sourcegraph", "sourcegraph", testBranch)
+
+		commits := []CommitInfo{
+			{Commit: "be7f0f51b73b1966254db4aac65b656daa36e2fb"}, // @davejrt
+			{Commit: "fac6d4973acad43fcd2f7579a3b496cd92619172"}, // @bobheadxi
+			{Commit: "06a8636c2e0bea69944d8419aafa03ff3992527a"}, // @bobheadxi
+			{Commit: "93971fa0b036b3e258cbb9a3eb7098e4032eefc4"}, // @jhchabran
+		}
+		lock, err := locker.Lock(ctx, commits, "dev-experience")
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.NotNil(t, lock, "has callback")
+
+		err = lock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Validate live state
+		validateLiveState := func() {
+			protects, _, err := ghc.Repositories.GetBranchProtection(ctx, "sourcegraph", "sourcegraph", testBranch)
+			if err != nil {
+				t.Fatal(err)
+			}
+			validateDefaultProtections(t, protects)
+
+			assert.NotNil(t, protects.Restrictions, "want push access restricted and granted")
+			users := []string{}
+			for _, u := range protects.Restrictions.Users {
+				users = append(users, *u.Login)
+			}
+			sort.Strings(users)
+			assert.Equal(t, []string{"bobheadxi", "davejrt", "jhchabran"}, users)
+
+			teams := []string{}
+			for _, t := range protects.Restrictions.Teams {
+				teams = append(teams, *t.Slug)
+			}
+			assert.Equal(t, []string{"dev-experience"}, teams)
+		}
+		validateLiveState()
+
+		// Repeated lock attempt shouldn't change anything
+		lock, err = locker.Lock(ctx, []CommitInfo{}, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Nil(t, lock, "should not have callback")
+
+		// should have same state as before
+		validateLiveState()
+	})
+
+	t.Run("unlock", func(t *testing.T) {
+		ghc, stop := newTestGitHubClient(ctx, t)
+		defer stop()
+		locker := NewBranchLocker(ghc, "sourcegraph", "sourcegraph", testBranch)
+
+		unlock, err := locker.Unlock(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.NotNil(t, unlock, "has callback")
+
+		err = unlock()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Validate live state
+		protects, _, err := ghc.Repositories.GetBranchProtection(ctx, "sourcegraph", "sourcegraph", testBranch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		validateDefaultProtections(t, protects)
+		assert.Nil(t, protects.Restrictions)
+
+		// Repeat unlock
+		unlock, err = locker.Unlock(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Nil(t, unlock, "should not have callback")
+	})
+}
--- a/dev/buildchecker/checker.go
+++ b/dev/buildchecker/checker.go
@ -0,0 +1,149 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/buildkite/go-buildkite/v3/buildkite"
+)
+
+type CheckOptions struct {
+	FailuresThreshold int
+	BuildTimeout      time.Duration
+}
+
+type CommitInfo struct {
+	Commit string
+	Author string
+}
+
+type CheckResults struct {
+	// LockBranch indicates whether or not the Action will lock the branch.
+	LockBranch bool
+	// Action is a callback to actually execute changes.
+	Action func() (err error)
+	// FailedCommits lists the commits with failed builds that were detected.
+	FailedCommits []CommitInfo
+}
+
+// CheckBuilds is the main buildchecker program. It checks the given builds for relevant
+// failures and runs lock/unlock operations on the given branch.
+func CheckBuilds(ctx context.Context, branch BranchLocker, builds []buildkite.Build, opts CheckOptions) (results *CheckResults, err error) {
+	results = &CheckResults{}
+
+	// Scan for first build with a meaningful state
+	var firstFailedBuildIndex int
+	for i, b := range builds {
+		if isBuildPassed(b) {
+			fmt.Printf("most recent finished build %d passed\n", *b.Number)
+			results.Action, err = branch.Unlock(ctx)
+			if err != nil {
+				return nil, fmt.Errorf("unlockBranch: %w", err)
+			}
+		}
+		if isBuildFailed(b, opts.BuildTimeout) {
+			fmt.Printf("most recent finished build %d failed\n", *b.Number)
+			firstFailedBuildIndex = i
+			break
+		}
+
+		// Otherwise, keep looking for a completed (failed or passed) build
+	}
+
+	// if failed, check if failures are consecutive
+	var exceeded bool
+	results.FailedCommits, exceeded = checkConsecutiveFailures(
+		builds[max(firstFailedBuildIndex-1, 0):], // Check builds starting with the one we found
+		opts.FailuresThreshold,
+		opts.BuildTimeout)
+	if !exceeded {
+		fmt.Println("threshold not exceeded")
+		results.Action, err = branch.Unlock(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("unlockBranch: %w", err)
+		}
+		return
+	}
+
+	fmt.Println("threshold exceeded, this is a big deal!")
+	results.LockBranch = true
+	results.Action, err = branch.Lock(ctx, results.FailedCommits, "dev-experience")
+	if err != nil {
+		return nil, fmt.Errorf("lockBranch: %w", err)
+	}
+	return
+}
+
+func isBuildPassed(build buildkite.Build) bool {
+	return build.State != nil && *build.State == "passed"
+}
+
+func isBuildFailed(build buildkite.Build, timeout time.Duration) bool {
+	// Has state and is failed
+	if build.State != nil && (*build.State == "failed" || *build.State == "cancelled") {
+		return true
+	}
+	// Created, but not done
+	if build.CreatedAt != nil && build.FinishedAt == nil {
+		// Failed if exceeded timeout
+		return time.Now().After(build.CreatedAt.Add(timeout))
+	}
+	return false
+}
+
+func buildSummary(build buildkite.Build) string {
+	summary := []string{*build.Commit}
+	if build.State != nil {
+		summary = append(summary, *build.State)
+	}
+	if build.CreatedAt != nil {
+		summary = append(summary, "started: "+build.CreatedAt.String())
+	}
+	if build.FinishedAt != nil {
+		summary = append(summary, "finished: "+build.FinishedAt.String())
+	}
+	return strings.Join(summary, ", ")
+}
+
+func checkConsecutiveFailures(builds []buildkite.Build, threshold int, timeout time.Duration) (failedCommits []CommitInfo, thresholdExceeded bool) {
+	failedCommits = []CommitInfo{}
+
+	var consecutiveFailures int
+	for _, b := range builds {
+		if !isBuildFailed(b, timeout) {
+			fmt.Printf("build %d not failed: %+v\n", *b.Number, buildSummary(b))
+
+			if !isBuildPassed(b) {
+				// we're only safe if non-failures are actually passed, otherwise
+				// keep looking
+				continue
+			}
+			return
+		}
+
+		consecutiveFailures += 1
+		var author string
+		if b.Author != nil {
+			author = fmt.Sprintf("%s (%s)", b.Author.Name, b.Author.Email)
+		}
+		failedCommits = append(failedCommits, CommitInfo{
+			Commit: *b.Commit,
+			Author: author,
+		})
+		fmt.Printf("build %d is a failure: count %d, %s\n", *b.Number, consecutiveFailures, buildSummary(b))
+		if consecutiveFailures >= threshold {
+			return failedCommits, true
+		}
+	}
+
+	return
+}
+
+func max(x, y int) int {
+	if x < y {
+		return y
+	}
+	return x
+}
--- a/dev/buildchecker/checker_test.go
+++ b/dev/buildchecker/checker_test.go
@ -0,0 +1,239 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/buildkite/go-buildkite/v3/buildkite"
+	"github.com/stretchr/testify/assert"
+)
+
+type mockBranchLocker struct{}
+
+func (m *mockBranchLocker) Unlock(context.Context) (func() error, error) {
+	return func() error { return nil }, nil
+}
+func (m *mockBranchLocker) Lock(context.Context, []CommitInfo, string) (func() error, error) {
+	return func() error { return nil }, nil
+}
+
+func TestCheckBuilds(t *testing.T) {
+	// Simple end-to-end tests of the buildchecker entrypoint with mostly fixed parameters
+	ctx := context.Background()
+	var lock BranchLocker = &mockBranchLocker{}
+	testOptions := CheckOptions{
+		FailuresThreshold: 2,
+		BuildTimeout:      time.Hour,
+	}
+
+	// Triggers a pass
+	passBuild := buildkite.Build{
+		Number: buildkite.Int(1),
+		Commit: buildkite.String("a"),
+		State:  buildkite.String("passed"),
+	}
+	// Triggers a fail
+	failSet := []buildkite.Build{{
+		Number: buildkite.Int(1),
+		Commit: buildkite.String("a"),
+		State:  buildkite.String("failed"),
+	}, {
+		Number: buildkite.Int(2),
+		Commit: buildkite.String("b"),
+		State:  buildkite.String("failed"),
+	}}
+	runningBuild := buildkite.Build{
+		Number:    buildkite.Int(1),
+		Commit:    buildkite.String("a"),
+		State:     buildkite.String("running"),
+		StartedAt: buildkite.NewTimestamp(time.Now()),
+	}
+
+	tests := []struct {
+		name       string
+		builds     []buildkite.Build
+		wantLocked bool
+	}{{
+		name:       "passed, should not lock",
+		builds:     []buildkite.Build{passBuild},
+		wantLocked: false,
+	}, {
+		name:       "not enough failed, should not lock",
+		builds:     []buildkite.Build{failSet[0]},
+		wantLocked: false,
+	}, {
+		name:       "should lock",
+		builds:     failSet,
+		wantLocked: true,
+	}, {
+		name:       "should skip leading running builds to pass",
+		builds:     []buildkite.Build{runningBuild, passBuild},
+		wantLocked: false,
+	}, {
+		name:       "should skip leading running builds to lock",
+		builds:     append([]buildkite.Build{runningBuild}, failSet...),
+		wantLocked: true,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			res, err := CheckBuilds(ctx, lock, tt.builds, testOptions)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.wantLocked, res.LockBranch)
+			// Mock always returns an action, check it's always assigned correctly
+			assert.NotNil(t, res.Action)
+		})
+	}
+}
+
+func TestCheckConsecutiveFailures(t *testing.T) {
+	type args struct {
+		builds    []buildkite.Build
+		threshold int
+		timeout   time.Duration
+	}
+	tests := []struct {
+		name                  string
+		args                  args
+		wantCommits           []string
+		wantThresholdExceeded bool
+	}{{
+		name: "not exceeded: passed",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("passed"),
+			}},
+			threshold: 3, timeout: time.Hour,
+		},
+		wantCommits:           []string{},
+		wantThresholdExceeded: false,
+	}, {
+		name: "not exceeded: failed",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("failed"),
+			}},
+			threshold: 3, timeout: time.Hour,
+		},
+		wantCommits:           []string{"a"},
+		wantThresholdExceeded: false,
+	}, {
+		name: "not exceeded: failed, passed",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("failed"),
+			}, {
+				Number: buildkite.Int(2),
+				Commit: buildkite.String("b"),
+				State:  buildkite.String("passed"),
+			}},
+			threshold: 3, timeout: time.Hour,
+		},
+		wantCommits:           []string{"a"},
+		wantThresholdExceeded: false,
+	}, {
+		name: "not exceeded: failed, passed, failed",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("failed"),
+			}, {
+				Number: buildkite.Int(2),
+				Commit: buildkite.String("b"),
+				State:  buildkite.String("passed"),
+			}, {
+				Number: buildkite.Int(3),
+				Commit: buildkite.String("c"),
+				State:  buildkite.String("failed"),
+			}},
+			threshold: 2, timeout: time.Hour,
+		},
+		wantCommits:           []string{"a"},
+		wantThresholdExceeded: false,
+	}, {
+		name: "exceeded: failed == threshold",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("failed"),
+			}},
+			threshold: 1, timeout: time.Hour,
+		},
+		wantCommits:           []string{"a"},
+		wantThresholdExceeded: true,
+	}, {
+		name: "exceeded: failed == threshold",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("failed"),
+			}},
+			threshold: 1, timeout: time.Hour,
+		},
+		wantCommits:           []string{"a"},
+		wantThresholdExceeded: true,
+	}, {
+		name: "exceeded: failed, timeout, failed",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("failed"),
+			}, {
+				Number:    buildkite.Int(2),
+				Commit:    buildkite.String("b"),
+				State:     buildkite.String("running"),
+				CreatedAt: buildkite.NewTimestamp(time.Now().Add(-2 * time.Hour)),
+			}, {
+				Number: buildkite.Int(3),
+				Commit: buildkite.String("c"),
+				State:  buildkite.String("failed"),
+			}},
+			threshold: 3, timeout: time.Hour,
+		},
+		wantCommits:           []string{"a", "b", "c"},
+		wantThresholdExceeded: true,
+	}, {
+		name: "exceeded: failed, running, failed",
+		args: args{
+			builds: []buildkite.Build{{
+				Number: buildkite.Int(1),
+				Commit: buildkite.String("a"),
+				State:  buildkite.String("failed"),
+			}, {
+				Number:    buildkite.Int(2),
+				Commit:    buildkite.String("b"),
+				State:     buildkite.String("running"),
+				CreatedAt: buildkite.NewTimestamp(time.Now()),
+			}, {
+				Number: buildkite.Int(3),
+				Commit: buildkite.String("c"),
+				State:  buildkite.String("failed"),
+			}},
+			threshold: 2, timeout: time.Hour,
+		},
+		wantCommits:           []string{"a", "c"},
+		wantThresholdExceeded: true,
+	}}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotCommits, gotThresholdExceeded := checkConsecutiveFailures(tt.args.builds, tt.args.threshold, tt.args.timeout)
+			assert.Equal(t, tt.wantThresholdExceeded, gotThresholdExceeded, "thresholdExceeded")
+
+			wantCommits := []CommitInfo{}
+			for _, c := range tt.wantCommits {
+				wantCommits = append(wantCommits, CommitInfo{Commit: c})
+			}
+			assert.Equal(t, wantCommits, gotCommits, "commits")
+		})
+	}
+}
--- a/dev/buildchecker/main.go
+++ b/dev/buildchecker/main.go
@ -0,0 +1,96 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/buildkite/go-buildkite/v3/buildkite"
+	"github.com/google/go-github/v41/github"
+	"golang.org/x/oauth2"
+)
+
+func main() {
+	var (
+		ctx            = context.Background()
+		buildkiteToken string
+		githubToken    string
+		slackWebhook   string
+		pipeline       string
+		branch         string
+		threshold      int
+		timeoutMins    int
+	)
+
+	flag.StringVar(&buildkiteToken, "buildkite.token", "", "mandatory buildkite token")
+	flag.StringVar(&githubToken, "github.token", "", "mandatory github token")
+	flag.StringVar(&slackWebhook, "slack.webhook", "", "Slack Webhook URL to post the results on")
+	flag.StringVar(&pipeline, "pipeline", "sourcegraph", "name of the pipeline to inspect")
+	flag.StringVar(&branch, "branch", "main", "name of the branch to inspect")
+	flag.IntVar(&threshold, "failures.threshold", 3, "failures required to trigger an incident")
+	flag.IntVar(&timeoutMins, "failures.timeout", 40, "duration of a run required to be considered a failure (minutes)")
+
+	config, err := buildkite.NewTokenConfig(buildkiteToken, false)
+	if err != nil {
+		panic(err)
+	}
+	// Buildkite client
+	bkc := buildkite.NewClient(config.Client())
+
+	// GitHub client
+	ghc := github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: githubToken},
+	)))
+
+	// Newest is returned first https://buildkite.com/docs/apis/rest-api/builds#list-builds-for-a-pipeline
+	builds, _, err := bkc.Builds.ListByPipeline("sourcegraph", pipeline, &buildkite.BuildsListOptions{
+		Branch: branch,
+		// Fix to high page size just in case, default is 30
+		// https://buildkite.com/docs/apis/rest-api#pagination
+		ListOptions: buildkite.ListOptions{PerPage: 99},
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	opts := CheckOptions{
+		FailuresThreshold: threshold,
+		BuildTimeout:      time.Duration(timeoutMins) * time.Minute,
+	}
+	fmt.Printf("running buildchecker over %d builds with option: %+v\n", len(builds), opts)
+	results, err := CheckBuilds(
+		ctx,
+		NewBranchLocker(ghc, "sourcegraph", "sourcegraph", branch),
+		builds,
+		opts,
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Only post an update if the lock has been modified
+	lockModified := results.Action != nil
+	if lockModified {
+		// Post update first to avoid invisible changes
+		if err := postSlackUpdate(slackWebhook, slackSummary(lockModified, results.FailedCommits)); err != nil {
+			// If action is an unlock, try to unlock anyway
+			if !results.LockBranch {
+				log.Println("slack update failed but action is an unlock, trying to unlock branch anyway")
+				goto POST
+			}
+			log.Fatal(err)
+		}
+
+	POST:
+		// If post works, do the thing
+		if err := results.Action(); err != nil {
+			slackErr := postSlackUpdate(slackWebhook, fmt.Sprintf("Failed to execute action (%+v): %s", results, err))
+			if slackErr != nil {
+				log.Println(slackErr)
+			}
+			log.Fatal(err)
+		}
+	}
+}
--- a/dev/buildchecker/run.sh
+++ b/dev/buildchecker/run.sh
@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+# Make this script independent of where it's called
+cd "$(dirname "${BASH_SOURCE[0]}")"/../..
+
+set -eu
+
+pushd dev/buildsheriff
+
+echo "--- Running buildsheriff"
+go run main.go \
+  -buildkite.token="$BUILDKITE_TOKEN" \
+  -github.token="$GITHUB_TOKEN" \
+  -slack.webhook="$SLACK_WEBHOOK"
+
+popd
--- a/dev/buildchecker/slack.go
+++ b/dev/buildchecker/slack.go
@ -0,0 +1,80 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+func slackSummary(locked bool, failedCommits []CommitInfo) string {
+	if !locked {
+		return ":white_check_mark: Pipeline healthy - branch unlocked!"
+	}
+	message := `:alert: *Consecutive build failures detected - branch has been locked.* :alert:
+The authors of the following failed commits have been granted merge access to investigate and resolve the issue:
+`
+	for _, commit := range failedCommits {
+		message += fmt.Sprintf("\n- <https://github.com/sourcegraph/sourcegraph/commit/%s|%s> - %s",
+			commit.Commit, commit.Commit, commit.Author)
+	}
+	message += `The branch will automatically be unlocked once a green build is run.
+Refer to the <https://handbook.sourcegraph.com/departments/product-engineering/engineering/process/incidents/playbooks/ci|CI incident playbook> for help.
+If unable to resolve the issue, please start an incident with the '/incident' Slack command.
+
+cc: @dev-experience-support`
+	return message
+}
+
+func postSlackUpdate(webhook string, summary string) error {
+	type slackText struct {
+		Type string `json:"type"`
+		Text string `json:"text"`
+	}
+
+	type slackBlock struct {
+		Type string     `json:"type"`
+		Text *slackText `json:"text,omitempty"`
+	}
+
+	// Generate request
+	body, err := json.MarshalIndent(struct {
+		Blocks []slackBlock `json:"blocks"`
+	}{
+		Blocks: []slackBlock{{
+			Type: "section",
+			Text: &slackText{
+				Type: "mrkdwn",
+				Text: summary,
+			},
+		}},
+	}, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to post on slack: %w", err)
+	}
+	req, err := http.NewRequest(http.MethodPost, webhook, bytes.NewBuffer(body))
+	if err != nil {
+		return fmt.Errorf("failed to post on slack: %w", err)
+	}
+	req.Header.Add("Content-Type", "application/json")
+
+	// Perform the HTTP Post on the webhook
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to post on slack: %w", err)
+	}
+
+	// Parse the response, to check if it succeeded
+	buf := new(bytes.Buffer)
+	_, err = buf.ReadFrom(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if buf.String() != "ok" {
+		return fmt.Errorf("failed to post on slack: %s", buf.String())
+	}
+	return nil
+}
--- a/dev/buildchecker/slack_test.go
+++ b/dev/buildchecker/slack_test.go
@ -0,0 +1,26 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSlackSummary(t *testing.T) {
+	t.Run("unlocked", func(t *testing.T) {
+		s := slackSummary(false, []CommitInfo{})
+		t.Log(s)
+		assert.Contains(t, s, "unlocked")
+	})
+
+	t.Run("locked", func(t *testing.T) {
+		s := slackSummary(true, []CommitInfo{
+			{Commit: "a", Author: "bob"},
+			{Commit: "b", Author: "alice"},
+		})
+		t.Log(s)
+		assert.Contains(t, s, "locked")
+		assert.Contains(t, s, "bob")
+		assert.Contains(t, s, "alice")
+	})
+}
--- a/dev/buildchecker/testdata/TestRepoBranchLocker/lock.yaml
+++ b/dev/buildchecker/testdata/TestRepoBranchLocker/lock.yaml
--- a/dev/buildchecker/testdata/TestRepoBranchLocker/unlock.yaml
+++ b/dev/buildchecker/testdata/TestRepoBranchLocker/unlock.yaml
@ -0,0 +1,271 @@
+---
+version: 1
+interactions:
+- request:
+    body: ""
+    form: {}
+    headers:
+      Accept:
+      - application/vnd.github.luke-cage-preview+json
+      User-Agent:
+      - go-github
+    url: https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection
+    method: GET
+  response:
+    body: '{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection","required_status_checks":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_status_checks","strict":false,"contexts":["buildkite/sourcegraph"],"contexts_url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_status_checks/contexts","checks":[{"context":"buildkite/sourcegraph","app_id":72}]},"restrictions":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/restrictions","users_url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/restrictions/users","teams_url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/restrictions/teams","apps_url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/restrictions/apps","users":[{"login":"jhchabran","id":10151,"node_id":"MDQ6VXNlcjEwMTUx","avatar_url":"https://avatars.githubusercontent.com/u/10151?v=4","gravatar_id":"","url":"https://api.github.com/users/jhchabran","html_url":"https://github.com/jhchabran","followers_url":"https://api.github.com/users/jhchabran/followers","following_url":"https://api.github.com/users/jhchabran/following{/other_user}","gists_url":"https://api.github.com/users/jhchabran/gists{/gist_id}","starred_url":"https://api.github.com/users/jhchabran/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jhchabran/subscriptions","organizations_url":"https://api.github.com/users/jhchabran/orgs","repos_url":"https://api.github.com/users/jhchabran/repos","events_url":"https://api.github.com/users/jhchabran/events{/privacy}","received_events_url":"https://api.github.com/users/jhchabran/received_events","type":"User","site_admin":false},{"login":"davejrt","id":2067825,"node_id":"MDQ6VXNlcjIwNjc4MjU=","avatar_url":"https://avatars.githubusercontent.com/u/2067825?v=4","gravatar_id":"","url":"https://api.github.com/users/davejrt","html_url":"https://github.com/davejrt","followers_url":"https://api.github.com/users/davejrt/followers","following_url":"https://api.github.com/users/davejrt/following{/other_user}","gists_url":"https://api.github.com/users/davejrt/gists{/gist_id}","starred_url":"https://api.github.com/users/davejrt/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davejrt/subscriptions","organizations_url":"https://api.github.com/users/davejrt/orgs","repos_url":"https://api.github.com/users/davejrt/repos","events_url":"https://api.github.com/users/davejrt/events{/privacy}","received_events_url":"https://api.github.com/users/davejrt/received_events","type":"User","site_admin":false},{"login":"bobheadxi","id":23356519,"node_id":"MDQ6VXNlcjIzMzU2NTE5","avatar_url":"https://avatars.githubusercontent.com/u/23356519?v=4","gravatar_id":"","url":"https://api.github.com/users/bobheadxi","html_url":"https://github.com/bobheadxi","followers_url":"https://api.github.com/users/bobheadxi/followers","following_url":"https://api.github.com/users/bobheadxi/following{/other_user}","gists_url":"https://api.github.com/users/bobheadxi/gists{/gist_id}","starred_url":"https://api.github.com/users/bobheadxi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bobheadxi/subscriptions","organizations_url":"https://api.github.com/users/bobheadxi/orgs","repos_url":"https://api.github.com/users/bobheadxi/repos","events_url":"https://api.github.com/users/bobheadxi/events{/privacy}","received_events_url":"https://api.github.com/users/bobheadxi/received_events","type":"User","site_admin":false}],"teams":[{"name":"Dev
+      Experience","id":5135343,"node_id":"T_kwDOADy5QM4ATlvv","slug":"dev-experience","description":"All
+      members of the Dev Experience team","privacy":"closed","url":"https://api.github.com/organizations/3979584/team/5135343","html_url":"https://github.com/orgs/sourcegraph/teams/dev-experience","members_url":"https://api.github.com/organizations/3979584/team/5135343/members{/member}","repositories_url":"https://api.github.com/organizations/3979584/team/5135343/repos","permission":"pull","parent":{"name":"Enablement","id":5143057,"node_id":"T_kwDOADy5QM4ATnoR","slug":"enablement","description":"Everyone
+      in the Enablement org","privacy":"closed","url":"https://api.github.com/organizations/3979584/team/5143057","html_url":"https://github.com/orgs/sourcegraph/teams/enablement","members_url":"https://api.github.com/organizations/3979584/team/5143057/members{/member}","repositories_url":"https://api.github.com/organizations/3979584/team/5143057/repos","permission":"pull"}}],"apps":[]},"required_pull_request_reviews":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_pull_request_reviews","dismiss_stale_reviews":false,"require_code_owner_reviews":false,"required_approving_review_count":1},"required_signatures":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_signatures","enabled":false},"enforce_admins":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/enforce_admins","enabled":false},"required_linear_history":{"enabled":true},"allow_force_pushes":{"enabled":false},"allow_deletions":{"enabled":false},"required_conversation_resolution":{"enabled":false}}'
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      Access-Control-Expose-Headers:
+      - ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
+        X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
+        X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
+        X-GitHub-Request-Id, Deprecation, Sunset
+      Cache-Control:
+      - private, max-age=60, s-maxage=60
+      Content-Security-Policy:
+      - default-src 'none'
+      Content-Type:
+      - application/json; charset=utf-8
+      Date:
+      - Thu, 16 Dec 2021 17:32:24 GMT
+      Etag:
+      - W/"841e1b09e9e21268c5d2b0b59620367f042c4a08ffe1f3e31e794b2cbbb79d56"
+      Referrer-Policy:
+      - origin-when-cross-origin, strict-origin-when-cross-origin
+      Server:
+      - GitHub.com
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubdomains; preload
+      Vary:
+      - Accept, Authorization, Cookie, X-GitHub-OTP
+      - Accept-Encoding, Accept, X-Requested-With
+      X-Accepted-Oauth-Scopes:
+      - ""
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Github-Media-Type:
+      - github.v3; param=luke-cage-preview; format=json
+      X-Github-Request-Id:
+      - E826:16A2:2126F1A:3DFF1D9:61BB7827
+      X-Oauth-Scopes:
+      - admin:enterprise, admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
+        admin:repo_hook, delete:packages, delete_repo, gist, notifications, repo,
+        user, workflow, write:discussion, write:packages
+      X-Ratelimit-Limit:
+      - "5000"
+      X-Ratelimit-Remaining:
+      - "4986"
+      X-Ratelimit-Reset:
+      - "1639679540"
+      X-Ratelimit-Resource:
+      - core
+      X-Ratelimit-Used:
+      - "14"
+      X-Xss-Protection:
+      - "0"
+    status: 200 OK
+    code: 200
+    duration: ""
+- request:
+    body: ""
+    form: {}
+    headers:
+      Accept:
+      - application/vnd.github.v3+json
+      User-Agent:
+      - go-github
+    url: https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/restrictions
+    method: DELETE
+  response:
+    body: ""
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      Access-Control-Expose-Headers:
+      - ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
+        X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
+        X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
+        X-GitHub-Request-Id, Deprecation, Sunset
+      Content-Security-Policy:
+      - default-src 'none'
+      Date:
+      - Thu, 16 Dec 2021 17:32:24 GMT
+      Referrer-Policy:
+      - origin-when-cross-origin, strict-origin-when-cross-origin
+      Server:
+      - GitHub.com
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubdomains; preload
+      Vary:
+      - Accept-Encoding, Accept, X-Requested-With
+      X-Accepted-Oauth-Scopes:
+      - ""
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Github-Media-Type:
+      - github.v3; format=json
+      X-Github-Request-Id:
+      - E826:16A2:2126F36:3DFF1FB:61BB7828
+      X-Oauth-Scopes:
+      - admin:enterprise, admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
+        admin:repo_hook, delete:packages, delete_repo, gist, notifications, repo,
+        user, workflow, write:discussion, write:packages
+      X-Ratelimit-Limit:
+      - "5000"
+      X-Ratelimit-Remaining:
+      - "4985"
+      X-Ratelimit-Reset:
+      - "1639679540"
+      X-Ratelimit-Resource:
+      - core
+      X-Ratelimit-Used:
+      - "15"
+      X-Xss-Protection:
+      - "0"
+    status: 204 No Content
+    code: 204
+    duration: ""
+- request:
+    body: ""
+    form: {}
+    headers:
+      Accept:
+      - application/vnd.github.luke-cage-preview+json
+      User-Agent:
+      - go-github
+    url: https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection
+    method: GET
+  response:
+    body: '{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection","required_status_checks":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_status_checks","strict":false,"contexts":["buildkite/sourcegraph"],"contexts_url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_status_checks/contexts","checks":[{"context":"buildkite/sourcegraph","app_id":72}]},"required_pull_request_reviews":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_pull_request_reviews","dismiss_stale_reviews":false,"require_code_owner_reviews":false,"required_approving_review_count":1},"required_signatures":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_signatures","enabled":false},"enforce_admins":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/enforce_admins","enabled":false},"required_linear_history":{"enabled":true},"allow_force_pushes":{"enabled":false},"allow_deletions":{"enabled":false},"required_conversation_resolution":{"enabled":false}}'
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      Access-Control-Expose-Headers:
+      - ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
+        X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
+        X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
+        X-GitHub-Request-Id, Deprecation, Sunset
+      Cache-Control:
+      - private, max-age=60, s-maxage=60
+      Content-Security-Policy:
+      - default-src 'none'
+      Content-Type:
+      - application/json; charset=utf-8
+      Date:
+      - Thu, 16 Dec 2021 17:32:24 GMT
+      Etag:
+      - W/"85ca80472d0df1875263e8356a74b6544623e56fdc54f3218ba5ada0ec01b186"
+      Referrer-Policy:
+      - origin-when-cross-origin, strict-origin-when-cross-origin
+      Server:
+      - GitHub.com
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubdomains; preload
+      Vary:
+      - Accept, Authorization, Cookie, X-GitHub-OTP
+      - Accept-Encoding, Accept, X-Requested-With
+      X-Accepted-Oauth-Scopes:
+      - ""
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Github-Media-Type:
+      - github.v3; param=luke-cage-preview; format=json
+      X-Github-Request-Id:
+      - E826:16A2:2126F65:3DFF235:61BB7828
+      X-Oauth-Scopes:
+      - admin:enterprise, admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
+        admin:repo_hook, delete:packages, delete_repo, gist, notifications, repo,
+        user, workflow, write:discussion, write:packages
+      X-Ratelimit-Limit:
+      - "5000"
+      X-Ratelimit-Remaining:
+      - "4984"
+      X-Ratelimit-Reset:
+      - "1639679540"
+      X-Ratelimit-Resource:
+      - core
+      X-Ratelimit-Used:
+      - "16"
+      X-Xss-Protection:
+      - "0"
+    status: 200 OK
+    code: 200
+    duration: ""
+- request:
+    body: ""
+    form: {}
+    headers:
+      Accept:
+      - application/vnd.github.luke-cage-preview+json
+      User-Agent:
+      - go-github
+    url: https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection
+    method: GET
+  response:
+    body: '{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection","required_status_checks":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_status_checks","strict":false,"contexts":["buildkite/sourcegraph"],"contexts_url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_status_checks/contexts","checks":[{"context":"buildkite/sourcegraph","app_id":72}]},"required_pull_request_reviews":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_pull_request_reviews","dismiss_stale_reviews":false,"require_code_owner_reviews":false,"required_approving_review_count":1},"required_signatures":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/required_signatures","enabled":false},"enforce_admins":{"url":"https://api.github.com/repos/sourcegraph/sourcegraph/branches/test-buildsherrif-branch/protection/enforce_admins","enabled":false},"required_linear_history":{"enabled":true},"allow_force_pushes":{"enabled":false},"allow_deletions":{"enabled":false},"required_conversation_resolution":{"enabled":false}}'
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      Access-Control-Expose-Headers:
+      - ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
+        X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
+        X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
+        X-GitHub-Request-Id, Deprecation, Sunset
+      Cache-Control:
+      - private, max-age=60, s-maxage=60
+      Content-Security-Policy:
+      - default-src 'none'
+      Content-Type:
+      - application/json; charset=utf-8
+      Date:
+      - Thu, 16 Dec 2021 17:32:24 GMT
+      Etag:
+      - W/"85ca80472d0df1875263e8356a74b6544623e56fdc54f3218ba5ada0ec01b186"
+      Referrer-Policy:
+      - origin-when-cross-origin, strict-origin-when-cross-origin
+      Server:
+      - GitHub.com
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubdomains; preload
+      Vary:
+      - Accept, Authorization, Cookie, X-GitHub-OTP
+      - Accept-Encoding, Accept, X-Requested-With
+      X-Accepted-Oauth-Scopes:
+      - ""
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Github-Media-Type:
+      - github.v3; param=luke-cage-preview; format=json
+      X-Github-Request-Id:
+      - E826:16A2:2126F75:3DFF25F:61BB7828
+      X-Oauth-Scopes:
+      - admin:enterprise, admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
+        admin:repo_hook, delete:packages, delete_repo, gist, notifications, repo,
+        user, workflow, write:discussion, write:packages
+      X-Ratelimit-Limit:
+      - "5000"
+      X-Ratelimit-Remaining:
+      - "4983"
+      X-Ratelimit-Reset:
+      - "1639679540"
+      X-Ratelimit-Resource:
+      - core
+      X-Ratelimit-Used:
+      - "17"
+      X-Xss-Protection:
+      - "0"
+    status: 200 OK
+    code: 200
+    duration: ""
--- a/go.mod
+++ b/go.mod
@ -330,6 +330,7 @@ require (
 	github.com/godbus/dbus/v5 v5.0.6 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-github/v41 v41.0.0
 	github.com/gopherjs/gopherjs v0.0.0-20211111143520-d0d5ecc1a356 // indirect
 	github.com/gopherjs/gopherwasm v1.1.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
--- a/go.sum
+++ b/go.sum
@ -721,6 +721,8 @@ github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBE
 github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
 github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo=
 github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM=
+github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27ueww4gg=
+github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
--- a/internal/httptestutil/recorder.go
+++ b/internal/httptestutil/recorder.go
@ -12,7 +12,10 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/httpcli"
 )

-// NewRecorder returns an HTTP interaction recorder with the given record mode and filters. It strips away the HTTP Authorization and Set-Cookie headers.
+// NewRecorder returns an HTTP interaction recorder with the given record mode and filters.
+// It strips away the HTTP Authorization and Set-Cookie headers.
+//
+// To save interactions, make sure to call .Stop().
 func NewRecorder(file string, record bool, filters ...cassette.Filter) (*recorder.Recorder, error) {
 	mode := recorder.ModeReplaying
 	if record {