artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 16:11:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(build-tracker): Failed back-compat doesn't count towards branch-locking quota (#63911)

Browse Source 
Closes
[DINF-51](https://linear.app/sourcegraph/issue/DINF-51/failed-back-compat-doesnt-count-towards-branch-locking-quota)

## Context

If a back-compat step on main fails, the build is marked as having
failed. However, we don't treat that as a failure in build-tracker,
resulting in no #buildkite-main post and not counting towards failed
build quota for locking main.

The reason why this was happening is that the Backcompat build wasn't
linked to the main Sourcegraph build in anyway. However, when a
backcompat fails the main build reflects the status of this failure, but
we do not use this field when determining the status of a build, so it
doesn't work for our use case.

![CleanShot 2024-07-18 at 15 04
15@2x](https://github.com/user-attachments/assets/9553330a-ad98-45cc-b4ce-03a22ca1b99d)

We [instead do a walkthrough of all the jobs associated with a build to
figure
out](https://sourcegraph.sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/dev/build-tracker/main.go?L349-372)
if the build has failed, fixed or is passing.

With this logic, it means we have to link the steps from child builds
that a particular build triggers to it's parent.

## Test plan

* Create a build that'll have backcompat failing
* The build tracker event associated with the main build will be
reported with a state of failed to buildkite.

![CleanShot 2024-07-18 at 15 10
45@2x](https://github.com/user-attachments/assets/1bf503ab-0020-47bf-9512-b3a9ee5d4e36)


## Changelog

<!-- OPTIONAL; info at
https://www.notion.so/sourcegraph/Writing-a-changelog-entry-dd997f411d524caabf0d8d38a24a878c
-->
This commit is contained in:
Bolaji Olajide 2024-07-25 12:45:09 +01:00 committed by GitHub
parent b84636fb55
commit 20b858f6c3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 37 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
deps.bzl
View File

@ -930,8 +930,8 @@ def go_dependencies():
name = "com_github_buildkite_go_buildkite_v3",
build_file_proto_mode = "disable_global",
importpath = "github.com/buildkite/go-buildkite/v3",
sum = "h1:5kX1fFDj3Co7cP6cqZKuW1VoCJz3u4cOx6wfdCeM4ZA=",
version = "v3.0.1",
sum = "h1:A43KDOuNczqrY8wqlsHNtPoYbgWXYC/slkB/2JYXr5E=",
version = "v3.11.0",
)
go_repository(
name = "com_github_burntsushi_toml",

2
dev/bkstats/main.go
View File

@ -80,7 +80,7 @@ func main() {
nextPage := 0
for {
bs, resp, err := client.Builds.ListByPipeline("sourcegraph", pipeline, &buildkite.BuildsListOptions{
Branch: "main",
Branch: []string{"main"},
// Select all builds that finished on or after the beginning of the day ...
FinishedFrom: BoD(t),
// To those who were created before or on the end of the day.

28
dev/build-tracker/build/build.go
View File

@ -154,6 +154,12 @@ func (b *Build) GetMessage() string {
return pointers.DerefZero(b.Message)
}
func (b *Build) AppendSteps(steps map[string]*Step) {
for name, step := range steps {
b.Steps[name] = step
}
}
// Pipeline wraps a buildkite.Pipeline and provides convenience functions to access values of the wrapped pipeline in a safe maner
type Pipeline struct {
buildkite.Pipeline `json:"pipeline"`
@ -263,6 +269,28 @@ func (s *Store) Add(event *Event) {
log.Int("totalSteps", len(build.Steps)),
log.String("status", build.GetState()))
// If the build was triggered from another build, we need to update the "trigger-er" with the jobs
// from the triggered build. This is so that any failures from the triggered build are reported as
// failures in the triggerer.
// We do this because we do not rely on the state of the build to determine if a build is "successful" or not.
// We instead depend on the state of the jobs associated with said build.
if event.Build.TriggeredFrom != nil {
parentBuild, ok := s.builds[*event.Build.TriggeredFrom.BuildNumber]
if ok {
parentBuild.Lock()
parentBuild.AppendSteps(build.Steps)
parentBuild.Unlock()
} else {
// If the triggered build doesn't exist, we'll just leave log a message
s.logger.Warn(
"build triggered from non-existent build",
log.Int("buildNumber", event.GetBuildNumber()),
log.String("pipeline", *event.Build.TriggeredFrom.BuildPipelineSlug),
log.Int("triggeredFrom", *event.Build.TriggeredFrom.BuildNumber),
)
}
}
// Track consecutive failures by pipeline + branch
// We update the global count of consecutiveFailures then we set the count on the individual build
// if we get a pass, we reset the global count of consecutiveFailures

2
dev/build-tracker/main.go
View File

@ -418,7 +418,7 @@ type Service struct{}
func (s Service) Initialize(ctx context.Context, logger log.Logger, contract runtime.ServiceContract, config config.Config) (background.Routine, error) {
logger.Info("config loaded from environment", log.Object("config", log.String("SlackChannel", config.SlackChannel), log.Bool("Production", config.Production)))
bqWriter, err := contract.BigQuery.GetTableWriter(context.Background(), "agent_status")
bqWriter, err := contract.BigQuery.GetTableWriter(ctx, "agent_status")
if err != nil {
return nil, err
}

2
dev/buildchecker/main.go
View File

@ -86,7 +86,7 @@ func cmdCheck(ctx context.Context, flags *Flags, checkFlags *cmdCheckFlags) {
// Newest is returned first https://buildkite.com/docs/apis/rest-api/builds#list-builds-for-a-pipeline
builds, _, err := bkc.Builds.ListByPipeline("sourcegraph", flags.Pipeline, &buildkite.BuildsListOptions{
Branch: flags.Branch,
Branch: []string{flags.Branch},
// Fix to high page size just in case, default is 30
// https://buildkite.com/docs/apis/rest-api#pagination
ListOptions: buildkite.ListOptions{PerPage: 99},

2
dev/sg/internal/bk/bk.go
View File

@ -99,7 +99,7 @@ func NewClient(ctx context.Context, out *std.Output) (*Client, error) {
// If no builds are found, an error will be returned.
func (c *Client) GetMostRecentBuild(ctx context.Context, pipeline, branch string) (*buildkite.Build, error) {
builds, _, err := c.bk.Builds.ListByPipeline(BuildkiteOrg, pipeline, &buildkite.BuildsListOptions{
Branch: branch,
Branch: []string{branch},
})
if err != nil {
if strings.Contains(err.Error(), "404 Not Found") {

2
go.mod
View File

@ -91,7 +91,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
github.com/aws/smithy-go v1.13.5
github.com/beevik/etree v1.3.0
github.com/buildkite/go-buildkite/v3 v3.0.1
github.com/buildkite/go-buildkite/v3 v3.11.0
github.com/cespare/xxhash/v2 v2.3.0
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/coreos/go-semver v0.3.1

7
go.sum
View File

@ -762,7 +762,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0=
@ -936,14 +935,13 @@ github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
github.com/buildkite/go-buildkite/v3 v3.0.1 h1:5kX1fFDj3Co7cP6cqZKuW1VoCJz3u4cOx6wfdCeM4ZA=
github.com/buildkite/go-buildkite/v3 v3.0.1/go.mod h1:6pweknacVv7He5Lvbf54urp2P0W6/b4Nrcxn718PQrE=
github.com/buildkite/go-buildkite/v3 v3.11.0 h1:A43KDOuNczqrY8wqlsHNtPoYbgWXYC/slkB/2JYXr5E=
github.com/buildkite/go-buildkite/v3 v3.11.0/go.mod h1:TmZggyr5HqkOhNbTrcdOdmwuYbQqcfwr9MSyKyMQWAA=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY=
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
github.com/cenkalti/backoff v1.1.1-0.20171020064038-309aa717adbf/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@ -1432,7 +1430,6 @@ github.com/google/go-github/v48 v48.2.0 h1:68puzySE6WqUY9KWmpOsDEQfDZsso98rT6pZc
github.com/google/go-github/v48 v48.2.0/go.mod h1:dDlehKBDo850ZPvCTK0sEqTCVWcrGl2LcDiajkYi89Y=
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=