diff --git a/release.yaml b/release.yaml index 28e2670bcac..eaba34663f3 100644 --- a/release.yaml +++ b/release.yaml @@ -1,8 +1,8 @@ meta: productName: sourcegraph - repository: "github.com/sourcegraph/sourcegraph" + repository: 'github.com/sourcegraph/sourcegraph' owners: - - "@sourcegraph/release" + - '@sourcegraph/release' requirements: - name: "curl" @@ -17,14 +17,14 @@ requirements: # #bolaji-release-testing # https://start.1password.com/open/i?a=HEDEDSLHPBFGRBTKAKJWE23XX4&v=isv4abynddpox72wwbhaamo76e&i=7zjax5rm5hlilbgrzeb257i62i&h=team-sourcegraph.1password.com - - name: "Slack Webhook URL" + - name: 'Slack Webhook URL' env: SLACK_WEBHOOK_URL internal: create: steps: patch: - - name: "buildkite" + - name: 'buildkite' cmd: | echo "Triggering build on sourcegraph/sourcegraph with VERSION={{version}} on branch {{git.branch}}" body=$(curl -s --fail-with-body -X POST "https://api.buildkite.com/v2/organizations/sourcegraph/pipelines/sourcegraph/builds" -H "Content-Type: application/json" -H "Authorization: Bearer $BUILDKITE_ACCESS_TOKEN" -d '{ @@ -50,10 +50,10 @@ internal: fi finalize: steps: - - name: "Register on release registry" + - name: 'Register on release registry' cmd: | echo "pretending to call release registry api" - - name: "notifications" + - name: 'notifications' cmd: | set -eu @@ -64,14 +64,14 @@ internal: test: steps: - - name: "check:git tag does not exist" + - name: 'check:git tag does not exist' cmd: | tags=$(git ls-remote --tags origin | sort -t '/' -k 3 | cut -f 2 | awk -F '/' '{gsub(/\^\{\}$/, "", $3); print $3}' | uniq) if echo "${tags}" | grep -q "^{{version}}$"; then echo "❌ Tag '{{version}}' already exists" exit 1 fi - - name: "check:migrator-schemas" + - name: 'check:migrator-schemas' cmd: | set -eu @@ -107,11 +107,35 @@ test: echo "migrator:{{tag}} does not contain the correct amount of schema description files for this release - expected more than 300 got ${count}" exit 1 fi + - name: 'db:migration:coherence_test' + cmd: | + set -eu + + aspectRC="/tmp/aspect-generated.bazelrc" + rosetta bazelrc > "$aspectRC" + bazelrc=(--bazelrc="$aspectRC" --bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc) + + # The upgradetest are inferring the stamp-version flag based on the version, so we need to unset it here. + _VERSION=$VERSION + unset VERSION + + # We purposely limit the concurrency to 6, because if we use the default, there are + # scenarios where we can exhaust available ports to the docker daemon and end up + # with an infrastructure flake. + max_routines=6 + # Hardcoding version, as for now I just want to make sure this works in CI. + bazel ${bazelrc[*]} run //testing/tools/upgradetest:release_test_run -- all \ + --post-release-version={{tag}} \ + --target-registry us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal/ \ + --max-routines $max_routines + + # Restoring it to avoid creating a footgun if we add more test steps later on. + VERSION=$_VERSION promoteToPublic: create: steps: - - name: "buildkite" + - name: 'buildkite' cmd: | # We set DISABLE_ASPECT_WORKFLOWS to true, because the promotion is purely about retagging images # and we don't rely on AW at all. @@ -140,10 +164,10 @@ promoteToPublic: fi finalize: steps: - - name: "Promote on release registry" + - name: 'Promote on release registry' cmd: | echo "pretending to call release registry api" - - name: "git:tag" + - name: 'git:tag' cmd: | set -eu @@ -159,7 +183,7 @@ promoteToPublic: EOF # tag is usually in the format `5.3.2` # while version is usually the tag prepended with a v, `v5.3.2` - - name: "Slack notification (#announce-engineering)" + - name: 'Slack notification (#announce-engineering)' cmd: | echo "Posting slack notification for release" tag="{{tag}}" diff --git a/testing/tools/upgradetest/BUILD.bazel b/testing/tools/upgradetest/BUILD.bazel index 51cbd58a91f..6fcfcd25ebf 100644 --- a/testing/tools/upgradetest/BUILD.bazel +++ b/testing/tools/upgradetest/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "upgradetest_lib", @@ -68,3 +68,14 @@ sh_binary( "//internal/database:generate_schemas", ], ) + +sh_binary( + name = "release_test_run", + srcs = ["release_test.sh"], + args = [ + "$(location :go_upgradetest)", + ], + data = [ + ":go_upgradetest", + ], +) diff --git a/testing/tools/upgradetest/main.go b/testing/tools/upgradetest/main.go index 8a19cdf38d8..d5d8bf6f765 100644 --- a/testing/tools/upgradetest/main.go +++ b/testing/tools/upgradetest/main.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "os" + "runtime" + "strings" "time" _ "github.com/lib/pq" @@ -14,13 +16,15 @@ import ( "github.com/sourcegraph/run" ) -// These commands are meant to be executed with a VERSION env var with a hypothetical stamped release version -// This type is used to assign the stamp version from VERSION type stampVersionKey struct{} type postReleaseKey struct{} +type targetRegistryKey struct{} +type fromRegistryKey struct{} // Register upgrade commands -- see README.md for more details. func main() { + fmt.Println("👉 Upgrade test ...") + app := &cli.App{ Name: "upgrade-test", Usage: "Upgrade test is a tool for testing the migrator services creation of upgrade paths and application of upgrade paths.\nWhen run relevant upgrade paths are tested for each version relevant to a given upgrade type, initializing Sourcegraph databases and frontend services for each version, and attempting to generate and apply an upgrade path to your current branches head.", @@ -41,11 +45,21 @@ func main() { Aliases: []string{"pv"}, Usage: "Select an already released version as the target version for the test suite.", }, + &cli.StringFlag{ + Name: "target-registry", + Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, + &cli.StringFlag{ + Name: "from-registry", + Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, &cli.IntFlag{ Name: "max-routines", Aliases: []string{"mr"}, - Usage: "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.", - Value: 10, + Usage: "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to CPU cores count minus two.", + Value: runtime.NumCPU() - 2, }, &cli.StringSliceFlag{ Name: "standard-versions", @@ -66,6 +80,8 @@ func main() { Action: func(cCtx *cli.Context) error { ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version")) ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version")) + ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry")) + ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry")) // check docker is running if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil { @@ -74,7 +90,12 @@ func main() { } // Get init versions to use for initializing upgrade environments for tests - latestMinorVersion, latestStableVersion, targetVersion, stdVersions, mvuVersions, autoVersions, err := handleVersions(cCtx, cCtx.StringSlice("standard-versions"), cCtx.StringSlice("mvu-versions"), cCtx.StringSlice("auto-versions"), cCtx.String("post-release-version")) + latestMinorVersion, latestStableVersion, targetVersion, stdVersions, mvuVersions, autoVersions, err := handleVersions(cCtx, + cCtx.StringSlice("standard-versions"), + cCtx.StringSlice("mvu-versions"), + cCtx.StringSlice("auto-versions"), + cCtx.String("post-release-version"), + ) if err != nil { fmt.Println("🚨 Error: failed to get test version ranges: ", err) os.Exit(1) @@ -83,7 +104,7 @@ func main() { var targetMigratorImage string switch { case ctx.Value(postReleaseKey{}) != "": - targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{})) + targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v")) case ctx.Value(stampVersionKey{}) != "": targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{})) default: @@ -168,6 +189,10 @@ func main() { // This is where we do the majority of our printing to stdout. results.OrderByVersion() results.PrintSimpleResults() + if results.Failed() { + results.DisplayErrors() + os.Exit(1) + } return nil }, @@ -188,10 +213,20 @@ func main() { Aliases: []string{"pv"}, Usage: "Select an already released version as the target version for the test suite.", }, + &cli.StringFlag{ + Name: "target-registry", + Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, + &cli.StringFlag{ + Name: "from-registry", + Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, &cli.IntFlag{ Name: "max-routines", Aliases: []string{"mr"}, Usage: "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.", - Value: 10, + Value: runtime.NumCPU() - 2, }, &cli.StringSliceFlag{ Name: "standard-versions", @@ -202,6 +237,8 @@ func main() { Action: func(cCtx *cli.Context) error { ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version")) ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version")) + ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry")) + ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry")) // check docker is running if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil { @@ -219,7 +256,7 @@ func main() { var targetMigratorImage string switch { case ctx.Value(postReleaseKey{}) != "": - targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{})) + targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v")) case ctx.Value(stampVersionKey{}) != "": targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{})) default: @@ -250,17 +287,30 @@ func main() { result := standardUpgradeTest(ctx, version, targetVersion, latestStableVersion) result.Runtime = time.Since(start) results.AddStdTest(result) + if len(result.Errors) > 0 { + return result.Errors[0] + } return nil }) } if err := stdTestPool.Wait(); err != nil { fmt.Println("🚨 Error: failed to run tests in pool: ", err) + for _, t := range results.StandardUpgradeTests { + fmt.Println("LOGS") + t.DisplayLog() + fmt.Println("ERROR") + t.DisplayErrors() + } return err } // This is where we do the majority of our printing to stdout. results.OrderByVersion() results.PrintSimpleResults() + if results.Failed() { + results.DisplayErrors() + os.Exit(1) + } return nil }, @@ -281,11 +331,21 @@ func main() { Aliases: []string{"pv"}, Usage: "Select an already released version as the target version for the test suite.", }, + &cli.StringFlag{ + Name: "target-registry", + Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, + &cli.StringFlag{ + Name: "from-registry", + Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, &cli.IntFlag{ Name: "max-routines", Aliases: []string{"mr"}, Usage: "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.", - Value: 10, + Value: runtime.NumCPU() - 2, }, &cli.StringSliceFlag{ Name: "mvu-versions", @@ -296,6 +356,8 @@ func main() { Action: func(cCtx *cli.Context) error { ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version")) ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version")) + ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry")) + ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry")) // check docker is running if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil { @@ -313,7 +375,7 @@ func main() { var targetMigratorImage string switch { case ctx.Value(postReleaseKey{}) != "": - targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{})) + targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v")) case ctx.Value(stampVersionKey{}) != "": targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{})) default: @@ -342,6 +404,9 @@ func main() { result := multiversionUpgradeTest(ctx, version, targetVersion, latestStableVersion) result.Runtime = time.Since(start) results.AddMVUTest(result) + if len(result.Errors) > 0 { + return result.Errors[0] + } return nil }) } @@ -352,6 +417,10 @@ func main() { results.OrderByVersion() results.PrintSimpleResults() + if results.Failed() { + results.DisplayErrors() + os.Exit(1) + } return nil }, @@ -372,11 +441,22 @@ func main() { Aliases: []string{"pv"}, Usage: "Select an already released version as the target version for the test suite.", }, + + &cli.StringFlag{ + Name: "target-registry", + Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, + &cli.StringFlag{ + Name: "from-registry", + Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:", + Value: "sourcegraph/", + }, &cli.IntFlag{ Name: "max-routines", Aliases: []string{"mr"}, Usage: "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.", - Value: 10, + Value: runtime.NumCPU() - 2, }, &cli.StringSliceFlag{ Name: "auto-versions", @@ -387,6 +467,8 @@ func main() { Action: func(cCtx *cli.Context) error { ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version")) ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version")) + ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry")) + ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry")) // check docker is running if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil { @@ -404,7 +486,7 @@ func main() { var targetMigratorImage string switch { case ctx.Value(postReleaseKey{}) != "": - targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{})) + targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v")) case ctx.Value(stampVersionKey{}) != "": targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{})) default: @@ -433,6 +515,10 @@ func main() { result := autoUpgradeTest(ctx, version, targetVersion, latestStableVersion) result.Runtime = time.Since(start) results.AddAutoTest(result) + if len(result.Errors) > 0 { + return result.Errors[0] + } + return nil }) } @@ -443,6 +529,10 @@ func main() { results.OrderByVersion() results.PrintSimpleResults() + if results.Failed() { + results.DisplayErrors() + os.Exit(1) + } return nil }, @@ -452,6 +542,7 @@ func main() { if err := app.Run(os.Args); err != nil { fmt.Println("🚨 Error: failed to run tests: ", err) + os.Exit(1) } } diff --git a/testing/tools/upgradetest/release_test.sh b/testing/tools/upgradetest/release_test.sh new file mode 100755 index 00000000000..2c0bf8d977d --- /dev/null +++ b/testing/tools/upgradetest/release_test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +RUNNER="$1" + +# internal/database/. artifacts are being loaded as arguments, 13 is the beginning of passed arguments to the cli tool +# Args: +# bazel-bin/testing/tools/upgradetest/sh_upgradetest_run +# testing/tools/upgradetest/upgradetest-darwin-arm64 +# cmd/migrator/image_tarball.sh +# cmd/frontend/image_tarball.sh +# internal/database/_codeinsights_squashed.sql +# internal/database/_codeintel_squashed.sql +# internal/database/_frontend_squashed.sql +# internal/database/_schema.codeinsights.json +# internal/database/_schema.codeinsights.md +# internal/database/_schema.codeintel.json +# internal/database/_schema.codeintel.md +# internal/database/_schema.json +# internal/database/_schema.md +"$RUNNER" "${@:2}" diff --git a/testing/tools/upgradetest/test.sh b/testing/tools/upgradetest/test.sh index 982344b559e..098fe8c6ed3 100755 --- a/testing/tools/upgradetest/test.sh +++ b/testing/tools/upgradetest/test.sh @@ -27,5 +27,3 @@ FRONTEND_TARBALL="$3" # internal/database/_schema.json # internal/database/_schema.md "$RUNNER" "${@:13}" - -exit diff --git a/testing/tools/upgradetest/tests-util.go b/testing/tools/upgradetest/tests-util.go index f30f7f566e7..56d61bfaa1b 100644 --- a/testing/tools/upgradetest/tests-util.go +++ b/testing/tools/upgradetest/tests-util.go @@ -43,7 +43,7 @@ func (t *Test) AddError(err error) { // DisplayErrors prints errors to stdout func (t *Test) DisplayErrors() { for _, err := range t.Errors { - fmt.Println(err) + fmt.Println(err.Error()) } } @@ -54,6 +54,11 @@ func (t *Test) DisplayLog() { } } +// Display if a test has failed +func (t *Test) Failed() bool { + return 0 < len(t.Errors) +} + // TestResults is a collection of tests, organized by type. Its methods are generally used to control its logging behavior. type TestResults struct { StandardUpgradeTests []Test @@ -83,6 +88,32 @@ func (r *TestResults) AddAutoTest(test Test) { r.AutoupgradeTests = append(r.AutoupgradeTests, test) } +// Failed returns true if any given test has errors registered. +func (r *TestResults) Failed() bool { + if 0 < len(r.StandardUpgradeTests) { + for _, test := range r.StandardUpgradeTests { + if test.Failed() { + return true + } + } + } + if 0 < len(r.MVUUpgradeTests) { + for _, test := range r.MVUUpgradeTests { + if test.Failed() { + return true + } + } + } + if 0 < len(r.AutoupgradeTests) { + for _, test := range r.AutoupgradeTests { + if test.Failed() { + return true + } + } + } + return false +} + // Used in all-type test type typeVersion struct { Type string @@ -118,7 +149,7 @@ func (r *TestResults) PrintSimpleResults() { if len(r.StandardUpgradeTests) != 0 { stdRes := []string{} for _, test := range r.StandardUpgradeTests { - if 0 < len(test.Errors) { + if test.Failed() { stdRes = append(stdRes, fmt.Sprintf("🚨 %s Failed -- %s\n%s", test.Version.String(), test.Runtime, test.Errors[len(test.Errors)-1])) } else { stdRes = append(stdRes, fmt.Sprintf("✅ %s Passed -- %s ", test.Version.String(), test.Runtime)) @@ -130,7 +161,7 @@ func (r *TestResults) PrintSimpleResults() { if len(r.MVUUpgradeTests) != 0 { mvuRes := []string{} for _, test := range r.MVUUpgradeTests { - if 0 < len(test.Errors) { + if test.Failed() { mvuRes = append(mvuRes, fmt.Sprintf("🚨 %s Failed -- %s\n%s", test.Version.String(), test.Runtime, test.Errors[len(test.Errors)-1])) } else { mvuRes = append(mvuRes, fmt.Sprintf("✅ %s Passed -- %s", test.Version.String(), test.Runtime)) @@ -142,7 +173,7 @@ func (r *TestResults) PrintSimpleResults() { if len(r.AutoupgradeTests) != 0 { autoRes := []string{} for _, test := range r.AutoupgradeTests { - if 0 < len(test.Errors) { + if test.Failed() { autoRes = append(autoRes, fmt.Sprintf("🚨 %s Failed -- %s\n%s", test.Version.String(), test.Runtime, test.Errors[len(test.Errors)-1])) } else { autoRes = append(autoRes, fmt.Sprintf("✅ %s Passed -- %s", test.Version.String(), test.Runtime)) @@ -158,21 +189,21 @@ func (r *TestResults) DisplayErrors() { r.Mutex.Lock() defer r.Mutex.Unlock() for _, test := range r.StandardUpgradeTests { - if 0 < len(test.Errors) { + if test.Failed() { fmt.Printf("--- 🚨 Standard Upgrade Test %s Failed:\n", test.Version.String()) - test.DisplayErrors() + test.DisplayLog() } } for _, test := range r.MVUUpgradeTests { - if 0 < len(test.Errors) { + if test.Failed() { fmt.Printf("--- 🚨 Multiversion Upgrade Test %s Failed:\n", test.Version.String()) - test.DisplayErrors() + test.DisplayLog() } } for _, test := range r.AutoupgradeTests { - if 0 < len(test.Errors) { + if test.Failed() { fmt.Printf("--- 🚨 Auto Upgrade Test %s Failed:\n", test.Version.String()) - test.DisplayErrors() + test.DisplayLog() } } } @@ -221,6 +252,23 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers test.AddLog(fmt.Sprintf("Upgrading from version (%s) to release candidate.", initVersion)) test.AddLog("-- 🏗️ setting up test environment") + // Pull images from registry if -target-registry is set + if ctx.Value(fromRegistryKey{}).(string) != "sourcegraph/" { + test.AddLog(fmt.Sprintf("🐋 pulling -target-registry images from %s", ctx.Value(fromRegistryKey{}).(string))) + out, err := run.Cmd(ctx, "docker", "image", "pull", fmt.Sprintf("%sfrontend:%s", ctx.Value(fromRegistryKey{}).(string), initVersion.String())).Run().String() + test.AddLog(out) + if err != nil { + test.AddError(errors.Newf("🚨 failed to pull images from -target-registry: %s", err)) + } + fmt.Println(out) + out, err = run.Cmd(ctx, "docker", "image", "pull", fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}).(string), initVersion.String())).Run().String() + test.AddLog(out) + if err != nil { + test.AddError(errors.Newf("🚨 failed to pull images from -target-registry: %s", err)) + } + fmt.Println(out) + } + // Create a docker network for testing // // Docker bridge networks take up a lot of the docker daemons available port allocation. We run only a limited amount of test parallelization to get around this. @@ -257,13 +305,24 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers "--name", db.ContainerName, "--network", networkName, "-p", "5432", - fmt.Sprintf("sourcegraph/%s:%s", db.Image, initVersion), + fmt.Sprintf("%s%s:%s", ctx.Value(fromRegistryKey{}), db.Image, initVersion), ).Run().Wait() if err != nil { test.AddError(errors.Newf("🚨 failed to create test databases: %s", err)) } + // get the dynamically allocated port and register it to the test - port, err := run.Cmd(ctx, "docker", "port", db.ContainerName, "5432").Run().String() + out, err := run.Cmd(ctx, "docker", "port", db.ContainerName, "5432").Run().String() + + // docker port can return multiple ports, ipv4 and ipv6, so we need to keep the former only. + ports := strings.Split(out, "\n") + var port string + if len(ports) < 1 { + test.AddError(errors.Newf("incorrect port output for %s", db.ContainerName)) + } else { + port = ports[0] + } + if err != nil { test.AddError(errors.Newf("🚨 failed to get port for %s: %s", db.ContainerName, err)) } @@ -272,7 +331,7 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers // Create a timeout to validate the databases have initialized, this is to prevent a hung test // When many goroutines are running this test this is a point of failure. - dbPingTimeout, cancel := context.WithTimeout(ctx, time.Second*120) + dbPingTimeout, cancel := context.WithTimeout(ctx, time.Second*220) wgDbPing := pool.New().WithErrors().WithContext(dbPingTimeout) defer cancel() @@ -313,7 +372,7 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers // Initialize the databases by running migrator with the `up` command. test.LogLines = append(test.LogLines, "-- 🏗️ initializing database schemas with migrator") - out, err = run.Cmd(ctx, dockerMigratorBaseString(test, "up", fmt.Sprintf("sourcegraph/migrator:%s", initVersion), networkName, dbs)...).Run().String() + out, err = run.Cmd(ctx, dockerMigratorBaseString(test, "up", fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), initVersion), networkName, dbs)...).Run().String() if err != nil { test.AddError(errors.Newf("🚨 failed to initialize database: %w", err)) } @@ -346,7 +405,7 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers //start frontend and poll db until initial version is set by frontend var cleanFrontend func() - cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", initVersion.String(), networkName, false, dbs) + cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(fromRegistryKey{})), initVersion.String(), networkName, false, dbs) if err != nil { test.AddError(errors.Newf("🚨 failed to start frontend: %w", err)) } @@ -517,8 +576,7 @@ func startFrontend(ctx context.Context, test Test, image, version, networkName s if auto { envString = append(envString, "-e", "SRC_AUTOUPGRADE=true") } - // ERROR - // {"SeverityText":"FATAL","Timestamp":1706224238009644720,"InstrumentationScope":"sourcegraph","Caller":"svcmain/svcmain.go:167","Function":"github.com/sourcegraph/sourcegraph/internal/service/svcmain.run.func1","Body":"failed to start service","Resource":{"service.name":"frontend","service.version":"0.0.0+dev","service.instance.id":"79a3e3ca0bfc"},"Attributes":{"service":"frontend","error":"failed to connect to frontend database: database schema out of date"}} + cmdString := []string{ "--network", networkName, fmt.Sprintf("%s:%s", image, version), @@ -526,6 +584,7 @@ func startFrontend(ctx context.Context, test Test, image, version, networkName s baseString = append(baseString, envString...) cmdString = append(baseString, cmdString...) + // TODO: Improve log aggregation of frontend container runs // Start the frontend container in goroutine to get logs errChan := make(chan error) go func() { @@ -703,6 +762,12 @@ func handleVersions(cCtx *cli.Context, overrideStd, overrideMVU, overrideAuto [] targetVersion = semver.MustParse("0.0.0+dev") // If no stamp version is set, we assume version is in dev } + // Ensure latest tags + err = run.Cmd(ctx, "git", "fetch", "--tags").Run().Wait() + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + // Sort latest stable release tags tags, err := run.Cmd(ctx, "git", "for-each-ref", diff --git a/testing/tools/upgradetest/tests.go b/testing/tools/upgradetest/tests.go index 6d107c5d0ce..588c9a84bee 100644 --- a/testing/tools/upgradetest/tests.go +++ b/testing/tools/upgradetest/tests.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "strings" "github.com/Masterminds/semver" @@ -14,7 +15,7 @@ import ( // standardUpgradeTest initializes Sourcegraph's dbs and runs a standard upgrade // i.e. an upgrade test between some last minor version and the current release candidate func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStableVersion *semver.Version) Test { - postRelease := ctx.Value(postReleaseKey{}).(string) + postRelease := strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v") // Post release version string //start test env test, networkName, dbs, cleanup, err := setupTestEnv(ctx, "standard", initVersion) @@ -28,9 +29,9 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest // Use the latest stable migrator for a pre release test, and the target version migrator if testing a released version var migratorImage string if postRelease != "" { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease) } else { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", latestStableVersion.String()) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), latestStableVersion.String()) } // ensure env correctly initialized @@ -42,7 +43,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest test.AddLog("-- ⚙️ performing standard upgrade") if postRelease != "" { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease) } else { migratorImage = "migrator:candidate" } @@ -58,7 +59,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest // Start frontend with candidate var cleanFrontend func() if postRelease != "" { - cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", postRelease, networkName, false, dbs) + cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(targetRegistryKey{})), postRelease, networkName, false, dbs) } else { cleanFrontend, err = startFrontend(ctx, test, "frontend", "candidate", networkName, false, dbs) } @@ -72,7 +73,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest test.AddLog("-- ⚙️ post upgrade validation") // Validate the upgrade if postRelease != "" { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease) } else { migratorImage = "migrator:candidate" } @@ -87,7 +88,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest // multiversionUpgradeTest tests the migrator upgrade command, // initializing the three main dbs and conducting an upgrade to the release candidate version func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStableVersion *semver.Version) Test { - postRelease := ctx.Value(postReleaseKey{}).(string) // Post release version string + postRelease := strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v") // Post release version string //start test env test, networkName, dbs, cleanup, err := setupTestEnv(ctx, "multiversion", initVersion) @@ -101,9 +102,9 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la // Use the latest stable migrator for a pre release test, and the target version migrator if testing a released version var migratorImage string if postRelease != "" { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease) } else { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", latestStableVersion.String()) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), latestStableVersion.String()) } // ensure env correctly initialized @@ -124,7 +125,7 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la } test.AddLog(fmt.Sprintf("-- ⚙️ performing multiversion upgrade (--from %s --to %s)", initVersion.String(), toVersion)) if postRelease != "" { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease) } else { migratorImage = "migrator:candidate" } @@ -152,7 +153,7 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la // Start frontend with candidate unless a post release version is specified var cleanFrontend func() if postRelease != "" { - cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", postRelease, networkName, false, dbs) + cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(targetRegistryKey{})), postRelease, networkName, false, dbs) } else { cleanFrontend, err = startFrontend(ctx, test, "frontend", "candidate", networkName, false, dbs) } @@ -178,7 +179,7 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la // Without this in place autoupgrade fails and exits while trying to make an oobmigration comparison here: https://sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/internal/cli/autoupgrade.go?L67-76 // {"SeverityText":"WARN","Timestamp":1706721478276103721,"InstrumentationScope":"frontend","Caller":"cli/autoupgrade.go:73","Function":"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/cli.tryAutoUpgrade","Body":"unexpected string for desired instance schema version, skipping auto-upgrade","Resource":{"service.name":"frontend","service.version":"devVersion","service.instance.id":"487754e1c54a"},"Attributes":{"version":"devVersion"}} func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStableVersion *semver.Version) Test { - postRelease := ctx.Value(postReleaseKey{}).(string) // Post release version string + postRelease := strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v") // Post release version string //start test env test, networkName, dbs, cleanup, err := setupTestEnv(ctx, "auto", initVersion) @@ -192,9 +193,9 @@ func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStab // Use the latest stable migrator for a pre release test, and the target version migrator if testing a released version var migratorImage string if postRelease != "" { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease) } else { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", latestStableVersion.String()) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), latestStableVersion.String()) } // ensure env correctly initialized @@ -210,7 +211,7 @@ func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStab // Start frontend with candidate var cleanFrontend func() if postRelease != "" { - cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", postRelease, networkName, true, dbs) + cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(targetRegistryKey{})), postRelease, networkName, true, dbs) } else { cleanFrontend, err = startFrontend(ctx, test, "frontend", "candidate", networkName, true, dbs) } @@ -224,7 +225,7 @@ func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStab test.AddLog("-- ⚙️ post upgrade validation") // Validate the upgrade if postRelease != "" { - migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease) + migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease) } else { migratorImage = "migrator:candidate" }