feat(rel): plug upgradetest into release tests (#61563)

---------

Co-authored-by: Warren Gifford <warrenbruceg@gmail.com>

release.yaml

@@ -1,8 +1,8 @@
 meta:
   productName: sourcegraph
-  repository: "github.com/sourcegraph/sourcegraph"
+  repository: 'github.com/sourcegraph/sourcegraph'
   owners:
-    - "@sourcegraph/release"
+    - '@sourcegraph/release'
 
 requirements:
   - name: "curl"
@@ -17,14 +17,14 @@ requirements:
 
   # #bolaji-release-testing
   # https://start.1password.com/open/i?a=HEDEDSLHPBFGRBTKAKJWE23XX4&v=isv4abynddpox72wwbhaamo76e&i=7zjax5rm5hlilbgrzeb257i62i&h=team-sourcegraph.1password.com
-  - name: "Slack Webhook URL"
+  - name: 'Slack Webhook URL'
     env: SLACK_WEBHOOK_URL
 
 internal:
   create:
     steps:
       patch:
-        - name: "buildkite"
+        - name: 'buildkite'
           cmd: |
             echo "Triggering build on sourcegraph/sourcegraph with VERSION={{version}} on branch {{git.branch}}"
             body=$(curl -s --fail-with-body -X POST "https://api.buildkite.com/v2/organizations/sourcegraph/pipelines/sourcegraph/builds" -H "Content-Type: application/json" -H "Authorization: Bearer $BUILDKITE_ACCESS_TOKEN" -d '{
@@ -50,10 +50,10 @@ internal:
             fi
   finalize:
     steps:
-      - name: "Register on release registry"
+      - name: 'Register on release registry'
         cmd: |
           echo "pretending to call release registry api"
-      - name: "notifications"
+      - name: 'notifications'
         cmd: |
           set -eu
 
@@ -64,14 +64,14 @@ internal:
 
 test:
   steps:
-    - name: "check:git tag does not exist"
+    - name: 'check:git tag does not exist'
       cmd: |
         tags=$(git ls-remote --tags origin | sort -t '/' -k 3 | cut -f 2 | awk -F '/' '{gsub(/\^\{\}$/, "", $3); print $3}' | uniq)
         if echo "${tags}" | grep -q "^{{version}}$"; then
           echo "❌ Tag '{{version}}' already exists"
           exit 1
         fi
-    - name: "check:migrator-schemas"
+    - name: 'check:migrator-schemas'
       cmd: |
         set -eu
 
@@ -107,11 +107,35 @@ test:
           echo "migrator:{{tag}} does not contain the correct amount of schema description files for this release - expected more than 300 got ${count}"
           exit 1
         fi
+    - name: 'db:migration:coherence_test'
+      cmd: |
+        set -eu
+
+        aspectRC="/tmp/aspect-generated.bazelrc"
+        rosetta bazelrc > "$aspectRC"
+        bazelrc=(--bazelrc="$aspectRC" --bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc)
+
+        # The upgradetest are inferring the stamp-version flag based on the version, so we need to unset it here.
+        _VERSION=$VERSION
+        unset VERSION
+
+        # We purposely limit the concurrency to 6, because if we use the default, there are
+        # scenarios where we can exhaust available ports to the docker daemon and end up
+        # with an infrastructure flake.
+        max_routines=6
+        # Hardcoding version, as for now I just want to make sure this works in CI.
+        bazel ${bazelrc[*]} run //testing/tools/upgradetest:release_test_run -- all \
+          --post-release-version={{tag}} \
+          --target-registry us-central1-docker.pkg.dev/sourcegraph-ci/rfc795-internal/ \
+          --max-routines $max_routines
+
+        # Restoring it to avoid creating a footgun if we add more test steps later on.
+        VERSION=$_VERSION
 
 promoteToPublic:
   create:
     steps:
-      - name: "buildkite"
+      - name: 'buildkite'
         cmd: |
           # We set DISABLE_ASPECT_WORKFLOWS to true, because the promotion is purely about retagging images
           # and we don't rely on AW at all.
@@ -140,10 +164,10 @@ promoteToPublic:
           fi
   finalize:
     steps:
-      - name: "Promote on release registry"
+      - name: 'Promote on release registry'
         cmd: |
           echo "pretending to call release registry api"
-      - name: "git:tag"
+      - name: 'git:tag'
         cmd: |
           set -eu
 
@@ -159,7 +183,7 @@ promoteToPublic:
           EOF
       # tag is usually in the format `5.3.2`
       # while version is usually the tag prepended with a v, `v5.3.2`
-      - name: "Slack notification (#announce-engineering)"
+      - name: 'Slack notification (#announce-engineering)'
         cmd: |
           echo "Posting slack notification for release"
           tag="{{tag}}"

testing/tools/upgradetest/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_cross_binary", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 
 go_library(
     name = "upgradetest_lib",
@@ -68,3 +68,14 @@ sh_binary(
         "//internal/database:generate_schemas",
     ],
 )
+
+sh_binary(
+    name = "release_test_run",
+    srcs = ["release_test.sh"],
+    args = [
+        "$(location :go_upgradetest)",
+    ],
+    data = [
+        ":go_upgradetest",
+    ],
+)

testing/tools/upgradetest/main.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime"
+	"strings"
 	"time"
 
 	_ "github.com/lib/pq"
@@ -14,13 +16,15 @@ import (
 	"github.com/sourcegraph/run"
 )
 
-// These commands are meant to be executed with a VERSION env var with a hypothetical stamped release version
-// This type is used to assign the stamp version from VERSION
 type stampVersionKey struct{}
 type postReleaseKey struct{}
+type targetRegistryKey struct{}
+type fromRegistryKey struct{}
 
 // Register upgrade commands -- see README.md for more details.
 func main() {
+	fmt.Println("👉 Upgrade test ...")
+
 	app := &cli.App{
 		Name:  "upgrade-test",
 		Usage: "Upgrade test is a tool for testing the migrator services creation of upgrade paths and application of upgrade paths.\nWhen run relevant upgrade paths are tested for each version relevant to a given upgrade type, initializing Sourcegraph databases and frontend services for each version, and attempting to generate and apply an upgrade path to your current branches head.",
@@ -41,11 +45,21 @@ func main() {
 						Aliases: []string{"pv"},
 						Usage:   "Select an already released version as the target version for the test suite.",
 					},
+					&cli.StringFlag{
+						Name:  "target-registry",
+						Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
+					&cli.StringFlag{
+						Name:  "from-registry",
+						Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
 					&cli.IntFlag{
 						Name:    "max-routines",
 						Aliases: []string{"mr"},
-						Usage:   "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.",
-						Value:   10,
+						Usage:   "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to CPU cores count minus two.",
+						Value:   runtime.NumCPU() - 2,
 					},
 					&cli.StringSliceFlag{
 						Name:    "standard-versions",
@@ -66,6 +80,8 @@ func main() {
 				Action: func(cCtx *cli.Context) error {
 					ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version"))
 					ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version"))
+					ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry"))
+					ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry"))
 
 					// check docker is running
 					if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil {
@@ -74,7 +90,12 @@ func main() {
 					}
 
 					// Get init versions to use for initializing upgrade environments for tests
-					latestMinorVersion, latestStableVersion, targetVersion, stdVersions, mvuVersions, autoVersions, err := handleVersions(cCtx, cCtx.StringSlice("standard-versions"), cCtx.StringSlice("mvu-versions"), cCtx.StringSlice("auto-versions"), cCtx.String("post-release-version"))
+					latestMinorVersion, latestStableVersion, targetVersion, stdVersions, mvuVersions, autoVersions, err := handleVersions(cCtx,
+						cCtx.StringSlice("standard-versions"),
+						cCtx.StringSlice("mvu-versions"),
+						cCtx.StringSlice("auto-versions"),
+						cCtx.String("post-release-version"),
+					)
 					if err != nil {
 						fmt.Println("🚨 Error: failed to get test version ranges: ", err)
 						os.Exit(1)
@@ -83,7 +104,7 @@ func main() {
 					var targetMigratorImage string
 					switch {
 					case ctx.Value(postReleaseKey{}) != "":
-						targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{}))
+						targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v"))
 					case ctx.Value(stampVersionKey{}) != "":
 						targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{}))
 					default:
@@ -168,6 +189,10 @@ func main() {
 					// This is where we do the majority of our printing to stdout.
 					results.OrderByVersion()
 					results.PrintSimpleResults()
+					if results.Failed() {
+						results.DisplayErrors()
+						os.Exit(1)
+					}
 
 					return nil
 				},
@@ -188,10 +213,20 @@ func main() {
 						Aliases: []string{"pv"},
 						Usage:   "Select an already released version as the target version for the test suite.",
 					},
+					&cli.StringFlag{
+						Name:  "target-registry",
+						Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
+					&cli.StringFlag{
+						Name:  "from-registry",
+						Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
 					&cli.IntFlag{
 						Name:    "max-routines",
 						Aliases: []string{"mr"}, Usage: "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.",
-						Value: 10,
+						Value: runtime.NumCPU() - 2,
 					},
 					&cli.StringSliceFlag{
 						Name:    "standard-versions",
@@ -202,6 +237,8 @@ func main() {
 				Action: func(cCtx *cli.Context) error {
 					ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version"))
 					ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version"))
+					ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry"))
+					ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry"))
 
 					// check docker is running
 					if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil {
@@ -219,7 +256,7 @@ func main() {
 					var targetMigratorImage string
 					switch {
 					case ctx.Value(postReleaseKey{}) != "":
-						targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{}))
+						targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v"))
 					case ctx.Value(stampVersionKey{}) != "":
 						targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{}))
 					default:
@@ -250,17 +287,30 @@ func main() {
 							result := standardUpgradeTest(ctx, version, targetVersion, latestStableVersion)
 							result.Runtime = time.Since(start)
 							results.AddStdTest(result)
+							if len(result.Errors) > 0 {
+								return result.Errors[0]
+							}
 							return nil
 						})
 					}
 					if err := stdTestPool.Wait(); err != nil {
 						fmt.Println("🚨 Error: failed to run tests in pool: ", err)
+						for _, t := range results.StandardUpgradeTests {
+							fmt.Println("LOGS")
+							t.DisplayLog()
+							fmt.Println("ERROR")
+							t.DisplayErrors()
+						}
 						return err
 					}
 
 					// This is where we do the majority of our printing to stdout.
 					results.OrderByVersion()
 					results.PrintSimpleResults()
+					if results.Failed() {
+						results.DisplayErrors()
+						os.Exit(1)
+					}
 
 					return nil
 				},
@@ -281,11 +331,21 @@ func main() {
 						Aliases: []string{"pv"},
 						Usage:   "Select an already released version as the target version for the test suite.",
 					},
+					&cli.StringFlag{
+						Name:  "target-registry",
+						Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
+					&cli.StringFlag{
+						Name:  "from-registry",
+						Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
 					&cli.IntFlag{
 						Name:    "max-routines",
 						Aliases: []string{"mr"},
 						Usage:   "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.",
-						Value:   10,
+						Value:   runtime.NumCPU() - 2,
 					},
 					&cli.StringSliceFlag{
 						Name:    "mvu-versions",
@@ -296,6 +356,8 @@ func main() {
 				Action: func(cCtx *cli.Context) error {
 					ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version"))
 					ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version"))
+					ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry"))
+					ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry"))
 
 					// check docker is running
 					if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil {
@@ -313,7 +375,7 @@ func main() {
 					var targetMigratorImage string
 					switch {
 					case ctx.Value(postReleaseKey{}) != "":
-						targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{}))
+						targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v"))
 					case ctx.Value(stampVersionKey{}) != "":
 						targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{}))
 					default:
@@ -342,6 +404,9 @@ func main() {
 							result := multiversionUpgradeTest(ctx, version, targetVersion, latestStableVersion)
 							result.Runtime = time.Since(start)
 							results.AddMVUTest(result)
+							if len(result.Errors) > 0 {
+								return result.Errors[0]
+							}
 							return nil
 						})
 					}
@@ -352,6 +417,10 @@ func main() {
 
 					results.OrderByVersion()
 					results.PrintSimpleResults()
+					if results.Failed() {
+						results.DisplayErrors()
+						os.Exit(1)
+					}
 
 					return nil
 				},
@@ -372,11 +441,22 @@ func main() {
 						Aliases: []string{"pv"},
 						Usage:   "Select an already released version as the target version for the test suite.",
 					},
+
+					&cli.StringFlag{
+						Name:  "target-registry",
+						Usage: "Registry host and path to pull the targeted version from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
+					&cli.StringFlag{
+						Name:  "from-registry",
+						Usage: "Registry host and path to pull versions we're upgrading from, i.e. index.docker.io/sourcegraph will pull index.docker.io/sourcegraph/migrator:<tag>",
+						Value: "sourcegraph/",
+					},
 					&cli.IntFlag{
 						Name:    "max-routines",
 						Aliases: []string{"mr"},
 						Usage:   "Maximum number of tests to run concurrently. Sets goroutine pool limit.\n Defaults to 10.",
-						Value:   10,
+						Value:   runtime.NumCPU() - 2,
 					},
 					&cli.StringSliceFlag{
 						Name:    "auto-versions",
@@ -387,6 +467,8 @@ func main() {
 				Action: func(cCtx *cli.Context) error {
 					ctx := context.WithValue(cCtx.Context, stampVersionKey{}, cCtx.String("stamp-version"))
 					ctx = context.WithValue(ctx, postReleaseKey{}, cCtx.String("post-release-version"))
+					ctx = context.WithValue(ctx, targetRegistryKey{}, cCtx.String("target-registry"))
+					ctx = context.WithValue(ctx, fromRegistryKey{}, cCtx.String("from-registry"))
 
 					// check docker is running
 					if err := run.Cmd(ctx, "docker", "ps").Run().Wait(); err != nil {
@@ -404,7 +486,7 @@ func main() {
 					var targetMigratorImage string
 					switch {
 					case ctx.Value(postReleaseKey{}) != "":
-						targetMigratorImage = fmt.Sprintf("sourcegraph/migrator:%s", ctx.Value(postReleaseKey{}))
+						targetMigratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v"))
 					case ctx.Value(stampVersionKey{}) != "":
 						targetMigratorImage = fmt.Sprintf("migrator:candidate stamped as %s", ctx.Value(stampVersionKey{}))
 					default:
@@ -433,6 +515,10 @@ func main() {
 							result := autoUpgradeTest(ctx, version, targetVersion, latestStableVersion)
 							result.Runtime = time.Since(start)
 							results.AddAutoTest(result)
+							if len(result.Errors) > 0 {
+								return result.Errors[0]
+							}
+
 							return nil
 						})
 					}
@@ -443,6 +529,10 @@ func main() {
 
 					results.OrderByVersion()
 					results.PrintSimpleResults()
+					if results.Failed() {
+						results.DisplayErrors()
+						os.Exit(1)
+					}
 
 					return nil
 				},
@@ -452,6 +542,7 @@ func main() {
 
 	if err := app.Run(os.Args); err != nil {
 		fmt.Println("🚨 Error: failed to run tests: ", err)
+		os.Exit(1)
 	}
 
 }

testing/tools/upgradetest/release_test.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -e
+
+RUNNER="$1"
+
+# internal/database/. artifacts are being loaded as arguments, 13 is the beginning of passed arguments to the cli tool
+# Args:
+# bazel-bin/testing/tools/upgradetest/sh_upgradetest_run
+# testing/tools/upgradetest/upgradetest-darwin-arm64
+# cmd/migrator/image_tarball.sh
+# cmd/frontend/image_tarball.sh
+# internal/database/_codeinsights_squashed.sql
+# internal/database/_codeintel_squashed.sql
+# internal/database/_frontend_squashed.sql
+# internal/database/_schema.codeinsights.json
+# internal/database/_schema.codeinsights.md
+# internal/database/_schema.codeintel.json
+# internal/database/_schema.codeintel.md
+# internal/database/_schema.json
+# internal/database/_schema.md
+"$RUNNER" "${@:2}"

testing/tools/upgradetest/test.sh

@@ -27,5 +27,3 @@ FRONTEND_TARBALL="$3"
 # internal/database/_schema.json
 # internal/database/_schema.md
 "$RUNNER" "${@:13}"
-
-exit

testing/tools/upgradetest/tests-util.go

@@ -43,7 +43,7 @@ func (t *Test) AddError(err error) {
 // DisplayErrors prints errors to stdout
 func (t *Test) DisplayErrors() {
 	for _, err := range t.Errors {
-		fmt.Println(err)
+		fmt.Println(err.Error())
 	}
 }
 
@@ -54,6 +54,11 @@ func (t *Test) DisplayLog() {
 	}
 }
 
+// Display if a test has failed
+func (t *Test) Failed() bool {
+	return 0 < len(t.Errors)
+}
+
 // TestResults is a collection of tests, organized by type. Its methods are generally used to control its logging behavior.
 type TestResults struct {
 	StandardUpgradeTests []Test
@@ -83,6 +88,32 @@ func (r *TestResults) AddAutoTest(test Test) {
 	r.AutoupgradeTests = append(r.AutoupgradeTests, test)
 }
 
+// Failed returns true if any given test has errors registered.
+func (r *TestResults) Failed() bool {
+	if 0 < len(r.StandardUpgradeTests) {
+		for _, test := range r.StandardUpgradeTests {
+			if test.Failed() {
+				return true
+			}
+		}
+	}
+	if 0 < len(r.MVUUpgradeTests) {
+		for _, test := range r.MVUUpgradeTests {
+			if test.Failed() {
+				return true
+			}
+		}
+	}
+	if 0 < len(r.AutoupgradeTests) {
+		for _, test := range r.AutoupgradeTests {
+			if test.Failed() {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // Used in all-type test
 type typeVersion struct {
 	Type    string
@@ -118,7 +149,7 @@ func (r *TestResults) PrintSimpleResults() {
 	if len(r.StandardUpgradeTests) != 0 {
 		stdRes := []string{}
 		for _, test := range r.StandardUpgradeTests {
-			if 0 < len(test.Errors) {
+			if test.Failed() {
 				stdRes = append(stdRes, fmt.Sprintf("🚨 %s Failed -- %s\n%s", test.Version.String(), test.Runtime, test.Errors[len(test.Errors)-1]))
 			} else {
 				stdRes = append(stdRes, fmt.Sprintf("✅ %s Passed -- %s ", test.Version.String(), test.Runtime))
@@ -130,7 +161,7 @@ func (r *TestResults) PrintSimpleResults() {
 	if len(r.MVUUpgradeTests) != 0 {
 		mvuRes := []string{}
 		for _, test := range r.MVUUpgradeTests {
-			if 0 < len(test.Errors) {
+			if test.Failed() {
 				mvuRes = append(mvuRes, fmt.Sprintf("🚨 %s Failed -- %s\n%s", test.Version.String(), test.Runtime, test.Errors[len(test.Errors)-1]))
 			} else {
 				mvuRes = append(mvuRes, fmt.Sprintf("✅ %s Passed -- %s", test.Version.String(), test.Runtime))
@@ -142,7 +173,7 @@ func (r *TestResults) PrintSimpleResults() {
 	if len(r.AutoupgradeTests) != 0 {
 		autoRes := []string{}
 		for _, test := range r.AutoupgradeTests {
-			if 0 < len(test.Errors) {
+			if test.Failed() {
 				autoRes = append(autoRes, fmt.Sprintf("🚨 %s Failed -- %s\n%s", test.Version.String(), test.Runtime, test.Errors[len(test.Errors)-1]))
 			} else {
 				autoRes = append(autoRes, fmt.Sprintf("✅ %s Passed -- %s", test.Version.String(), test.Runtime))
@@ -158,21 +189,21 @@ func (r *TestResults) DisplayErrors() {
 	r.Mutex.Lock()
 	defer r.Mutex.Unlock()
 	for _, test := range r.StandardUpgradeTests {
-		if 0 < len(test.Errors) {
+		if test.Failed() {
 			fmt.Printf("--- 🚨 Standard Upgrade Test %s Failed:\n", test.Version.String())
-			test.DisplayErrors()
+			test.DisplayLog()
 		}
 	}
 	for _, test := range r.MVUUpgradeTests {
-		if 0 < len(test.Errors) {
+		if test.Failed() {
 			fmt.Printf("--- 🚨 Multiversion Upgrade Test %s Failed:\n", test.Version.String())
-			test.DisplayErrors()
+			test.DisplayLog()
 		}
 	}
 	for _, test := range r.AutoupgradeTests {
-		if 0 < len(test.Errors) {
+		if test.Failed() {
 			fmt.Printf("--- 🚨 Auto Upgrade Test %s Failed:\n", test.Version.String())
-			test.DisplayErrors()
+			test.DisplayLog()
 		}
 	}
 }
@@ -221,6 +252,23 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers
 	test.AddLog(fmt.Sprintf("Upgrading from version (%s) to release candidate.", initVersion))
 	test.AddLog("-- 🏗️  setting up test environment")
 
+	// Pull images from registry if -target-registry is set
+	if ctx.Value(fromRegistryKey{}).(string) != "sourcegraph/" {
+		test.AddLog(fmt.Sprintf("🐋 pulling -target-registry images from %s", ctx.Value(fromRegistryKey{}).(string)))
+		out, err := run.Cmd(ctx, "docker", "image", "pull", fmt.Sprintf("%sfrontend:%s", ctx.Value(fromRegistryKey{}).(string), initVersion.String())).Run().String()
+		test.AddLog(out)
+		if err != nil {
+			test.AddError(errors.Newf("🚨 failed to pull images from -target-registry: %s", err))
+		}
+		fmt.Println(out)
+		out, err = run.Cmd(ctx, "docker", "image", "pull", fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}).(string), initVersion.String())).Run().String()
+		test.AddLog(out)
+		if err != nil {
+			test.AddError(errors.Newf("🚨 failed to pull images from -target-registry: %s", err))
+		}
+		fmt.Println(out)
+	}
+
 	// Create a docker network for testing
 	//
 	// Docker bridge networks take up a lot of the docker daemons available port allocation. We run only a limited amount of test parallelization to get around this.
@@ -257,13 +305,24 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers
 			"--name", db.ContainerName,
 			"--network", networkName,
 			"-p", "5432",
-			fmt.Sprintf("sourcegraph/%s:%s", db.Image, initVersion),
+			fmt.Sprintf("%s%s:%s", ctx.Value(fromRegistryKey{}), db.Image, initVersion),
 		).Run().Wait()
 		if err != nil {
 			test.AddError(errors.Newf("🚨 failed to create test databases: %s", err))
 		}
+
 		// get the dynamically allocated port and register it to the test
-		port, err := run.Cmd(ctx, "docker", "port", db.ContainerName, "5432").Run().String()
+		out, err := run.Cmd(ctx, "docker", "port", db.ContainerName, "5432").Run().String()
+
+		// docker port can return multiple ports, ipv4 and ipv6, so we need to keep the former only.
+		ports := strings.Split(out, "\n")
+		var port string
+		if len(ports) < 1 {
+			test.AddError(errors.Newf("incorrect port output for %s", db.ContainerName))
+		} else {
+			port = ports[0]
+		}
+
 		if err != nil {
 			test.AddError(errors.Newf("🚨 failed to get port for %s: %s", db.ContainerName, err))
 		}
@@ -272,7 +331,7 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers
 
 	// Create a timeout to validate the databases have initialized, this is to prevent a hung test
 	// When many goroutines are running this test this is a point of failure.
-	dbPingTimeout, cancel := context.WithTimeout(ctx, time.Second*120)
+	dbPingTimeout, cancel := context.WithTimeout(ctx, time.Second*220)
 	wgDbPing := pool.New().WithErrors().WithContext(dbPingTimeout)
 	defer cancel()
 
@@ -313,7 +372,7 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers
 
 	// Initialize the databases by running migrator with the `up` command.
 	test.LogLines = append(test.LogLines, "-- 🏗️  initializing database schemas with migrator")
-	out, err = run.Cmd(ctx, dockerMigratorBaseString(test, "up", fmt.Sprintf("sourcegraph/migrator:%s", initVersion), networkName, dbs)...).Run().String()
+	out, err = run.Cmd(ctx, dockerMigratorBaseString(test, "up", fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), initVersion), networkName, dbs)...).Run().String()
 	if err != nil {
 		test.AddError(errors.Newf("🚨 failed to initialize database: %w", err))
 	}
@@ -346,7 +405,7 @@ func setupTestEnv(ctx context.Context, testType string, initVersion *semver.Vers
 
 	//start frontend and poll db until initial version is set by frontend
 	var cleanFrontend func()
-	cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", initVersion.String(), networkName, false, dbs)
+	cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(fromRegistryKey{})), initVersion.String(), networkName, false, dbs)
 	if err != nil {
 		test.AddError(errors.Newf("🚨 failed to start frontend: %w", err))
 	}
@@ -517,8 +576,7 @@ func startFrontend(ctx context.Context, test Test, image, version, networkName s
 	if auto {
 		envString = append(envString, "-e", "SRC_AUTOUPGRADE=true")
 	}
-	// ERROR
-	// {"SeverityText":"FATAL","Timestamp":1706224238009644720,"InstrumentationScope":"sourcegraph","Caller":"svcmain/svcmain.go:167","Function":"github.com/sourcegraph/sourcegraph/internal/service/svcmain.run.func1","Body":"failed to start service","Resource":{"service.name":"frontend","service.version":"0.0.0+dev","service.instance.id":"79a3e3ca0bfc"},"Attributes":{"service":"frontend","error":"failed to connect to frontend database: database schema out of date"}}
+
 	cmdString := []string{
 		"--network", networkName,
 		fmt.Sprintf("%s:%s", image, version),
@@ -526,6 +584,7 @@ func startFrontend(ctx context.Context, test Test, image, version, networkName s
 	baseString = append(baseString, envString...)
 	cmdString = append(baseString, cmdString...)
 
+	// TODO: Improve log aggregation of frontend container runs
 	// Start the frontend container in goroutine to get logs
 	errChan := make(chan error)
 	go func() {
@@ -703,6 +762,12 @@ func handleVersions(cCtx *cli.Context, overrideStd, overrideMVU, overrideAuto []
 		targetVersion = semver.MustParse("0.0.0+dev") // If no stamp version is set, we assume version is in dev
 	}
 
+	// Ensure latest tags
+	err = run.Cmd(ctx, "git", "fetch", "--tags").Run().Wait()
+	if err != nil {
+		return nil, nil, nil, nil, nil, nil, err
+	}
+
 	// Sort latest stable release tags
 	tags, err := run.Cmd(ctx, "git",
 		"for-each-ref",

testing/tools/upgradetest/tests.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/Masterminds/semver"
 
@@ -14,7 +15,7 @@ import (
 // standardUpgradeTest initializes Sourcegraph's dbs and runs a standard upgrade
 // i.e. an upgrade test between some last minor version and the current release candidate
 func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStableVersion *semver.Version) Test {
-	postRelease := ctx.Value(postReleaseKey{}).(string)
+	postRelease := strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v") // Post release version string
 
 	//start test env
 	test, networkName, dbs, cleanup, err := setupTestEnv(ctx, "standard", initVersion)
@@ -28,9 +29,9 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest
 	// Use the latest stable migrator for a pre release test, and the target version migrator if testing a released version
 	var migratorImage string
 	if postRelease != "" {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease)
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease)
 	} else {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", latestStableVersion.String())
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), latestStableVersion.String())
 	}
 
 	// ensure env correctly initialized
@@ -42,7 +43,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest
 	test.AddLog("-- ⚙️  performing standard upgrade")
 
 	if postRelease != "" {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease)
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease)
 	} else {
 		migratorImage = "migrator:candidate"
 	}
@@ -58,7 +59,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest
 	// Start frontend with candidate
 	var cleanFrontend func()
 	if postRelease != "" {
-		cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", postRelease, networkName, false, dbs)
+		cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(targetRegistryKey{})), postRelease, networkName, false, dbs)
 	} else {
 		cleanFrontend, err = startFrontend(ctx, test, "frontend", "candidate", networkName, false, dbs)
 	}
@@ -72,7 +73,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest
 	test.AddLog("-- ⚙️  post upgrade validation")
 	// Validate the upgrade
 	if postRelease != "" {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease)
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease)
 	} else {
 		migratorImage = "migrator:candidate"
 	}
@@ -87,7 +88,7 @@ func standardUpgradeTest(ctx context.Context, initVersion, targetVersion, latest
 // multiversionUpgradeTest tests the migrator upgrade command,
 // initializing the three main dbs and conducting an upgrade to the release candidate version
 func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStableVersion *semver.Version) Test {
-	postRelease := ctx.Value(postReleaseKey{}).(string) // Post release version string
+	postRelease := strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v") // Post release version string
 
 	//start test env
 	test, networkName, dbs, cleanup, err := setupTestEnv(ctx, "multiversion", initVersion)
@@ -101,9 +102,9 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la
 	// Use the latest stable migrator for a pre release test, and the target version migrator if testing a released version
 	var migratorImage string
 	if postRelease != "" {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease)
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease)
 	} else {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", latestStableVersion.String())
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), latestStableVersion.String())
 	}
 
 	// ensure env correctly initialized
@@ -124,7 +125,7 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la
 	}
 	test.AddLog(fmt.Sprintf("-- ⚙️  performing multiversion upgrade (--from %s --to %s)", initVersion.String(), toVersion))
 	if postRelease != "" {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease)
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease)
 	} else {
 		migratorImage = "migrator:candidate"
 	}
@@ -152,7 +153,7 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la
 	// Start frontend with candidate unless a post release version is specified
 	var cleanFrontend func()
 	if postRelease != "" {
-		cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", postRelease, networkName, false, dbs)
+		cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(targetRegistryKey{})), postRelease, networkName, false, dbs)
 	} else {
 		cleanFrontend, err = startFrontend(ctx, test, "frontend", "candidate", networkName, false, dbs)
 	}
@@ -178,7 +179,7 @@ func multiversionUpgradeTest(ctx context.Context, initVersion, targetVersion, la
 // Without this in place autoupgrade fails and exits while trying to make an oobmigration comparison here: https://sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/internal/cli/autoupgrade.go?L67-76
 // {"SeverityText":"WARN","Timestamp":1706721478276103721,"InstrumentationScope":"frontend","Caller":"cli/autoupgrade.go:73","Function":"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/cli.tryAutoUpgrade","Body":"unexpected string for desired instance schema version, skipping auto-upgrade","Resource":{"service.name":"frontend","service.version":"devVersion","service.instance.id":"487754e1c54a"},"Attributes":{"version":"devVersion"}}
 func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStableVersion *semver.Version) Test {
-	postRelease := ctx.Value(postReleaseKey{}).(string) // Post release version string
+	postRelease := strings.TrimPrefix(ctx.Value(postReleaseKey{}).(string), "v") // Post release version string
 
 	//start test env
 	test, networkName, dbs, cleanup, err := setupTestEnv(ctx, "auto", initVersion)
@@ -192,9 +193,9 @@ func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStab
 	// Use the latest stable migrator for a pre release test, and the target version migrator if testing a released version
 	var migratorImage string
 	if postRelease != "" {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease)
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease)
 	} else {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", latestStableVersion.String())
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(fromRegistryKey{}), latestStableVersion.String())
 	}
 
 	// ensure env correctly initialized
@@ -210,7 +211,7 @@ func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStab
 	// Start frontend with candidate
 	var cleanFrontend func()
 	if postRelease != "" {
-		cleanFrontend, err = startFrontend(ctx, test, "sourcegraph/frontend", postRelease, networkName, true, dbs)
+		cleanFrontend, err = startFrontend(ctx, test, fmt.Sprintf("%sfrontend", ctx.Value(targetRegistryKey{})), postRelease, networkName, true, dbs)
 	} else {
 		cleanFrontend, err = startFrontend(ctx, test, "frontend", "candidate", networkName, true, dbs)
 	}
@@ -224,7 +225,7 @@ func autoUpgradeTest(ctx context.Context, initVersion, targetVersion, latestStab
 	test.AddLog("-- ⚙️  post upgrade validation")
 	// Validate the upgrade
 	if postRelease != "" {
-		migratorImage = fmt.Sprintf("sourcegraph/migrator:%s", postRelease)
+		migratorImage = fmt.Sprintf("%smigrator:%s", ctx.Value(targetRegistryKey{}), postRelease)
 	} else {
 		migratorImage = "migrator:candidate"
 	}