support single-program execution for local dev (#56750)

support single-program execution

Now, `sg start single-program` starts a single-binary local dev server. This is similar to Cody app, but instead of using a Tauri desktop app UI and limiting to only Cody-related functionality, it runs a full Sourcegraph instance and lets you access it through your web browser. It is useful for local dev because it's less resource-intensive and has faster recompile/relink times than `sg start` (which runs many processes).

cmd/executor/internal/run/util.go

@@ -40,7 +40,7 @@ func newQueueTelemetryOptions(ctx context.Context, runner util.CmdRunner, useFir
 		logger.Error("Failed to get git version", log.Error(err))
 	}
 
-	if !config.IsKubernetes() && (!deploy.IsApp() || deploy.IsAppFullSourcegraph()) {
+	if !config.IsKubernetes() && !deploy.IsApp() {
 		t.SrcCliVersion, err = util.GetSrcVersion(ctx, runner)
 		if err != nil {
 			logger.Error("Failed to get src-cli version", log.Error(err))

cmd/frontend/graphqlbackend/site_alerts.go

@@ -94,7 +94,7 @@ var disableSecurity, _ = strconv.ParseBool(env.Get("DISABLE_SECURITY", "false",
 
 func init() {
 	conf.ContributeWarning(func(c conftypes.SiteConfigQuerier) (problems conf.Problems) {
-		if deploy.IsDeployTypeSingleDockerContainer(deploy.Type()) || deploy.IsApp() {
+		if deploy.IsDeployTypeSingleDockerContainer(deploy.Type()) || deploy.IsSingleBinary() {
 			return nil
 		}
 		if c.SiteConfig().ExternalURL == "" {
@@ -267,7 +267,7 @@ func isMinorUpdateAvailable(currentVersion, updateVersion string) bool {
 }
 
 func emailSendingNotConfiguredAlert(args AlertFuncArgs) []*Alert {
-	if !args.IsSiteAdmin || deploy.IsDeployTypeSingleDockerContainer(deploy.Type()) || deploy.IsApp() {
+	if !args.IsSiteAdmin || deploy.IsDeployTypeSingleDockerContainer(deploy.Type()) || deploy.IsSingleBinary() {
 		return nil
 	}
 	if conf.Get().EmailSmtp == nil || conf.Get().EmailSmtp.Host == "" {

cmd/frontend/internal/app/app.go

@@ -25,7 +25,7 @@ import (
 // and sets the actor in the request context.
 func NewHandler(db database.DB, logger log.Logger, githubAppSetupHandler http.Handler) http.Handler {
 	session.SetSessionStore(session.NewRedisStore(func() bool {
-		if deploy.IsApp() {
+		if deploy.IsSingleBinary() {
 			// Safari / WebKit-based browsers refuse to set cookies on localhost as it is not treated
 			// as a secure domain, in contrast to all other browsers.
 			// https://bugs.webkit.org/show_bug.cgi?id=232088

cmd/frontend/internal/app/ui/help.go

@@ -27,7 +27,7 @@ func serveHelp(w http.ResponseWriter, r *http.Request) {
 	logger := sglog.Scoped("serveHelp", "")
 	logger.Info("redirecting to docs", sglog.String("page", page), sglog.String("versionStr", versionStr))
 
-	// For App, help links are handled in the frontend. We should never get here.
+	// For Cody App, help links are handled in the frontend. We should never get here.
 	if deploy.IsApp() {
 		// This should never happen, but if it does, we want to know about it.
 		logger.Error("help link was clicked in App and handled in the backend, this should never happer")

cmd/frontend/internal/cli/serve_cmd.go

@@ -105,7 +105,7 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
 	stdr.SetVerbosity(10)
 
 	if os.Getenv("SRC_DISABLE_OOBMIGRATION_VALIDATION") != "" {
-		if !deploy.IsApp() {
+		if !deploy.IsSingleBinary() {
 			logger.Warn("Skipping out-of-band migrations check")
 		}
 	} else {

cmd/frontend/internal/executorqueue/init.go

@@ -24,7 +24,7 @@ func Init(
 	batchesWorkspaceFileExistsHandler := enterpriseServices.BatchesChangesFileGetHandler
 
 	accessToken := func() string {
-		if deploy.IsApp() {
+		if deploy.IsSingleBinary() {
 			return confdefaults.AppInMemoryExecutorPassword
 		}
 		return conf.SiteConfig().ExecutorsAccessToken

cmd/sourcegraph/README.md

@@ -10,6 +10,8 @@ Cody App is a single-binary distribution of Sourcegraph that runs on your local
 sg start app
 ```
 
+(Or `sg start single-program` if you want all of Sourcegraph, not just Cody functionality.)
+
 If your are running app on a fresh database instance you also have to perform the following steps:
 
 - After opening the web app you will be directed to `/sign-in`, NOT the local repo setup step that is shown in production.

cmd/worker/internal/migrations/init.go

@@ -50,7 +50,7 @@ func (m *migrator) Routines(startupCtx context.Context, observationCtx *observat
 	}
 
 	if os.Getenv("SRC_DISABLE_OOBMIGRATION_VALIDATION") != "" {
-		if !deploy.IsApp() {
+		if !deploy.IsSingleBinary() {
 			observationCtx.Logger.Warn("Skipping out-of-band migrations check")
 		}
 	} else {

doc/dev/background-information/sg/reference.md

@@ -51,6 +51,7 @@ Available comamndsets in `sg.config.yaml`:
 * monitoring-alerts
 * otel
 * qdrant
+* single-program
 * web-standalone
 * web-standalone-prod
 
@@ -105,6 +106,7 @@ Available commands in `sg.config.yaml`:
 * codeintel-executor-firecracker
 * codeintel-executor-kubernetes
 * codeintel-worker
+* cody-app: Cody App
 * cody-gateway
 * debug-env: Debug env vars
 * docsite: Docsite instance serving the docs
@@ -130,7 +132,7 @@ Available commands in `sg.config.yaml`:
 * repo-updater
 * searcher
 * server: Run an all-in-one sourcegraph/server image
-* sourcegraph: Single program (Go static binary) distribution
+* sourcegraph: Single-program distribution
 * storybook
 * symbols
 * syntax-highlighter

internal/api/internalapi/client.go

@@ -38,7 +38,7 @@ var frontendInternal = func() *url.URL {
 var enableGRPC = env.MustGetBool("SRC_GRPC_ENABLE_CONF", false, "Enable gRPC for configuration updates")
 
 func defaultFrontendInternal() string {
-	if deploy.IsApp() {
+	if deploy.IsSingleBinary() {
 		return "localhost:3090"
 	}
 	return "sourcegraph-frontend-internal"

internal/conf/computed.go

@@ -40,13 +40,15 @@ func defaultConfigForDeployment() conftypes.RawUnified {
 		return confdefaults.KubernetesOrDockerComposeOrPureDocker
 	case deploy.IsDeployTypeApp(deployType):
 		return confdefaults.App
+	case deploy.IsDeployTypeSingleProgram(deployType):
+		return confdefaults.SingleProgram
 	default:
-		panic("deploy type did not register default configuration")
+		panic("deploy type did not register default configuration: " + deployType)
 	}
 }
 
 func ExecutorsAccessToken() string {
-	if deploy.IsApp() {
+	if deploy.IsSingleBinary() {
 		return confdefaults.AppInMemoryExecutorPassword
 	}
 	return Get().ExecutorsAccessToken

internal/conf/conf.go

@@ -67,8 +67,8 @@ func getMode() configurationMode {
 }
 
 func getModeUncached() configurationMode {
-	if deploy.IsApp() {
-		// App always uses the server mode because everything is running in the same process.
+	if deploy.IsSingleBinary() {
+		// When running everything in the same process, use server mode.
 		return modeServer
 	}
 
@@ -217,12 +217,12 @@ func startSiteConfigEscapeHatchWorker(c ConfigurationSource) {
 	}
 
 	siteConfigEscapeHatchPath = os.ExpandEnv(siteConfigEscapeHatchPath)
-	if deploy.IsApp() {
-		// App always store the site config on disk, and this is achieved through
+	if deploy.IsSingleBinary() {
+		// For single-binary mode, always store the site config on disk, and this is achieved through
 		// making the "escape hatch file" point to our desired location on disk.
-		// The concept of an escape hatch file is not something App users care
+		// The concept of an escape hatch file is not something users care
 		// about (it only makes sense in Docker/Kubernetes, e.g. to edit the config
-		// file if the sourcegraph-frontend container is crashing) - App runs
+		// file if the sourcegraph-frontend container is crashing) - it runs
 		// natively and this mechanism is just a convenient way for us to keep
 		// the file on disk as our source of truth.
 		siteConfigEscapeHatchPath = os.Getenv("SITE_CONFIG_FILE")

internal/conf/confdefaults/confdefaults.go

@@ -77,7 +77,7 @@ var KubernetesOrDockerComposeOrPureDocker = conftypes.RawUnified{
 // between the bundled executor and the publicly-facing executor API.
 var AppInMemoryExecutorPassword = uuid.NewV4().String()
 
-// App is the default configuration for the Cody app (which is also a single Go static binary.)
+// App is the default configuration for the Cody app.
 var App = conftypes.RawUnified{
 	Site: `{
 	"auth.providers": [
@@ -103,6 +103,28 @@ var App = conftypes.RawUnified{
 }`,
 }
 
+// SingleProgram is the default configuration for the single-program distribution.
+var SingleProgram = conftypes.RawUnified{
+	Site: `{
+	"auth.providers": [
+		{ "type": "builtin" }
+	],
+	"externalURL": "http://localhost:3080",
+	"codeIntelAutoIndexing.enabled": true,
+	"codeIntelAutoIndexing.allowGlobalPolicies": true,
+	"executors.frontendURL": "http://host.docker.internal:3080",
+	"cody.enabled": true,
+	"completions": {
+		"enabled": true,
+		"provider": "sourcegraph"
+	},
+	"embeddings": {
+		"enabled": true,
+		"provider": "sourcegraph"
+	}
+}`,
+}
+
 // Default is the default for *this* deployment type. It is populated by
 // pkg/conf at init time.
 var Default conftypes.RawUnified

internal/conf/deploy/deploytype.go

@@ -2,7 +2,6 @@ package deploy
 
 import (
 	"os"
-	"strconv"
 )
 
 // Deploy type constants. Any changes here should be reflected in the DeployType type declared in client/web/src/jscontext.ts:
@@ -16,6 +15,7 @@ const (
 	Helm          = "helm"
 	Kustomize     = "kustomize"
 	App           = "app"
+	SingleProgram = "single-program"
 	K3s           = "k3s"
 )
 
@@ -73,11 +73,16 @@ func IsDeployTypeSingleDockerContainer(deployType string) bool {
 	return deployType == SingleDocker
 }
 
-// IsDeployTypeSingleProgram tells if the given deployment type is a single Go program.
+// IsDeployTypeApp tells if the given deployment is Cody App.
 func IsDeployTypeApp(deployType string) bool {
 	return deployType == App
 }
 
+// IsDeployTypeSingleProgram tells if the given deployment is a single program.
+func IsDeployTypeSingleProgram(deployType string) bool {
+	return deployType == SingleProgram
+}
+
 // IsDev tells if the given deployment type is "dev".
 func IsDev(deployType string) bool {
 	return deployType == Dev
@@ -91,7 +96,8 @@ func IsValidDeployType(deployType string) bool {
 		IsDeployTypePureDocker(deployType) ||
 		IsDeployTypeSingleDockerContainer(deployType) ||
 		IsDev(deployType) ||
-		IsDeployTypeApp(deployType)
+		IsDeployTypeApp(deployType) ||
+		IsDeployTypeSingleProgram(deployType)
 }
 
 // IsApp tells if the running deployment is a Cody App deployment.
@@ -107,15 +113,6 @@ func IsApp() bool {
 	return Type() == App
 }
 
-// IsAppFullSourcegraph tells if the Cody app should run a full Sourcegraph instance (true),
-// or whether components not needed for the baseline Cody experience should be disabled
-// such as precise code intel, zoekt, etc.
-func IsAppFullSourcegraph() bool {
-	return IsApp() && appFullSourcegraph
-}
-
-var appFullSourcegraph, _ = strconv.ParseBool(os.Getenv("APP_FULL_SOURCEGRAPH"))
-
 // IsSingleBinary tells if the running deployment is a single-binary or not.
 //
 // Cody App is always a single-binary, but not all single-binary deployments are
@@ -126,7 +123,5 @@ var appFullSourcegraph, _ = strconv.ParseBool(os.Getenv("APP_FULL_SOURCEGRAPH"))
 // run in a single-binary setting, and use IsApp() for code that should only run as part of the
 // Sourcegraph desktop app.
 func IsSingleBinary() bool {
-	// TODO(single-binary): check in the future if this is any single-binary deployment, not just
-	// app.
-	return Type() == App
+	return Type() == App || Type() == SingleProgram
 }

internal/conf/deploy/endpoints.go

@@ -7,10 +7,10 @@ import (
 // BlobstoreEndpoint returns the default blobstore endpoint that should be used for this deployment
 // type.
 func BlobstoreDefaultEndpoint() string {
-	if IsApp() {
+	if IsSingleBinary() {
 		return "http://127.0.0.1:49000"
 	}
-	if IsSingleBinary() || IsDeployTypeSingleDockerContainer(Type()) {
+	if IsDeployTypeSingleDockerContainer(Type()) {
 		return "http://127.0.0.1:9000"
 	}
 	return "http://blobstore:9000"
@@ -18,10 +18,10 @@ func BlobstoreDefaultEndpoint() string {
 
 // BlobstoreHostPort returns the host/port that should be listened on for this deployment type.
 func BlobstoreHostPort() (string, string) {
-	if IsApp() {
+	if IsApp() || IsSingleBinary() {
 		return "127.0.0.1", "49000"
 	}
-	if env.InsecureDev || IsSingleBinary() || IsDeployTypeSingleDockerContainer(Type()) {
+	if env.InsecureDev || IsDeployTypeSingleDockerContainer(Type()) {
 		return "127.0.0.1", "9000"
 	}
 	return "", "9000"

internal/highlight/highlight.go

@@ -421,7 +421,7 @@ func Code(ctx context.Context, p Params) (response *HighlightedCode, aborted boo
 
 	query.Filetype = filetypeQuery.Language
 
-	// Cody App: we do not use syntect_server/syntax-highlighter
+	// Single-program mode: we do not use syntect_server/syntax-highlighter
 	//
 	// 1. It makes cross-compilation harder (requires a full Rust toolchain for the target, plus
 	//    a full C/C++ toolchain for the target.) Complicates macOS code signing.
@@ -431,8 +431,8 @@ func Code(ctx context.Context, p Params) (response *HighlightedCode, aborted boo
 	//    hack to workaround https://github.com/trishume/syntect/issues/202 - and by extension needs
 	//    two separate binaries, and separate processes, to function semi-reliably.
 	//
-	// Instead, in Cody App we defer to Chroma for syntax highlighting.
-	if deploy.IsApp() {
+	// Instead, in single-program mode we defer to Chroma for syntax highlighting.
+	if deploy.IsSingleBinary() {
 		document, err := highlightWithChroma(code, p.Filepath)
 		if err != nil {
 			return unhighlightedCode(err, code)

internal/ratelimit/globallimiter.go

@@ -79,12 +79,12 @@ func NewGlobalRateLimiter(logger log.Logger, bucketName string) GlobalLimiter {
 	// prevent the instance from breaking entirely. Note that the limits may NOT
 	// be enforced like configured then and should be treated as best effort only.
 	// Errors will be logged frequently.
-	// In App, this will still correctly limit globally, because all the services
-	// run in the same process and share memory. Outside of App, it is best effort only.
+	// In single-program mode, this will still correctly limit globally, because all the services
+	// run in the same process and share memory. Otherwise, it is best effort only.
 	pool, ok := kv().Pool()
 	if !ok {
-		if !deploy.IsApp() {
-			// Outside of app, this should be considered a configuration mistake.
+		if !deploy.IsSingleBinary() {
+			// Outside of single-program mode, this should be considered a configuration mistake.
 			logger.Error("Redis pool not set, global rate limiter will not work as expected")
 		}
 		rl := -1 // Documented default in site-config JSON schema. -1 means infinite.

internal/repoupdater/client.go

@@ -41,7 +41,7 @@ func repoUpdaterURLDefault() string {
 		return u
 	}
 
-	if deploy.IsApp() {
+	if deploy.IsSingleBinary() {
 		return "http://127.0.0.1:3182"
 	}
 

internal/service/svcmain/svcmain.go

@@ -39,10 +39,10 @@ type Config struct {
 //
 // args is the commandline arguments (usually os.Args).
 func Main(services []sgservice.Service, config Config, args []string) {
-	// Unlike other sourcegraph binaries we expect Cody App to be run
+	// Unlike other sourcegraph binaries we expect to be run
 	// by a user instead of deployed to a cloud. So adjust the default output
 	// format before initializing log.
-	if _, ok := os.LookupEnv(log.EnvLogFormat); !ok && deploy.IsApp() {
+	if _, ok := os.LookupEnv(log.EnvLogFormat); !ok && deploy.IsSingleBinary() {
 		os.Setenv(log.EnvLogFormat, string(output.FormatConsole))
 	}
 
@@ -251,7 +251,7 @@ func run(
 			defer ready()
 
 			// Don't run executors for Cody App
-			if deploy.IsApp() && !deploy.IsAppFullSourcegraph() && service.Name() == "executor" {
+			if deploy.IsApp() && service.Name() == "executor" {
 				logger.Info("Skipping", log.String("service", service.Name()))
 				return
 			}

internal/singleprogram/singleprogram.go

@@ -29,9 +29,8 @@ type CleanupFunc func() error
 func Init(logger log.Logger) CleanupFunc {
 	if deploy.IsApp() {
 		fmt.Fprintln(os.Stderr, "✱ Cody App version:", version.Version(), runtime.GOOS, runtime.GOARCH)
-	}
-	if deploy.IsAppFullSourcegraph() {
-		fmt.Fprintln(os.Stderr, "✱✱✱ Cody App ✱✱✱ full Sourcegraph mode enabled!")
+	} else if deploy.IsDeployTypeSingleProgram(deploy.Type()) {
+		fmt.Fprintln(os.Stderr, "✱ Sourcegraph (single-program) version:", version.Version(), runtime.GOOS, runtime.GOARCH)
 	}
 
 	// TODO(sqs) TODO(single-binary): see the env.HackClearEnvironCache docstring, we should be able to remove this
@@ -162,7 +161,7 @@ func Init(logger log.Logger) CleanupFunc {
 		}
 	}
 
-	if deploy.IsAppFullSourcegraph() || !deploy.IsApp() {
+	if !deploy.IsApp() {
 		setDefaultEnv(logger, "CTAGS_PROCESSES", "2")
 
 		haveDocker := isDockerAvailable()

sg.config.yaml

@@ -922,8 +922,8 @@ commands:
     cmd: pnpm --filter @sourcegraph/browser dev
     install: pnpm install
 
-  sourcegraph:
-    description: Single program (Go static binary) distribution
+  sourcegraph: &sourcegraph_command
+    description: Single-program distribution
     cmd: |
       unset SRC_GIT_SERVERS INDEXED_SEARCH_SERVERS REDIS_ENDPOINT
 
@@ -937,7 +937,7 @@ commands:
       if [ -n "$DELVE" ]; then
         export GCFLAGS='all=-N -l'
       fi
-      go build -gcflags="$GCFLAGS" -ldflags="-X github.com/sourcegraph/sourcegraph/internal/conf/deploy.forceType=app" -o .bin/sourcegraph github.com/sourcegraph/sourcegraph/cmd/sourcegraph
+      go build -gcflags="$GCFLAGS" -ldflags="-X github.com/sourcegraph/sourcegraph/internal/conf/deploy.forceType=single-program" -o .bin/sourcegraph github.com/sourcegraph/sourcegraph/cmd/sourcegraph
     checkBinary: .bin/sourcegraph
     env:
       ENTERPRISE: 1
@@ -951,6 +951,15 @@ commands:
       - lib
       - schema
 
+  cody-app:
+    <<: *sourcegraph_command
+    description: Cody App
+    install: |
+      if [ -n "$DELVE" ]; then
+        export GCFLAGS='all=-N -l'
+      fi
+      go build -gcflags="$GCFLAGS" -ldflags="-X github.com/sourcegraph/sourcegraph/internal/conf/deploy.forceType=app" -o .bin/sourcegraph github.com/sourcegraph/sourcegraph/cmd/sourcegraph
+
   tauri:
     description: App shell (Tauri)
     cmd: pnpm tauri dev --config src-tauri/tauri.dev.conf.json
@@ -1475,12 +1484,26 @@ commandsets:
       - otel-collector
       - jaeger
 
-  app:
+  single-program:
     requiresDevPrivate: true
     checks:
       - git
     commands:
       - sourcegraph
+      - web
+      - caddy
+    env:
+      DISABLE_CODE_INSIGHTS: false
+      PRECISE_CODE_INTEL_UPLOAD_AWS_ENDPOINT: http://localhost:49000
+      EMBEDDINGS_UPLOAD_AWS_ENDPOINT: http://localhost:49000
+      USE_EMBEDDED_POSTGRESQL: false
+
+  app:
+    requiresDevPrivate: true
+    checks:
+      - git
+    commands:
+      - cody-app
       - docsite
       - web
       - caddy