artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 15:51:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sourcegraph App (single-binary branch) (#46547)

Browse Source 
* internal: add service and singleprogram packages
* sg.config.yaml: add single-binary build targets
* internal/env: add a function for clearing environ cache
* internal/{workerutil,metrics}: add a hack to allow running 2 executors in the same process
* internal/conf: add single-program deploy type
* internal/singleprogram: clarify security
* cmd/sourcegraph-oss: add initial single-binary main (will not build yet)
* enterprise/cmd/sourcegraph: initial enterprise single-binary
* Add multi-platform builds for single-program
* single-binary: correctly build JS artifacts into binary
* license_finder licenses add github.com/xi2/xz "Public domain"
* internal/service/svcmain: correctly initialize logger for DeprecatedSingleServiceMain
* worker: refactor to new service pattern
* cmd/github-proxy: refactor to use new service pattern
* symbols: refactor to use new service pattern
* gitserver: refactor to user new service pattern
* searcher: refactor to use new service pattern
* gitserver: refactor to use new service pattern
* repo-updater: refactor to use new service pattern
* frontend: refactor to use new service pattern
* executor: refactor to use new service pattern
* internal/symbols: use new LoadConfig pattern
* precise-code-intel-worker: refactor to use new service pattern
* internal/symbols: load config for tests
* cmd/repo-updater: remove LoadConfig approach
* cmd/symbols: workaround env var conflict with searcher
* executor: internal: add workaround to allow running 2 instances in same process
* executors: add EXECUTOR_QUEUE_DISABLE_ACCESS_TOKEN for single-binary and dev deployments only
* single-binary: use EXECUTOR_QUEUE_DISABLE_ACCESS_TOKEN
* extsvc/github: fix default value for single-program deploy type
* single-binary: stop relying on a local ctags image
* single-binary: use unix sockets for postgres
* release App snapshots in CI when pushed to app/release-snapshot branch
* internal/service/svcmain: update TODO comment
* executor: correct DEPLOY_TYPE check
* dev/check: allow single-binary to import dbconn
* executor: remove accidental reliance on dbconn package
* executor: improve error logging when running commands (#46546)
* executor: improve error logging when running commands
* executor: do not attempt std config validation running e.g. install cmd
* executor: do not pull in the conf package / frontend reliance
* ci: executors: correct site config for passwordless auth
* server: fix bug where github-proxy would try to be a conf server
* CI: executors: fix integration test passwordless auth
* executors: allow passwordless auth in sourcegraph/server for testing
* repo-updater: fix enterprise init (caused regression in repository syncing)

Signed-off-by: Stephen Gutekanst <stephen@sourcegraph.com>
Co-authored-by: Peter Guy <peter.guy@sourcegraph.com>
Co-authored-by: Quinn Slack <quinn@slack.org>
This commit is contained in:
Stephen Gutekanst 2023-01-19 17:35:39 -07:00 committed by GitHub
parent 02c985e3d7
commit be4f4409a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
94 changed files with 1890 additions and 596 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -184,3 +184,8 @@ go.work.sum
# SCIP
index.scip
# Buildkite helper and cache files
/an
/tr
/cache-*.tar

10
cmd/frontend/internal/cli/config.go
View File

@ -18,6 +18,7 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/cmd/frontend/envvar"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/highlight"
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/conf"
@ -29,6 +30,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
"github.com/sourcegraph/sourcegraph/internal/jsonc"
"github.com/sourcegraph/sourcegraph/internal/symbols"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
@ -551,7 +553,7 @@ func serviceConnections(logger log.Logger) conftypes.ServiceConnections {
}
var (
searcherURL = env.Get("SEARCHER_URL", "k8s+http://searcher:3181", "searcher server URL")
searcherURL string
searcherURLsOnce sync.Once
searcherURLs *endpoint.Map
@ -572,6 +574,12 @@ var (
}()
)
func LoadConfig() {
searcherURL = env.Get("SEARCHER_URL", "k8s+http://searcher:3181", "searcher server URL")
highlight.LoadConfig()
symbols.LoadConfig()
}
func computeSearcherEndpoints() *endpoint.Map {
searcherURLsOnce.Do(func() {
if len(strings.Fields(searcherURL)) == 0 {

45
cmd/frontend/internal/cli/serve_cmd.go
View File

@ -8,10 +8,8 @@ import (
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/getsentry/sentry-go"
"github.com/graph-gophers/graphql-go"
"github.com/keegancsmith/tmpfriend"
sglog "github.com/sourcegraph/log"
@ -26,7 +24,6 @@ import (
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/app/ui"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/app/updatecheck"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/bg"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/cli/loghandlers"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/httpapi"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/siteid"
oce "github.com/sourcegraph/sourcegraph/cmd/frontend/oneclickexport"
@ -36,20 +33,16 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/encryption/keyring"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/oobmigration"
"github.com/sourcegraph/sourcegraph/internal/profiler"
"github.com/sourcegraph/sourcegraph/internal/redispool"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/sysreq"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/users"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/internal/version/upgradestore"
@ -93,27 +86,11 @@ func InitDB(logger sglog.Logger) (*sql.DB, error) {
return sqlDB, nil
}
type SetupFunc func(database.DB, conftypes.UnifiedWatchable) enterprise.Services
// Main is the main entrypoint for the frontend server program.
func Main(enterpriseSetupHook func(database.DB, conftypes.UnifiedWatchable) enterprise.Services) error {
ctx := context.Background()
log.SetFlags(0)
log.SetPrefix("")
liblog := sglog.Init(sglog.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
}, sglog.NewSentrySinkWith(
sglog.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
)) // Experimental: DevX is observing how sampling affects the errors signal
defer liblog.Sync()
logger := sglog.Scoped("server", "the frontend server program")
ready := make(chan struct{})
go debugserver.NewServerRoutine(ready).Start()
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, enterpriseSetupHook SetupFunc) error {
logger := observationCtx.Logger
sqlDB, err := InitDB(logger)
if err != nil {
@ -121,8 +98,6 @@ func Main(enterpriseSetupHook func(database.DB, conftypes.UnifiedWatchable) ente
}
db := database.NewDB(logger, sqlDB)
observationCtx := observation.NewContext(logger)
if os.Getenv("SRC_DISABLE_OOBMIGRATION_VALIDATION") != "" {
logger.Warn("Skipping out-of-band migrations check")
} else {
@ -142,9 +117,7 @@ func Main(enterpriseSetupHook func(database.DB, conftypes.UnifiedWatchable) ente
return errors.Wrap(err, "failed to apply site config overrides")
}
globals.ConfigurationServerFrontendOnly = conf.InitConfigurationServerFrontendOnly(newConfigurationSource(logger, db))
conf.Init()
conf.MustValidateDefaults()
go conf.Watch(liblog.Update(conf.GetLogSinks))
// now we can init the keyring, as it depends on site config
if err := keyring.Init(ctx); err != nil {
@ -161,12 +134,6 @@ func Main(enterpriseSetupHook func(database.DB, conftypes.UnifiedWatchable) ente
return errors.Wrap(err, "failed to override external service config")
}
// Filter trace logs
d, _ := time.ParseDuration(traceThreshold)
logging.Init(logging.Filter(loghandlers.Trace(strings.Fields(traceFields), d))) //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
tracer.Init(sglog.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
profiler.Init()
// Run enterprise setup hook
enterprise := enterpriseSetupHook(db, conf.DefaultClient())
@ -283,7 +250,7 @@ func Main(enterpriseSetupHook func(database.DB, conftypes.UnifiedWatchable) ente
println(fmt.Sprintf("\n\n%s\n\n", logoColor))
}
logger.Info(fmt.Sprintf("✱ Sourcegraph is ready at: %s", globals.ExternalURL()))
close(ready)
ready()
goroutine.MonitorBackgroundRoutines(context.Background(), routines...)
return nil

14
cmd/frontend/internal/highlight/highlight.go
View File

@ -34,10 +34,12 @@ import (
"github.com/sourcegraph/sourcegraph/lib/errors"
)
var (
syntectServer = env.Get("SRC_SYNTECT_SERVER", "http://syntect-server:9238", "syntect_server HTTP(s) address")
client *gosyntect.Client
)
func LoadConfig() {
syntectServer := env.Get("SRC_SYNTECT_SERVER", "http://syntect-server:9238", "syntect_server HTTP(s) address")
client = gosyntect.New(syntectServer)
}
var client *gosyntect.Client
var (
highlightOpOnce sync.Once
@ -63,10 +65,6 @@ func getHighlightOp() *observation.Operation {
return highlightOp
}
func init() {
client = gosyntect.New(syntectServer)
}
// IsBinary is a helper to tell if the content of a file is binary or not.
// TODO(tjdevries): This doesn't make sense to be here, IMO
func IsBinary(content []byte) bool {

21
cmd/frontend/main.go
View File

@ -1,26 +1,11 @@
// Command frontend is a service that serves the web frontend and API.
package main
import (
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
"github.com/sourcegraph/sourcegraph/cmd/frontend/shared"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
)
// Note: All frontend code should be added to shared.Main, not here. See that
// function for details.
func main() {
// Set dummy authz provider to unblock channel for checking permissions in GraphQL APIs.
// See https://github.com/sourcegraph/sourcegraph/issues/3847 for details.
authz.SetProviders(true, []authz.Provider{})
env.Lock()
env.HandleHelpFlag()
shared.Main(func(_ database.DB, _ conftypes.UnifiedWatchable) enterprise.Services {
return enterprise.DefaultServices()
})
osscmd.DeprecatedSingleServiceMainOSS(shared.Service)
}

27
cmd/frontend/shared/frontend.go
View File

@ -1,27 +0,0 @@
// Package shared contains the frontend command implementation shared
package shared
import (
"fmt"
"os"
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/cli"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
_ "github.com/sourcegraph/sourcegraph/cmd/frontend/registry/api"
)
// Main is the main function that runs the frontend process.
//
// It is exposed as function in a package so that it can be called by other
// main package implementations such as Sourcegraph Enterprise, which import
// proprietary/private code.
func Main(enterpriseSetupHook func(database.DB, conftypes.UnifiedWatchable) enterprise.Services) {
err := cli.Main(enterpriseSetupHook)
if err != nil {
fmt.Fprintln(os.Stderr, "fatal:", err)
os.Exit(1)
}
}

39
cmd/frontend/shared/service.go Normal file
View File

@ -0,0 +1,39 @@
// Package shared contains the frontend command implementation shared
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/cli"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "frontend" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
CLILoadConfig()
return nil, nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
ossSetupHook := func(_ database.DB, _ conftypes.UnifiedWatchable) enterprise.Services {
return enterprise.DefaultServices()
}
return CLIMain(ctx, observationCtx, ready, ossSetupHook)
}
var Service service.Service = svc{}
// Reexported to get around `internal` package.
var (
CLILoadConfig = cli.LoadConfig
CLIMain = cli.Main
)

6
cmd/github-proxy/github-proxy.go
View File

@ -2,11 +2,9 @@ package main
import (
"github.com/sourcegraph/sourcegraph/cmd/github-proxy/shared"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
)
func main() {
env.Lock()
env.HandleHelpFlag()
shared.Main()
osscmd.DeprecatedSingleServiceMainOSS(shared.Service)
}

22
cmd/github-proxy/shared/service.go Normal file
View File

@ -0,0 +1,22 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "github-proxy" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) { return nil, nil }
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, _ env.Config) error {
return Main(ctx, observationCtx, ready)
}
var Service service.Service = svc{}

32
cmd/github-proxy/shared/shared.go
View File

@ -15,7 +15,6 @@ import (
"syscall"
"time"
"github.com/getsentry/sentry-go"
"github.com/gorilla/handlers"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
@ -24,14 +23,12 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/instrumentation"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/version"
)
var logRequests, _ = strconv.ParseBool(env.Get("LOG_REQUESTS", "", "log HTTP requests"))
@ -56,28 +53,11 @@ var hopHeaders = map[string]struct{}{
"Upgrade": {},
}
func Main() {
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
}, log.NewSentrySinkWith(
log.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
)) // Experimental: DevX is observing how sampling affects the errors signal
defer liblog.Sync()
conf.Init()
go conf.Watch(liblog.Update(conf.GetLogSinks))
tracer.Init(log.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc) error {
logger := observationCtx.Logger
// Ready immediately
ready := make(chan struct{})
close(ready)
go debugserver.NewServerRoutine(ready).Start()
logger := log.Scoped("server", "the github-proxy service")
ready()
p := &githubProxy{
logger: logger,
@ -129,6 +109,8 @@ func Main() {
if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Fatal(err.Error())
}
return nil
}
func instrumentHandler(r prometheus.Registerer, h http.Handler) http.Handler {

7
cmd/gitserver/main.go
View File

@ -3,12 +3,9 @@ package main // import "github.com/sourcegraph/sourcegraph/cmd/gitserver"
import (
"github.com/sourcegraph/sourcegraph/cmd/gitserver/shared"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
)
func main() {
env.Lock()
env.HandleHelpFlag()
shared.Main(nil)
osscmd.DeprecatedSingleServiceMainOSS(shared.Service)
}

24
cmd/gitserver/shared/service.go Normal file
View File

@ -0,0 +1,24 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "gitserver" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
return LoadConfig(), nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return Main(ctx, observationCtx, ready, config.(*Config), nil)
}
var Service service.Service = svc{}

78
cmd/gitserver/shared/shared.go
View File

@ -14,7 +14,6 @@ import (
"syscall"
"time"
"github.com/getsentry/sentry-go"
jsoniter "github.com/json-iterator/go"
"github.com/sourcegraph/log"
"github.com/tidwall/gjson"
@ -30,7 +29,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/encryption/keyring"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
@ -45,22 +43,18 @@ import (
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/internal/instrumentation"
"github.com/sourcegraph/sourcegraph/internal/jsonc"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/profiler"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
"github.com/sourcegraph/sourcegraph/internal/repos"
"github.com/sourcegraph/sourcegraph/internal/requestclient"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
)
var (
reposDir = env.Get("SRC_REPOS_DIR", "/data/repos", "Root dir containing repos.")
// Align these variables with the 'disk_space_remaining' alerts in monitoring
wantPctFree = env.MustGetInt("SRC_REPOS_DESIRED_PERCENT_FREE", 10, "Target percentage of free space on disk.")
@ -77,48 +71,42 @@ var (
type EnterpriseInit func(db database.DB)
func Main(enterpriseInit EnterpriseInit) {
ctx := context.Background()
type Config struct {
env.BaseConfig
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
ReposDir string
}
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
}, log.NewSentrySinkWith(
log.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
))
defer liblog.Sync()
func (c *Config) Load() {
c.ReposDir = c.Get("SRC_REPOS_DIR", "/data/repos", "Root dir containing repos.")
}
conf.Init()
go conf.Watch(liblog.Update(conf.GetLogSinks))
func LoadConfig() *Config {
var config Config
config.Load()
return &config
}
tracer.Init(log.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
profiler.Init()
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config *Config, enterpriseInit EnterpriseInit) error {
logger := observationCtx.Logger
logger := log.Scoped("server", "the gitserver service")
observationCtx := observation.NewContext(logger)
if reposDir == "" {
logger.Fatal("SRC_REPOS_DIR is required")
if config.ReposDir == "" {
return errors.New("SRC_REPOS_DIR is required")
}
if err := os.MkdirAll(reposDir, os.ModePerm); err != nil {
logger.Fatal("failed to create SRC_REPOS_DIR", log.Error(err))
if err := os.MkdirAll(config.ReposDir, os.ModePerm); err != nil {
return errors.Wrap(err, "creating SRC_REPOS_DIR")
}
wantPctFree2, err := getPercent(wantPctFree)
if err != nil {
logger.Fatal("SRC_REPOS_DESIRED_PERCENT_FREE is out of range", log.Error(err))
return errors.Wrap(err, "SRC_REPOS_DESIRED_PERCENT_FREE is out of range")
}
sqlDB, err := getDB(observationCtx)
if err != nil {
logger.Fatal("failed to initialize database stores", log.Error(err))
return errors.Wrap(err, "initializing database stores")
}
db := database.NewDB(logger, sqlDB)
db := database.NewDB(observationCtx.Logger, sqlDB)
repoStore := db.Repos()
dependenciesSvc := dependencies.NewService(observationCtx, db)
@ -126,7 +114,7 @@ func Main(enterpriseInit EnterpriseInit) {
err = keyring.Init(ctx)
if err != nil {
logger.Fatal("failed to initialise keyring", log.Error(err))
return errors.Wrap(err, "initializing keyring")
}
if enterpriseInit != nil {
@ -134,19 +122,19 @@ func Main(enterpriseInit EnterpriseInit) {
}
if err != nil {
logger.Fatal("Failed to create sub-repo client", log.Error(err))
return errors.Wrap(err, "creating sub-repo client")
}
gitserver := server.Server{
Logger: logger,
ObservationCtx: observationCtx,
ReposDir: reposDir,
ReposDir: config.ReposDir,
DesiredPercentFree: wantPctFree2,
GetRemoteURLFunc: func(ctx context.Context, repo api.RepoName) (string, error) {
return getRemoteURLFunc(ctx, externalServiceStore, repoStore, nil, repo)
},
GetVCSSyncer: func(ctx context.Context, repo api.RepoName) (server.VCSSyncer, error) {
return getVCSSyncer(ctx, externalServiceStore, repoStore, dependenciesSvc, repo, reposDir)
return getVCSSyncer(ctx, externalServiceStore, repoStore, dependenciesSvc, repo, config.ReposDir)
},
Hostname: hostname.Get(),
DB: db,
@ -157,11 +145,11 @@ func Main(enterpriseInit EnterpriseInit) {
gitserver.RegisterMetrics(observationCtx, db)
if tmpDir, err := gitserver.SetupAndClearTmp(); err != nil {
logger.Fatal("failed to setup temporary directory", log.Error(err))
return errors.Wrap(err, "failed to setup temporary directory")
} else if err := os.Setenv("TMP_DIR", tmpDir); err != nil {
// Additionally, set TMP_DIR so other temporary files we may accidentally
// create are on the faster RepoDir mount.
logger.Fatal("Setting TMP_DIR", log.Error(err))
return errors.Wrap(err, "setting TMP_DIR")
}
// Create Handler now since it also initializes state
@ -172,10 +160,6 @@ func Main(enterpriseInit EnterpriseInit) {
handler = trace.HTTPMiddleware(logger, handler, conf.DefaultClient())
handler = instrumentation.HTTPMiddleware("", handler)
// Ready immediately
ready := make(chan struct{})
close(ready)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -185,8 +169,10 @@ func Main(enterpriseInit EnterpriseInit) {
logger.Warn("error performing initial rate limit sync", log.Error(err))
}
// Ready immediately
ready()
go syncRateLimiters(ctx, logger, externalServiceStore, rateLimitSyncerLimitPerSecond)
go debugserver.NewServerRoutine(ready).Start()
go gitserver.Janitor(actor.WithInternalActor(ctx), janitorInterval)
go gitserver.SyncRepoState(syncRepoStateInterval, syncRepoStateBatchSize, syncRepoStateUpdatePerSecond)
@ -239,6 +225,8 @@ func Main(enterpriseInit EnterpriseInit) {
// The most important thing this does is kill all our clones. If we just
// shutdown they will be orphaned and continue running.
gitserver.Stop()
return nil
}
func configureFusionClient(conn schema.PerforceConnection) server.FusionConfig {

12
cmd/repo-updater/main.go
View File

@ -4,17 +4,9 @@ package main
import (
"github.com/sourcegraph/sourcegraph/cmd/repo-updater/shared"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
)
func main() {
env.Lock()
env.HandleHelpFlag()
// Set dummy authz provider to unblock channel for checking permissions in GraphQL APIs.
// See https://github.com/sourcegraph/sourcegraph/issues/3847 for details.
authz.SetProviders(true, []authz.Provider{})
shared.Main(nil)
osscmd.DeprecatedSingleServiceMainOSS(shared.Service)
}

67
cmd/repo-updater/shared/main.go
View File

@ -11,7 +11,6 @@ import (
"strconv"
"time"
"github.com/getsentry/sentry-go"
"github.com/graph-gophers/graphql-go/relay"
"github.com/prometheus/client_golang/prometheus"
"github.com/sourcegraph/log"
@ -36,19 +35,16 @@ import (
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
"github.com/sourcegraph/sourcegraph/internal/instrumentation"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/profiler"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
"github.com/sourcegraph/sourcegraph/internal/repos"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
const port = "3182"
@ -78,44 +74,16 @@ type LazyDebugserverEndpoint struct {
manualPurgeEndpoint http.HandlerFunc
}
func Main(enterpriseInit EnterpriseInit) {
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, debugserverEndpoints *LazyDebugserverEndpoint, enterpriseInit EnterpriseInit) error {
// NOTE: Internal actor is required to have full visibility of the repo table
// (i.e. bypass repository authorization).
ctx := actor.WithInternalActor(context.Background())
ctx = actor.WithInternalActor(ctx)
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
}, log.NewSentrySinkWith(
log.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
)) // Experimental: DevX is observing how sampling affects the errors signal
defer liblog.Sync()
conf.Init()
go conf.Watch(liblog.Update(conf.GetLogSinks))
tracer.Init(log.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
profiler.Init()
logger := log.Scoped("service", "repo-updater service")
observationCtx := observation.NewContext(logger)
// Signals health of startup
ready := make(chan struct{})
// Start debug server
debugserverEndpoints := LazyDebugserverEndpoint{}
debugServerRoutine := createDebugServerRoutine(ready, &debugserverEndpoints)
go debugServerRoutine.Start()
logger := observationCtx.Logger
clock := func() time.Time { return time.Now().UTC() }
if err := keyring.Init(ctx); err != nil {
logger.Fatal("error initialising encryption keyring", log.Error(err))
return errors.Wrap(err, "initializing encryption keyring")
}
dsn := conf.GetServiceConnectionValueAndRestartOnChange(func(serviceConnections conftypes.ServiceConnections) string {
@ -123,7 +91,7 @@ func Main(enterpriseInit EnterpriseInit) {
})
sqlDB, err := connections.EnsureNewFrontendDB(observationCtx, dsn, "repo-updater")
if err != nil {
logger.Fatal("failed to initialize database store", log.Error(err))
return errors.Wrap(err, "initializing database store")
}
db := database.NewDB(logger, sqlDB)
@ -252,7 +220,7 @@ func Main(enterpriseInit EnterpriseInit) {
// the debugserver constructed at the top of this function. This ensures we don't
// have a race between becoming ready and a debugserver request failing directly
// after being unblocked.
close(ready)
ready()
// NOTE: Internal actor is required to have full visibility of the repo table
// (i.e. bypass repository authorization).
@ -269,12 +237,13 @@ func Main(enterpriseInit EnterpriseInit) {
trace.HTTPMiddleware(logger, authzBypass(handler), conf.DefaultClient())),
})
goroutine.MonitorBackgroundRoutines(ctx, httpSrv)
return nil
}
func createDebugServerRoutine(ready chan struct{}, debugserverEndpoints *LazyDebugserverEndpoint) goroutine.BackgroundRoutine {
return debugserver.NewServerRoutine(
ready,
debugserver.Endpoint{
func createDebugServerEndpoints(ready chan struct{}, debugserverEndpoints *LazyDebugserverEndpoint) []debugserver.Endpoint {
return []debugserver.Endpoint{
{
Name: "Repo Updater State",
Path: "/repo-updater-state",
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -284,7 +253,7 @@ func createDebugServerRoutine(ready chan struct{}, debugserverEndpoints *LazyDeb
debugserverEndpoints.repoUpdaterStateEndpoint(w, r)
}),
},
debugserver.Endpoint{
{
Name: "List Authz Providers",
Path: "/list-authz-providers",
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -294,7 +263,7 @@ func createDebugServerRoutine(ready chan struct{}, debugserverEndpoints *LazyDeb
debugserverEndpoints.listAuthzProvidersEndpoint(w, r)
}),
},
debugserver.Endpoint{
{
Name: "Gitserver Repo Status",
Path: "/gitserver-repo-status",
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -302,7 +271,7 @@ func createDebugServerRoutine(ready chan struct{}, debugserverEndpoints *LazyDeb
debugserverEndpoints.gitserverReposStatusEndpoint(w, r)
}),
},
debugserver.Endpoint{
{
Name: "Rate Limiter State",
Path: "/rate-limiter-state",
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -310,7 +279,7 @@ func createDebugServerRoutine(ready chan struct{}, debugserverEndpoints *LazyDeb
debugserverEndpoints.rateLimiterStateEndpoint(w, r)
}),
},
debugserver.Endpoint{
{
Name: "Manual Repo Purge",
Path: "/manual-purge",
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -318,7 +287,7 @@ func createDebugServerRoutine(ready chan struct{}, debugserverEndpoints *LazyDeb
debugserverEndpoints.manualPurgeEndpoint(w, r)
}),
},
)
}
}
func gitserverReposStatusHandler(db database.DB) http.HandlerFunc {

44
cmd/repo-updater/shared/service.go Normal file
View File

@ -0,0 +1,44 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct {
enterpriseInit EnterpriseInit
ready chan struct{}
debugserverEndpoints LazyDebugserverEndpoint
}
func (svc) Name() string { return "repo-updater" }
func (s *svc) Configure() (env.Config, []debugserver.Endpoint) {
// Signals health of startup.
s.ready = make(chan struct{})
return nil, createDebugServerEndpoints(s.ready, &s.debugserverEndpoints)
}
func (s *svc) Start(ctx context.Context, observationCtx *observation.Context, signalReadyToParent service.ReadyFunc, _ env.Config) error {
// This service's debugserver endpoints should start responding when this service is ready (and
// not ewait for *all* services to be ready). Therefore, we need to track whether we are ready
// separately.
ready := service.ReadyFunc(func() {
close(s.ready)
signalReadyToParent()
})
return Main(ctx, observationCtx, ready, &s.debugserverEndpoints, s.enterpriseInit)
}
var Service service.Service = NewServiceWithEnterpriseInit(nil)
func NewServiceWithEnterpriseInit(enterpriseInit EnterpriseInit) *svc {
return &svc{enterpriseInit: enterpriseInit}
}

7
cmd/searcher/main.go
View File

@ -4,12 +4,9 @@ package main
import (
"github.com/sourcegraph/sourcegraph/cmd/searcher/shared"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
)
func main() {
env.Lock()
env.HandleHelpFlag()
shared.Main()
osscmd.DeprecatedSingleServiceMainOSS(shared.Service)
}

22
cmd/searcher/shared/service.go Normal file
View File

@ -0,0 +1,22 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "searcher" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) { return nil, nil }
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, _ env.Config) error {
return Start(ctx, observationCtx, ready)
}
var Service service.Service = svc{}

46
cmd/searcher/shared/shared.go
View File

@ -5,7 +5,6 @@ package shared
import (
"context"
"io"
stdlog "log"
"net"
"net/http"
"os"
@ -17,7 +16,6 @@ import (
"golang.org/x/sync/errgroup"
"github.com/getsentry/sentry-go"
"github.com/keegancsmith/tmpfriend"
"github.com/sourcegraph/log"
@ -28,20 +26,15 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/instrumentation"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/profiler"
sharedsearch "github.com/sourcegraph/sourcegraph/internal/search"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -116,11 +109,11 @@ func setupTmpDir() error {
return nil
}
func run(logger log.Logger) error {
func Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc) error {
logger := observationCtx.Logger
// Ready immediately
ready := make(chan struct{})
close(ready)
go debugserver.NewServerRoutine(ready).Start()
ready()
var cacheSizeBytes int64
if i, err := strconv.ParseInt(cacheSizeMB, 10, 64); err != nil {
@ -198,7 +191,7 @@ func run(logger log.Logger) error {
handler = trace.HTTPMiddleware(logger, handler, conf.DefaultClient())
handler = instrumentation.HTTPMiddleware("", handler)
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
g, ctx := errgroup.WithContext(ctx)
@ -241,30 +234,3 @@ func run(logger log.Logger) error {
return g.Wait()
}
func Main() {
stdlog.SetFlags(0)
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
}, log.NewSentrySinkWith(
log.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
)) // Experimental: DevX is observing how sampling affects the errors signal
defer liblog.Sync()
conf.Init()
go conf.Watch(liblog.Update(conf.GetLogSinks))
tracer.Init(log.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
profiler.Init()
logger := log.Scoped("server", "the searcher service")
err := run(logger)
if err != nil {
logger.Fatal("searcher failed", log.Error(err))
}
}

29
cmd/sourcegraph-oss/main.go Normal file
View File

@ -0,0 +1,29 @@
// Command sourcegraph-oss is a single program that runs all of Sourcegraph (OSS variant).
package main
import (
frontend_shared "github.com/sourcegraph/sourcegraph/cmd/frontend/shared"
githubproxy_shared "github.com/sourcegraph/sourcegraph/cmd/github-proxy/shared"
gitserver_shared "github.com/sourcegraph/sourcegraph/cmd/gitserver/shared"
repoupdater_shared "github.com/sourcegraph/sourcegraph/cmd/repo-updater/shared"
searcher_shared "github.com/sourcegraph/sourcegraph/cmd/searcher/shared"
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
symbols_shared "github.com/sourcegraph/sourcegraph/cmd/symbols/shared"
worker_shared "github.com/sourcegraph/sourcegraph/cmd/worker/shared"
"github.com/sourcegraph/sourcegraph/internal/service"
)
// services is a list of services to run in the OSS build.
var services = []service.Service{
frontend_shared.Service,
gitserver_shared.Service,
repoupdater_shared.Service,
searcher_shared.Service,
symbols_shared.Service,
worker_shared.Service,
githubproxy_shared.Service,
}
func main() {
osscmd.MainOSS(services)
}

30
cmd/sourcegraph-oss/osscmd/osscmd.go Normal file
View File

@ -0,0 +1,30 @@
// Package osscmd defines entrypoint functions for the OSS build of Sourcegraph's single-program
// distribution. It is invoked by all OSS commands' main functions.
package osscmd
import (
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/service/svcmain"
)
var config = svcmain.Config{
AfterConfigure: func() {
// Set dummy authz provider to unblock channel for checking permissions in GraphQL APIs.
// See https://github.com/sourcegraph/sourcegraph/issues/3847 for details.
authz.SetProviders(true, []authz.Provider{})
},
}
// Main is called from the `main` function of the `sourcegraph-oss` command.
func MainOSS(services []service.Service) {
svcmain.Main(services, config)
}
// DeprecatedSingleServiceMainOSS is called from the `main` function of a command in the OSS build
// to start a single service (such as frontend or gitserver).
//
// DEPRECATED: See svcmain.DeprecatedSingleServiceMain documentation for more info.
func DeprecatedSingleServiceMainOSS(service service.Service) {
svcmain.DeprecatedSingleServiceMain(service, config, true, true)
}

3
cmd/symbols/main.go
View File

@ -3,9 +3,10 @@
package main
import (
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
"github.com/sourcegraph/sourcegraph/cmd/symbols/shared"
)
func main() {
shared.Main(shared.SetupSqlite)
osscmd.DeprecatedSingleServiceMainOSS(shared.Service)
}

50
cmd/symbols/shared/main.go
View File

@ -9,8 +9,6 @@ import (
"strconv"
"time"
"github.com/getsentry/sentry-go"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher"
@ -23,58 +21,36 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/honey"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
"github.com/sourcegraph/sourcegraph/internal/instrumentation"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/profiler"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
var sanityCheck, _ = strconv.ParseBool(env.Get("SANITY_CHECK", "false", "check that go-sqlite3 works then exit 0 if it's ok or 1 if not"))
var (
baseConfig = env.BaseConfig{}
RepositoryFetcherConfig = types.LoadRepositoryFetcherConfig(baseConfig)
CtagsConfig = types.LoadCtagsConfig(baseConfig)
RepositoryFetcherConfig types.RepositoryFetcherConfig
CtagsConfig types.CtagsConfig
)
const addr = ":3184"
type SetupFunc func(observationCtx *observation.Context, db database.DB, gitserverClient gitserver.GitserverClient, repositoryFetcher fetcher.RepositoryFetcher) (types.SearchFunc, func(http.ResponseWriter, *http.Request), []goroutine.BackgroundRoutine, string, error)
func Main(setup SetupFunc) {
// Initialization
env.HandleHelpFlag()
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
}, log.NewSentrySinkWith(
log.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
)) // Experimental: DevX is observing how sampling affects the errors signal
defer liblog.Sync()
conf.Init()
go conf.Watch(liblog.Update(conf.GetLogSinks))
tracer.Init(log.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
profiler.Init()
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, setup SetupFunc) error {
logger := observationCtx.Logger
routines := []goroutine.BackgroundRoutine{}
// Initialize tracing/metrics
logger := log.Scoped("service", "the symbols service")
observationCtx := observation.NewContext(logger, observation.Honeycomb(&honey.Dataset{
observationCtx = observation.NewContext(logger, observation.Honeycomb(&honey.Dataset{
Name: "codeintel-symbols",
SampleRate: 20,
}))
@ -104,14 +80,10 @@ func Main(setup SetupFunc) {
repositoryFetcher := fetcher.NewRepositoryFetcher(observationCtx, gitserverClient, RepositoryFetcherConfig.MaxTotalPathsLength, int64(RepositoryFetcherConfig.MaxFileSizeKb)*1000)
searchFunc, handleStatus, newRoutines, ctagsBinary, err := setup(observationCtx, db, gitserverClient, repositoryFetcher)
if err != nil {
logger.Fatal("Failed to set up", log.Error(err))
return errors.Wrap(err, "failed to set up")
}
routines = append(routines, newRoutines...)
// Start debug server
ready := make(chan struct{})
go debugserver.NewServerRoutine(ready).Start()
// Create HTTP server
handler := api.NewHandler(searchFunc, gitserverClient.ReadFile, handleStatus, ctagsBinary)
handler = handlePanic(logger, handler)
@ -126,8 +98,10 @@ func Main(setup SetupFunc) {
routines = append(routines, server)
// Mark health server as ready and go!
close(ready)
goroutine.MonitorBackgroundRoutines(context.Background(), routines...)
ready()
goroutine.MonitorBackgroundRoutines(ctx, routines...)
return nil
}
func mustInitializeFrontendDB(observationCtx *observation.Context) *sql.DB {

25
cmd/symbols/shared/service.go Normal file
View File

@ -0,0 +1,25 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "symbols" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
LoadConfig()
return nil, nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return Main(ctx, observationCtx, ready, SetupSqlite)
}
var Service service.Service = svc{}

8
cmd/symbols/shared/sqlite.go
View File

@ -23,7 +23,13 @@ import (
"github.com/sourcegraph/sourcegraph/internal/observation"
)
var config = types.LoadSqliteConfig(baseConfig, CtagsConfig, RepositoryFetcherConfig)
func LoadConfig() {
RepositoryFetcherConfig = types.LoadRepositoryFetcherConfig(baseConfig)
CtagsConfig = types.LoadCtagsConfig(baseConfig)
config = types.LoadSqliteConfig(baseConfig, CtagsConfig, RepositoryFetcherConfig)
}
var config types.SqliteConfig
func SetupSqlite(observationCtx *observation.Context, db database.DB, gitserverClient gitserver.GitserverClient, repositoryFetcher fetcher.RepositoryFetcher) (types.SearchFunc, func(http.ResponseWriter, *http.Request), []goroutine.BackgroundRoutine, string, error) {
logger := observationCtx.Logger.Scoped("sqlite.setup", "SQLite setup")

28
cmd/symbols/types/types.go
View File

@ -24,11 +24,29 @@ type SqliteConfig struct {
MaxConcurrentlyIndexing int
}
func aliasEnvVar(oldName, newName string) {
if os.Getenv(newName) != "" {
return // prefer using the new name
}
oldValue := os.Getenv(oldName)
if oldValue == "" {
return // old name was not set
}
// New name not in use, but old name is, so update the env.
_ = os.Setenv(newName, oldValue)
}
func LoadSqliteConfig(baseConfig env.BaseConfig, ctags CtagsConfig, repositoryFetcher RepositoryFetcherConfig) SqliteConfig {
// Variable was renamed to have SYMBOLS_ prefix to avoid a conflict with the same env var name
// in searcher when running as a single binary. The old name is treated as an alias to prevent
// customer environments from breaking if they still use it, because we have no way of migrating
// environment variables today.
aliasEnvVar("CACHE_DIR", "SYMBOLS_CACHE_DIR")
return SqliteConfig{
Ctags: ctags,
RepositoryFetcher: repositoryFetcher,
CacheDir: baseConfig.Get("CACHE_DIR", "/tmp/symbols-cache", "directory in which to store cached symbols"),
CacheDir: baseConfig.Get("SYMBOLS_CACHE_DIR", "/tmp/symbols-cache", "directory in which to store cached symbols"),
CacheSizeMB: baseConfig.GetInt("SYMBOLS_CACHE_SIZE_MB", "100000", "maximum size of the disk cache (in megabytes)"),
NumCtagsProcesses: baseConfig.GetInt("CTAGS_PROCESSES", strconv.Itoa(runtime.GOMAXPROCS(0)), "number of concurrent parser processes to run"),
RequestBufferSize: baseConfig.GetInt("REQUEST_BUFFER_SIZE", "8192", "maximum size of buffered parser request channel"),
@ -78,8 +96,14 @@ type RepositoryFetcherConfig struct {
}
func LoadRepositoryFetcherConfig(baseConfig env.BaseConfig) RepositoryFetcherConfig {
// Variable was renamed to have SYMBOLS_ prefix to avoid a conflict with the same env var name
// in searcher when running as a single binary. The old name is treated as an alias to prevent
// customer environments from breaking if they still use it, because we have no way of migrating
// environment variables today.
aliasEnvVar("MAX_TOTAL_PATHS_LENGTH", "SYMBOLS_MAX_TOTAL_PATHS_LENGTH")
return RepositoryFetcherConfig{
MaxTotalPathsLength: baseConfig.GetInt("MAX_TOTAL_PATHS_LENGTH", "100000", "maximum sum of lengths of all paths in a single call to git archive"),
MaxTotalPathsLength: baseConfig.GetInt("SYMBOLS_MAX_TOTAL_PATHS_LENGTH", "100000", "maximum sum of lengths of all paths in a single call to git archive"),
MaxFileSizeKb: baseConfig.GetInt("MAX_FILE_SIZE_KB", "1000", "maximum file size in KB, the contents of bigger files are ignored"),
}
}

24
cmd/worker/main.go
View File

@ -1,30 +1,10 @@
package main
import (
"os"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss/osscmd"
"github.com/sourcegraph/sourcegraph/cmd/worker/shared"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/version"
)
func main() {
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
})
defer liblog.Sync()
logger := log.Scoped("worker", "worker oss edition")
observationCtx := observation.NewContext(logger)
authz.SetProviders(true, []authz.Provider{})
if err := shared.Start(observationCtx, nil, nil, nil); err != nil {
logger.Error(err.Error())
os.Exit(1)
}
osscmd.DeprecatedSingleServiceMainOSS(shared.Service)
}

3
cmd/worker/shared/config.go
View File

@ -3,6 +3,7 @@ package shared
import (
"strings"
"github.com/sourcegraph/sourcegraph/cmd/worker/job"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -14,6 +15,8 @@ type Config struct {
env.BaseConfig
names []string
Jobs map[string]job.Job
JobAllowlist []string
JobBlocklist []string
}

62
cmd/worker/shared/main.go
View File

@ -10,7 +10,8 @@ import (
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine/recorder"
"github.com/sourcegraph/sourcegraph/cmd/worker/internal/codeintel"
@ -23,19 +24,15 @@ import (
"github.com/sourcegraph/sourcegraph/cmd/worker/internal/zoektrepos"
"github.com/sourcegraph/sourcegraph/cmd/worker/job"
workerdb "github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/db"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/encryption/keyring"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/oobmigration"
"github.com/sourcegraph/sourcegraph/internal/oobmigration/migrations"
"github.com/sourcegraph/sourcegraph/internal/profiler"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/symbols"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -48,8 +45,9 @@ type namedBackgroundRoutine struct {
JobName string
}
// Start runs the worker.
func Start(observationCtx *observation.Context, additionalJobs map[string]job.Job, registerEnterpriseMigrators oobmigration.RegisterMigratorsFunc, enterpriseInit EnterpriseInit) error {
func LoadConfig(additionalJobs map[string]job.Job, registerEnterpriseMigrators oobmigration.RegisterMigratorsFunc) *Config {
symbols.LoadConfig()
registerMigrators := oobmigration.ComposeRegisterMigratorsFuncs(migrations.RegisterOSSMigrators, registerEnterpriseMigrators)
builtins := map[string]job.Job{
@ -63,26 +61,31 @@ func Start(observationCtx *observation.Context, additionalJobs map[string]job.Jo
"outbound-webhook-sender": outboundwebhooks.NewSender(),
}
jobs := map[string]job.Job{}
var config Config
config.Jobs = map[string]job.Job{}
for name, job := range builtins {
jobs[name] = job
config.Jobs[name] = job
}
for name, job := range additionalJobs {
jobs[name] = job
config.Jobs[name] = job
}
// Setup environment variables
loadConfigs(jobs)
loadConfigs(config.Jobs)
env.Lock()
env.HandleHelpFlag()
conf.Init()
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
tracer.Init(log.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
profiler.Init()
// Validate environment variables
if err := validateConfigs(config.Jobs); err != nil {
config.AddError(err)
}
if err := keyring.Init(context.Background()); err != nil {
return errors.Wrap(err, "Failed to intialise keyring")
return &config
}
// Start runs the worker.
func Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config *Config, enterpriseInit EnterpriseInit) error {
if err := keyring.Init(ctx); err != nil {
return errors.Wrap(err, "initializing keyring")
}
if enterpriseInit != nil {
@ -94,23 +97,14 @@ func Start(observationCtx *observation.Context, additionalJobs map[string]job.Jo
enterpriseInit(db)
}
// Start debug server
ready := make(chan struct{})
go debugserver.NewServerRoutine(ready).Start()
// Validate environment variables
if err := validateConfigs(jobs); err != nil {
return err
}
// Emit metrics to help site admins detect instances that accidentally
// omit a job from from the instance's deployment configuration.
emitJobCountMetrics(jobs)
emitJobCountMetrics(config.Jobs)
// Create the background routines that the worker will monitor for its
// lifetime. There may be a non-trivial startup time on this step as we
// connect to external databases, wait for migrations, etc.
allRoutinesWithJobNames, err := createBackgroundRoutines(observationCtx, jobs)
allRoutinesWithJobNames, err := createBackgroundRoutines(observationCtx, config.Jobs)
if err != nil {
return err
}
@ -138,7 +132,7 @@ func Start(observationCtx *observation.Context, additionalJobs map[string]job.Jo
// We're all set up now
// Respond positively to ready checks
close(ready)
ready()
// This method blocks while the app is live - the following return is only to appease
// the type checker.
@ -147,7 +141,7 @@ func Start(observationCtx *observation.Context, additionalJobs map[string]job.Jo
allRoutines = append(allRoutines, r.Routine)
}
goroutine.MonitorBackgroundRoutines(context.Background(), allRoutines...)
goroutine.MonitorBackgroundRoutines(ctx, allRoutines...)
return nil
}

24
cmd/worker/shared/service.go Normal file
View File

@ -0,0 +1,24 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "worker" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
return LoadConfig(nil, nil), nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return Start(ctx, observationCtx, ready, config.(*Config), nil)
}
var Service service.Service = svc{}

3
dev/check/go-dbconn-import.sh
View File

@ -27,6 +27,9 @@ allowed_prefix=(
github.com/sourcegraph/sourcegraph/cmd/symbols
# Transitively depends on zoekt package which imports but does not use DB
github.com/sourcegraph/sourcegraph/cmd/searcher
# Main entrypoints for running all services, so they must be allowed to import it.
github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss
github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph
)
# Create regex ^(a|b|c)

15
dev/ci/runtype/runtype.go
View File

@ -18,9 +18,10 @@ const (
// Nightly builds - must be first because they take precedence
ReleaseNightly // release branch nightly healthcheck builds
BextNightly // browser extension nightly build
VsceNightly // vs code extension nightly build
ReleaseNightly // release branch nightly healthcheck builds
BextNightly // browser extension nightly build
VsceNightly // vs code extension nightly build
AppSnapshotRelease // app snapshot build
// Release branches
@ -107,6 +108,12 @@ func (t RunType) Matcher() *RunTypeMatcher {
BranchExact: true,
}
case AppSnapshotRelease:
return &RunTypeMatcher{
Branch: "app/release-snapshot",
BranchExact: true,
}
case TaggedRelease:
return &RunTypeMatcher{
TagPrefix: "v",
@ -175,6 +182,8 @@ func (t RunType) String() string {
return "Browser extension nightly release build"
case VsceNightly:
return "VS Code extension nightly release build"
case AppSnapshotRelease:
return "App snapshot release"
case TaggedRelease:
return "Tagged release"
case ReleaseBranch:

6
dev/ci/runtype/runtype_test.go
View File

@ -66,6 +66,12 @@ func TestComputeRunType(t *testing.T) {
branch: "vsce/release",
},
want: VsceReleaseBranch,
}, {
name: "app release",
args: args{
branch: "app/release-snapshot",
},
want: AppSnapshotRelease,
}, {
name: "release nightly",
args: args{

7
doc/dependency_decisions.yml
View File

@ -452,3 +452,10 @@
:why: Inference broken, LICENSE file lives at https://www.npmjs.com/package/stdin#license
:versions: []
:when: 2023-01-09 04:58:41.642155000 Z
- - :license
- github.com/xi2/xz
- Public domain
- :who:
:why:
:versions: []
:when: 2023-01-12 01:20:17.504279000 Z

9
doc/dev/background-information/ci/reference.md
View File

@ -133,6 +133,15 @@ Base pipeline (more steps might be included based on branch changes):
- Stylelint (all)
- Upload build trace
### App snapshot release
The run type for branches matching `app/release-snapshot` (exact match).
Base pipeline (more steps might be included based on branch changes):
- App release
- Upload build trace
### Tagged release
The run type for tags starting with `v`.

4
doc/dev/background-information/sg/reference.md
View File

@ -37,10 +37,12 @@ Available comamndsets in `sg.config.yaml`:
* enterprise-codeinsights
* enterprise-codeintel 🧠
* enterprise-e2e
* enterprise-single-program
* iam
* monitoring
* monitoring-alerts
* oss
* oss-single-program
* oss-web-standalone
* oss-web-standalone-prod
* otel
@ -122,6 +124,8 @@ Available commands in `sg.config.yaml`:
* repo-updater
* searcher
* server: Run an all-in-one sourcegraph/server image
* sourcegraph-oss: Single program (Go static binary) distribution, OSS variant
* sourcegraph: Single program (Go static binary) distribution
* storybook
* symbols
* syntax-highlighter

12
enterprise/cmd/executor/internal/command/observability.go
View File

@ -28,7 +28,7 @@ type Operations struct {
}
func NewOperations(observationCtx *observation.Context) *Operations {
metrics := metrics.NewREDMetrics(
redMetrics := metrics.NewREDMetrics(
observationCtx.Registerer,
"apiworker_command",
metrics.WithLabels("op"),
@ -39,7 +39,7 @@ func NewOperations(observationCtx *observation.Context) *Operations {
return observationCtx.Operation(observation.Op{
Name: fmt.Sprintf("apiworker.%s", opName),
MetricLabelValues: []string{opName},
Metrics: metrics,
Metrics: redMetrics,
})
}
@ -47,13 +47,17 @@ func NewOperations(observationCtx *observation.Context) *Operations {
Name: "src_executor_run_lock_wait_total",
Help: "The number of milliseconds spent waiting for the run lock.",
})
observationCtx.Registerer.MustRegister(runLockWaitTotal)
// TODO(sqs): TODO(single-binary): We use IgnoreDuplicate here to allow running 2 executor instances in
// the same process, but ideally we shouldn't need IgnoreDuplicate as that is a bit of a hack.
runLockWaitTotal = metrics.MustRegisterIgnoreDuplicate(observationCtx.Registerer, runLockWaitTotal)
runLockHeldTotal := prometheus.NewCounter(prometheus.CounterOpts{
Name: "src_executor_run_lock_held_total",
Help: "The number of milliseconds spent holding the run lock.",
})
observationCtx.Registerer.MustRegister(runLockHeldTotal)
// TODO(sqs): TODO(single-binary): We use IgnoreDuplicate here to allow running 2 executor instances in
// the same process, but ideally we shouldn't need IgnoreDuplicate as that is a bit of a hack.
runLockHeldTotal = metrics.MustRegisterIgnoreDuplicate(observationCtx.Registerer, runLockHeldTotal)
return &Operations{
SetupGitInit: op("setup.git.init"),

36
enterprise/cmd/executor/internal/run/util.go
View File

@ -1,6 +1,7 @@
package run
import (
"bytes"
"context"
"fmt"
"net/url"
@ -19,6 +20,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/internal/workerutil"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func newQueueTelemetryOptions(ctx context.Context, useFirecracker bool, logger log.Logger) queue.TelemetryOptions {
@ -56,39 +58,39 @@ func newQueueTelemetryOptions(ctx context.Context, useFirecracker bool, logger l
}
func getGitVersion(ctx context.Context) (string, error) {
cmd := exec.CommandContext(ctx, "git", "version")
out, err := cmd.Output()
out, err := execOutput(ctx, "git", "version")
if err != nil {
return "", err
}
return strings.TrimPrefix(strings.TrimSpace(string(out)), "git version "), nil
return strings.TrimPrefix(out, "git version "), nil
}
func getSrcVersion(ctx context.Context) (string, error) {
cmd := exec.CommandContext(ctx, "src", "version", "-client-only")
out, err := cmd.Output()
out, err := execOutput(ctx, "src", "version", "-client-only")
if err != nil {
return "", err
}
return strings.TrimPrefix(strings.TrimSpace(string(out)), "Current version: "), nil
return strings.TrimPrefix(out, "Current version: "), nil
}
func getDockerVersion(ctx context.Context) (string, error) {
cmd := exec.CommandContext(ctx, "docker", "version", "-f", "{{.Server.Version}}")
out, err := cmd.Output()
if err != nil {
return "", err
}
return strings.TrimSpace(string(out)), nil
return execOutput(ctx, "docker", "version", "-f", "{{.Server.Version}}")
}
func getIgniteVersion(ctx context.Context) (string, error) {
cmd := exec.CommandContext(ctx, "ignite", "version", "-o", "short")
out, err := cmd.Output()
if err != nil {
return "", err
return execOutput(ctx, "ignite", "version", "-o", "short")
}
func execOutput(ctx context.Context, name string, args ...string) (string, error) {
var buf bytes.Buffer
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stderr = &buf
cmd.Stdout = &buf
if err := cmd.Run(); err != nil {
cmdLine := strings.Join(append([]string{name}, args...), " ")
return "", errors.Wrap(err, fmt.Sprintf("'%s': %s", cmdLine, buf.String()))
}
return strings.TrimSpace(string(out)), nil
return strings.TrimSpace(buf.String()), nil
}
func apiWorkerOptions(c *config.Config, queueTelemetryOptions queue.TelemetryOptions) apiworker.Options {

5
enterprise/cmd/executor/main.go
View File

@ -1,9 +1,10 @@
package main
import (
shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd/executorcmd"
)
func main() {
shared.Main()
executorcmd.DeprecatedSingleServiceMainEnterprise(shared.Service)
}

47
enterprise/cmd/executor/shared/service.go Normal file
View File

@ -0,0 +1,47 @@
package shared
import (
"context"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/config"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "executor" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
var config config.Config
config.Load()
return &config, nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, cfg env.Config) error {
config := cfg.(*config.Config)
// TODO(sqs) HACK(sqs): run executors for both queues
if deploy.IsDeployTypeSingleProgram(deploy.Type()) {
otherConfig := *config
if config.QueueName == "batches" {
otherConfig.QueueName = "codeintel"
} else {
otherConfig.QueueName = "batches"
}
go func() {
if err := Main(ctx, observationCtx, ready, &otherConfig); err != nil {
observationCtx.Logger.Fatal("executor for other queue failed", log.Error(err))
}
}()
}
return Main(ctx, observationCtx, ready, config)
}
var Service service.Service = svc{}

27
enterprise/cmd/executor/shared/shared.go
View File

@ -12,33 +12,19 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/config"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/run"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/version"
// This import is required to force a binary hash change when the src-cli version is bumped.
_ "github.com/sourcegraph/sourcegraph/internal/src-cli"
)
func Main() {
cfg := &config.Config{}
cfg.Load()
env.Lock()
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
})
defer liblog.Sync()
logger := log.Scoped("executor", "the executor service polls the public Sourcegraph frontend API for work to perform")
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, cfg *config.Config) error {
makeActionHandler := func(handler func(cliCtx *cli.Context, logger log.Logger, config *config.Config) error) func(*cli.Context) error {
return func(ctx *cli.Context) error {
return handler(ctx, logger, cfg)
return handler(ctx, observationCtx.Logger, cfg)
}
}
@ -159,8 +145,5 @@ func Main() {
},
}
if err := app.RunContext(context.Background(), os.Args); err != nil {
println(err.Error())
os.Exit(1)
}
return app.RunContext(ctx, os.Args)
}

2
enterprise/cmd/frontend/internal/codeintel/init.go
View File

@ -20,7 +20,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/observation"
)
func init() {
func LoadConfig() {
ConfigInst.Load()
}

36
enterprise/cmd/frontend/internal/executorqueue/init.go
View File

@ -2,11 +2,14 @@ package executorqueue
import (
"context"
"strconv"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/env"
metricsstore "github.com/sourcegraph/sourcegraph/internal/metrics/store"
"github.com/sourcegraph/sourcegraph/internal/observation"
@ -16,6 +19,16 @@ import (
codeintelqueue "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/executorqueue/queues/codeintel"
)
func queueDisableAccessTokenDefault() string {
isSingleProgram := deploy.IsDeployTypeSingleProgram(deploy.Type())
if isSingleProgram {
return "true"
}
return "false"
}
var queueDisableAccessToken = env.Get("EXECUTOR_QUEUE_DISABLE_ACCESS_TOKEN_INSECURE", queueDisableAccessTokenDefault(), "Disable usage of an access token between executors and Sourcegraph (DANGEROUS")
// Init initializes the executor endpoints required for use with the executor service.
func Init(
ctx context.Context,
@ -27,9 +40,30 @@ func Init(
codeintelUploadHandler := enterpriseServices.NewCodeIntelUploadHandler(false)
batchesWorkspaceFileGetHandler := enterpriseServices.BatchesChangesFileGetHandler
batchesWorkspaceFileExistsHandler := enterpriseServices.BatchesChangesFileGetHandler
accessToken := func() string { return conf.SiteConfig().ExecutorsAccessToken }
logger := log.Scoped("executorqueue", "")
accessToken := func() (token string, accessTokenEnabled bool) {
token = conf.SiteConfig().ExecutorsAccessToken
wantDisableAccessToken, _ := strconv.ParseBool(queueDisableAccessToken)
if wantDisableAccessToken {
isSingleProgram := deploy.IsDeployTypeSingleProgram(deploy.Type())
isSingleDockerContainer := deploy.IsDeployTypeSingleDockerContainer(deploy.Type())
allowedDeployType := isSingleProgram || isSingleDockerContainer || env.InsecureDev
if allowedDeployType && token == "" {
// Disable the access token.
return "", false
}
// Respect the access token.
logger.Warn("access token may only be disabled if executors.accessToken is empty in site config AND the deployment type is single-program, single-docker-container, or dev")
return token, true
}
// Respect the access token.
return token, true
}
metricsStore := metricsstore.NewDistributedStore("executors:")
executorStore := db.Executors()

7
enterprise/cmd/frontend/internal/executorqueue/queuehandler.go
View File

@ -17,7 +17,7 @@ import (
metricsstore "github.com/sourcegraph/sourcegraph/internal/metrics/store"
)
func newExecutorQueueHandler(logger log.Logger, db database.DB, queueHandlers []handler.ExecutorHandler, accessToken func() string, uploadHandler http.Handler, batchesWorkspaceFileGetHandler http.Handler, batchesWorkspaceFileExistsHandler http.Handler) func() http.Handler {
func newExecutorQueueHandler(logger log.Logger, db database.DB, queueHandlers []handler.ExecutorHandler, accessToken func() (token string, enabled bool), uploadHandler http.Handler, batchesWorkspaceFileGetHandler http.Handler, batchesWorkspaceFileExistsHandler http.Handler) func() http.Handler {
metricsStore := metricsstore.NewDistributedStore("executors:")
executorStore := db.Executors()
gitserverClient := gitserver.NewClient(db)
@ -54,9 +54,10 @@ func newExecutorQueueHandler(logger log.Logger, db database.DB, queueHandlers []
// with the correct "token-executor <token>" value. This should only be used
// for internal _services_, not users, in which a shared key exchange can be
// done so safely.
func authMiddleware(accessToken func() string, next http.Handler) http.Handler {
func authMiddleware(accessToken func() (token string, enabled bool), next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if validateExecutorToken(w, r, accessToken()) {
token, tokenEnabled := accessToken()
if !tokenEnabled || validateExecutorToken(w, r, token) {
next.ServeHTTP(w, r)
}
})

2
enterprise/cmd/frontend/internal/executorqueue/queuehandler_test.go
View File

@ -12,7 +12,7 @@ func TestInternalProxyAuthTokenMiddleware(t *testing.T) {
accessToken := "hunter2"
ts := httptest.NewServer(authMiddleware(
func() string { return accessToken },
func() (token string, tokenEnabled bool) { return accessToken, true },
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusTeapot)
}),

2
enterprise/cmd/frontend/internal/executorqueue/queues/batches/queue.go
View File

@ -13,7 +13,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/observation"
)
func QueueOptions(observationCtx *observation.Context, db database.DB, _ func() string) handler.QueueOptions[*btypes.BatchSpecWorkspaceExecutionJob] {
func QueueOptions(observationCtx *observation.Context, db database.DB, _ func() (token string, tokenEnabled bool)) handler.QueueOptions[*btypes.BatchSpecWorkspaceExecutionJob] {
logger := log.Scoped("executor-queue.batches", "The executor queue handlers for the batches queue")
recordTransformer := func(ctx context.Context, version string, record *btypes.BatchSpecWorkspaceExecutionJob, _ handler.ResourceMetadata) (apiclient.Job, error) {
batchesStore := store.New(db, observationCtx, nil)

4
enterprise/cmd/frontend/internal/executorqueue/queues/codeintel/queue.go
View File

@ -12,9 +12,9 @@ import (
"github.com/sourcegraph/sourcegraph/internal/workerutil/dbworker/store"
)
func QueueOptions(observationCtx *observation.Context, db database.DB, accessToken func() string) handler.QueueOptions[types.Index] {
func QueueOptions(observationCtx *observation.Context, db database.DB, accessToken func() (token string, tokenEnabled bool)) handler.QueueOptions[types.Index] {
recordTransformer := func(ctx context.Context, _ string, record types.Index, resourceMetadata handler.ResourceMetadata) (apiclient.Job, error) {
return transformRecord(ctx, db, record, resourceMetadata, accessToken())
return transformRecord(ctx, db, record, resourceMetadata, accessToken)
}
store := store.New(observationCtx, db.Handle(), autoindexing.IndexWorkerStoreOptions)

7
enterprise/cmd/frontend/internal/executorqueue/queues/codeintel/transform.go
View File

@ -38,7 +38,7 @@ func (e *accessLogTransformer) Create(ctx context.Context, log *database.Executo
return e.ExecutorSecretAccessLogCreator.Create(ctx, log)
}
func transformRecord(ctx context.Context, db database.DB, index types.Index, resourceMetadata handler.ResourceMetadata, accessToken string) (apiclient.Job, error) {
func transformRecord(ctx context.Context, db database.DB, index types.Index, resourceMetadata handler.ResourceMetadata, accessToken func() (token string, tokenEnabled bool)) (apiclient.Job, error) {
resourceEnvironment := makeResourceEnvironment(resourceMetadata)
var secrets []*database.ExecutorSecret
@ -94,8 +94,9 @@ func transformRecord(ctx context.Context, db database.DB, index types.Index, res
})
}
token, _ := accessToken()
frontendURL := conf.ExecutorsFrontendURL()
authorizationHeader := makeAuthHeaderValue(accessToken)
authorizationHeader := makeAuthHeaderValue(token)
redactedAuthorizationHeader := makeAuthHeaderValue("REDACTED")
srcCliImage := fmt.Sprintf("%s:%s", conf.ExecutorsSrcCLIImage(), conf.ExecutorsSrcCLIImageTag())
@ -145,7 +146,7 @@ func transformRecord(ctx context.Context, db database.DB, index types.Index, res
// Authorization header to src-cli, which we trust not to ship the
// values to a third party, but not to trust to ensure the values
// are absent from the command's stdout or stderr streams.
accessToken: "PASSWORD_REMOVED",
token: "PASSWORD_REMOVED",
}
// 🚨 SECURITY: Catch uses of executor secrets from the executor secret store
maps.Copy(allRedactedValues, redactedEnvVars)

12
enterprise/cmd/frontend/internal/executorqueue/queues/codeintel/transform_test.go
View File

@ -80,7 +80,8 @@ func TestTransformRecord(t *testing.T) {
conf.Mock(nil)
})
job, err := transformRecord(context.Background(), db, index, testCase.resourceMetadata, "hunter2")
accessToken := func() (token string, tokenEnabled bool) { return "hunter2", true }
job, err := transformRecord(context.Background(), db, index, testCase.resourceMetadata, accessToken)
if err != nil {
t.Fatalf("unexpected error transforming record: %s", err)
}
@ -176,7 +177,8 @@ func TestTransformRecordWithoutIndexer(t *testing.T) {
conf.Mock(nil)
})
job, err := transformRecord(context.Background(), db, index, handler.ResourceMetadata{}, "hunter2")
accessToken := func() (token string, tokenEnabled bool) { return "hunter2", true }
job, err := transformRecord(context.Background(), db, index, handler.ResourceMetadata{}, accessToken)
if err != nil {
t.Fatalf("unexpected error transforming record: %s", err)
}
@ -307,7 +309,8 @@ func TestTransformRecordWithSecrets(t *testing.T) {
conf.Mock(nil)
})
job, err := transformRecord(context.Background(), db, index, testCase.resourceMetadata, "hunter2")
accessToken := func() (token string, tokenEnabled bool) { return "hunter2", true }
job, err := transformRecord(context.Background(), db, index, testCase.resourceMetadata, accessToken)
if err != nil {
t.Fatalf("unexpected error transforming record: %s", err)
}
@ -391,7 +394,8 @@ func TestTransformRecordDockerAuthConfig(t *testing.T) {
}, 0, nil)
db.ExecutorSecretAccessLogsFunc.SetDefaultReturn(database.NewMockExecutorSecretAccessLogStore())
job, err := transformRecord(context.Background(), db, types.Index{ID: 42}, handler.ResourceMetadata{}, "hunter2")
accessToken := func() (token string, tokenEnabled bool) { return "hunter2", true }
job, err := transformRecord(context.Background(), db, types.Index{ID: 42}, handler.ResourceMetadata{}, accessToken)
if err != nil {
t.Fatal(err)
}

16
enterprise/cmd/frontend/main.go
View File

@ -2,20 +2,10 @@
package main
import (
shared "github.com/sourcegraph/sourcegraph/cmd/frontend/shared"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/registry"
shared_enterprise "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/shared"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/oobmigration"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
)
func main() {
env.Lock()
env.HandleHelpFlag()
shared.Main(shared_enterprise.EnterpriseSetupHook)
}
func init() {
oobmigration.ReturnEnterpriseMigrations = true
enterprisecmd.DeprecatedSingleServiceMainEnterprise(shared.Service)
}

31
enterprise/cmd/frontend/shared/service.go Normal file
View File

@ -0,0 +1,31 @@
package shared
import (
"context"
frontend_shared "github.com/sourcegraph/sourcegraph/cmd/frontend/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/codeintel"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
_ "github.com/sourcegraph/sourcegraph/cmd/frontend/registry/api"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/registry"
)
type svc struct{}
func (svc) Name() string { return "frontend" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
frontend_shared.CLILoadConfig()
codeintel.LoadConfig()
return nil, nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return frontend_shared.CLIMain(ctx, observationCtx, ready, EnterpriseSetupHook)
}
var Service service.Service = svc{}

11
enterprise/cmd/gitserver/main.go
View File

@ -1,14 +1,11 @@
// Command frontend is the enterprise frontend program.
package main
import (
"github.com/sourcegraph/sourcegraph/cmd/gitserver/shared"
enterprise_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver/shared"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
)
func main() {
env.Lock()
env.HandleHelpFlag()
shared.Main(enterprise_shared.EnterpriseInit)
enterprisecmd.DeprecatedSingleServiceMainEnterprise(shared.Service)
}

25
enterprise/cmd/gitserver/shared/service.go Normal file
View File

@ -0,0 +1,25 @@
package shared
import (
"context"
shared "github.com/sourcegraph/sourcegraph/cmd/gitserver/shared"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "gitserver" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
return shared.LoadConfig(), nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return shared.Main(ctx, observationCtx, ready, config.(*shared.Config), enterpriseInit)
}
var Service service.Service = svc{}

2
enterprise/cmd/gitserver/shared/shared.go
View File

@ -13,7 +13,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/database"
)
func EnterpriseInit(db database.DB) {
func enterpriseInit(db database.DB) {
logger := log.Scoped("enterprise", "gitserver enterprise edition")
var err error
authz.DefaultSubRepoPermsChecker, err = srp.NewSubRepoPermsClient(edb.NewEnterpriseDB(db).SubRepoPerms())

5
enterprise/cmd/precise-code-intel-worker/main.go
View File

@ -1,9 +1,10 @@
package main
import (
shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
)
func main() {
shared.Main()
enterprisecmd.DeprecatedSingleServiceMainEnterprise(shared.Service)
}

28
enterprise/cmd/precise-code-intel-worker/shared/service.go Normal file
View File

@ -0,0 +1,28 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/symbols"
)
type svc struct{}
func (svc) Name() string { return "precise-code-intel-worker" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
symbols.LoadConfig()
var config Config
config.Load()
return &config, nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return Main(ctx, observationCtx, ready, *config.(*Config))
}
var Service service.Service = svc{}

61
enterprise/cmd/precise-code-intel-worker/shared/shared.go
View File

@ -21,59 +21,28 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/encryption/keyring"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/honey"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/profiler"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/uploadstore"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
const addr = ":3188"
func Main() {
config := &Config{}
config.Load()
env.Lock()
env.HandleHelpFlag()
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
})
defer liblog.Sync()
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config Config) error {
logger := observationCtx.Logger
// Initialize tracing/metrics
logger := log.Scoped("codeintel-worker", "The precise-code-intel-worker service converts LSIF upload file into Postgres data.")
observationCtx := observation.NewContext(logger, observation.Honeycomb(&honey.Dataset{
observationCtx = observation.NewContext(logger, observation.Honeycomb(&honey.Dataset{
Name: "codeintel-worker",
}))
conf.Init()
go conf.Watch(liblog.Update(conf.GetLogSinks))
tracer.Init(logger.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
profiler.Init()
if err := config.Validate(); err != nil {
logger.Error("Failed for load config", log.Error(err))
}
// Start debug server
ready := make(chan struct{})
go debugserver.NewServerRoutine(ready).Start()
if err := keyring.Init(context.Background()); err != nil {
logger.Fatal("Failed to intialise keyring", log.Error(err))
if err := keyring.Init(ctx); err != nil {
return errors.Wrap(err, "initializing keyring")
}
// Connect to databases
@ -83,13 +52,13 @@ func Main() {
// Migrations may take a while, but after they're done we'll immediately
// spin up a server and can accept traffic. Inform external clients we'll
// be ready for traffic.
close(ready)
ready()
// Initialize sub-repo permissions client
var err error
authz.DefaultSubRepoPermsChecker, err = srp.NewSubRepoPermsClient(edb.NewEnterpriseDB(db).SubRepoPerms())
if err != nil {
logger.Fatal("Failed to create sub-repo client", log.Error(err))
return errors.Wrap(err, "creating sub-repo client")
}
services, err := codeintel.NewServices(codeintel.ServiceDependencies{
@ -98,16 +67,16 @@ func Main() {
ObservationCtx: observationCtx,
})
if err != nil {
logger.Fatal("Failed to create codeintel services", log.Error(err))
return errors.Wrap(err, "creating codeintel services")
}
// Initialize stores
uploadStore, err := lsifuploadstore.New(context.Background(), observationCtx, config.LSIFUploadStoreConfig)
uploadStore, err := lsifuploadstore.New(ctx, observationCtx, config.LSIFUploadStoreConfig)
if err != nil {
logger.Fatal("Failed to create upload store", log.Error(err))
return errors.Wrap(err, "creating upload store")
}
if err := initializeUploadStore(context.Background(), uploadStore); err != nil {
logger.Fatal("Failed to initialize upload store", log.Error(err))
if err := initializeUploadStore(ctx, uploadStore); err != nil {
return errors.Wrap(err, "initializing upload store")
}
// Initialize worker
@ -130,7 +99,9 @@ func Main() {
})
// Go!
goroutine.MonitorBackgroundRoutines(context.Background(), worker, server)
goroutine.MonitorBackgroundRoutines(ctx, worker, server)
return nil
}
func mustInitializeDB(observationCtx *observation.Context) *sql.DB {

10
enterprise/cmd/repo-updater/main.go
View File

@ -1,14 +1,10 @@
package main
import (
"github.com/sourcegraph/sourcegraph/cmd/repo-updater/shared"
enterprise_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater/shared"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
)
func main() {
env.Lock()
env.HandleHelpFlag()
shared.Main(enterprise_shared.EnterpriseInit)
enterprisecmd.DeprecatedSingleServiceMainEnterprise(shared.Service)
}

8
enterprise/cmd/repo-updater/shared/service.go Normal file
View File

@ -0,0 +1,8 @@
package shared
import (
shared "github.com/sourcegraph/sourcegraph/cmd/repo-updater/shared"
"github.com/sourcegraph/sourcegraph/internal/service"
)
var Service service.Service = shared.NewServiceWithEnterpriseInit(EnterpriseInit)

45
enterprise/cmd/sourcegraph/README.md Normal file
View File

@ -0,0 +1,45 @@
# Sourcegraph App
Sourcegraph App is a single-binary distribution of Sourcegraph that runs on your local machine.
**Status:** alpha (only for internal use at Sourcegraph)
## Development
```shell
sg start enterprise-single-program
```
## Usage
Sourcegraph App is in alpha (only for internal use at Sourcegraph).
Check the **Sourcegraph App release** bot in [`#app`](https://app.slack.com/client/T02FSM7DL/C04F9E7GUDP) (in the Sourcegraph internal Slack) for the latest release information.
## Build and release
### Snapshot releases
> Sourcegraph App is in internal alpha and only has snapshot releases. There are no versioned or tagged releases yet.
To build and release a snapshot for other people to use, push a commit to the special `app/release-snapshot` branch:
```shell
git push -f origin HEAD:app/release-snapshot
```
This runs the `../../dev/app/release.sh` script in CI, which uses [goreleaser](https://goreleaser.com/) to build for many platforms, package, and publish to the `sourcegraph-app-releases` Google Cloud Storage bucket.
Check the build status in [Buildkite `app/release-snapshot` branch builds](https://buildkite.com/sourcegraph/sourcegraph/builds?branch=app%2Frelease-snapshot).
### Local builds (without releasing)
To build it locally for all platforms (without releasing, uploading, or publishing it anywhere), run:
```shell
enterprise/dev/app/release.sh --snapshot
```
The builds are written to the `dist` directory.
If you just need a local build for your current platform, run `sg start enterprise-single-program` (as mentioned in the [Development](#development) section) and then grab the `.bin/sourcegraph` binary. This binary does not have the web bundle (JavaScript/CSS) embedded into it.

30
enterprise/cmd/sourcegraph/enterprisecmd/enterprisecmd.go Normal file
View File

@ -0,0 +1,30 @@
// Package enterprisecmd defines entrypoint functions for the enterprise (non-OSS) build of
// Sourcegraph's single-program distribution. It is invoked by all enterprise (non-OSS) commands'
// main functions.
package enterprisecmd
import (
"github.com/sourcegraph/sourcegraph/internal/oobmigration"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/service/svcmain"
)
var config = svcmain.Config{}
// MainEnterprise is called from the `main` function of the `sourcegraph` command.
func MainEnterprise(services []service.Service) {
svcmain.Main(services, config)
}
// DeprecatedSingleServiceMainEnterprise is called from the `main` function of a command in the
// enterprise (non-OSS) build to start a single service (such as frontend or gitserver).
//
// DEPRECATED: See svcmain.DeprecatedSingleServiceMain documentation for more info.
func DeprecatedSingleServiceMainEnterprise(service service.Service) {
svcmain.DeprecatedSingleServiceMain(service, config, true, true)
}
func init() {
// TODO(sqs): TODO(single-binary): could we move this out of init?
oobmigration.ReturnEnterpriseMigrations = true
}

21
enterprise/cmd/sourcegraph/enterprisecmd/executorcmd/executorcmd.go Normal file
View File

@ -0,0 +1,21 @@
// Package executorcmd similar to enterprisecmd, except that it has customizations specific to the
// executor command. The executor command (1) does not connect to a database, and so dbconn is a
// a forbidden import and (2) is not just a service (it has commands like `executor install all`)
// which means environment variable configuration is not always present, and as such that must not
// be enforced in a standard way like in our other service cmds.
package executorcmd
import (
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/service/svcmain"
)
var config = svcmain.Config{}
// DeprecatedSingleServiceMainEnterprise is called from the `main` function of a command in the
// enterprise (non-OSS) build to start a single service (such as frontend or gitserver).
//
// DEPRECATED: See svcmain.DeprecatedSingleServiceMain documentation for more info.
func DeprecatedSingleServiceMainEnterprise(service service.Service) {
svcmain.DeprecatedSingleServiceMain(service, config, false, false)
}

33
enterprise/cmd/sourcegraph/main.go Normal file
View File

@ -0,0 +1,33 @@
package main
import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
"github.com/sourcegraph/sourcegraph/internal/service"
githubproxy_shared "github.com/sourcegraph/sourcegraph/cmd/github-proxy/shared"
searcher_shared "github.com/sourcegraph/sourcegraph/cmd/searcher/shared"
executor_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/shared"
frontend_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/shared"
gitserver_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/gitserver/shared"
precise_code_intel_worker_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker/shared"
repoupdater_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater/shared"
symbols_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols/shared"
worker_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/shared"
)
// services is a list of services to run in the enterprise build.
var services = []service.Service{
frontend_shared.Service,
gitserver_shared.Service,
repoupdater_shared.Service,
searcher_shared.Service,
symbols_shared.Service,
worker_shared.Service,
githubproxy_shared.Service,
precise_code_intel_worker_shared.Service,
executor_shared.Service,
}
func main() {
enterprisecmd.MainEnterprise(services)
}

6
enterprise/cmd/symbols/main.go
View File

@ -1,10 +1,10 @@
package main
import (
"github.com/sourcegraph/sourcegraph/cmd/symbols/shared"
enterpriseshared "github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols/shared"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols/shared"
)
func main() {
shared.Main(enterpriseshared.CreateSetup(shared.CtagsConfig, shared.RepositoryFetcherConfig))
enterprisecmd.DeprecatedSingleServiceMainEnterprise(shared.Service)
}

28
enterprise/cmd/symbols/shared/service.go Normal file
View File

@ -0,0 +1,28 @@
package shared
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
symbols_shared "github.com/sourcegraph/sourcegraph/cmd/symbols/shared"
)
type svc struct{}
func (svc) Name() string { return "symbols" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
symbols_shared.LoadConfig()
config := loadRockskipConfig(env.BaseConfig{}, symbols_shared.CtagsConfig, symbols_shared.RepositoryFetcherConfig)
return &config, nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
return symbols_shared.Main(ctx, observationCtx, ready, CreateSetup(*config.(*rockskipConfig)))
}
var Service service.Service = svc{}

14
enterprise/cmd/symbols/shared/setup.go
View File

@ -3,7 +3,6 @@ package shared
import (
"context"
"database/sql"
stdlog "log"
"net/http"
"strings"
@ -36,13 +35,7 @@ var (
minRepoSizeMb = env.MustGetInt("ROCKSKIP_MIN_REPO_SIZE_MB", -1, "all repos that are at least this big will be indexed using Rockskip")
)
func CreateSetup(ctags types.CtagsConfig, repositoryFetcher types.RepositoryFetcherConfig) shared.SetupFunc {
baseConfig := env.BaseConfig{}
config := loadRockskipConfig(baseConfig, ctags, repositoryFetcher)
if err := baseConfig.Validate(); err != nil {
stdlog.Fatal("failed to load configuration:", err)
}
func CreateSetup(config rockskipConfig) shared.SetupFunc {
repoToSize := map[string]int64{}
if useRockskip {
@ -99,6 +92,7 @@ func CreateSetup(ctags types.CtagsConfig, repositoryFetcher types.RepositoryFetc
}
type rockskipConfig struct {
env.BaseConfig
Ctags types.CtagsConfig
RepositoryFetcher types.RepositoryFetcherConfig
MaxRepos int
@ -110,6 +104,10 @@ type rockskipConfig struct {
SearchLastIndexedCommit bool
}
func (c *rockskipConfig) Load() {
// TODO(sqs): TODO(single-binary): load rockskip config from here
}
func loadRockskipConfig(baseConfig env.BaseConfig, ctags types.CtagsConfig, repositoryFetcher types.RepositoryFetcherConfig) rockskipConfig {
return rockskipConfig{
Ctags: ctags,

47
enterprise/cmd/worker/main.go
View File

@ -1,51 +1,10 @@
package main
import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/cmd/worker/shared"
enterprise_shared "github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/shared"
srp "github.com/sourcegraph/sourcegraph/enterprise/internal/authz/subrepoperms"
edb "github.com/sourcegraph/sourcegraph/enterprise/internal/database"
"github.com/sourcegraph/sourcegraph/enterprise/internal/oobmigration/migrations"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/oobmigration"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph/enterprisecmd"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/shared"
)
func getEnterpriseInit(logger log.Logger) func(database.DB) {
return func(ossDB database.DB) {
enterpriseDB := edb.NewEnterpriseDB(ossDB)
var err error
authz.DefaultSubRepoPermsChecker, err = srp.NewSubRepoPermsClient(enterpriseDB.SubRepoPerms())
if err != nil {
logger.Fatal("Failed to create sub-repo client", log.Error(err))
}
}
}
func main() {
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
})
defer liblog.Sync()
logger := log.Scoped("worker", "worker enterprise edition")
observationCtx := observation.NewContext(logger)
go enterprise_shared.SetAuthzProviders(observationCtx)
if err := shared.Start(observationCtx, enterprise_shared.AdditionalJobs, migrations.RegisterEnterpriseMigrators, getEnterpriseInit(logger)); err != nil {
logger.Fatal(err.Error())
}
}
func init() {
oobmigration.ReturnEnterpriseMigrations = true
enterprisecmd.DeprecatedSingleServiceMainEnterprise(shared.Service)
}

27
enterprise/cmd/worker/shared/service.go Normal file
View File

@ -0,0 +1,27 @@
package shared
import (
"context"
shared "github.com/sourcegraph/sourcegraph/cmd/worker/shared"
"github.com/sourcegraph/sourcegraph/enterprise/internal/oobmigration/migrations"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/service"
)
type svc struct{}
func (svc) Name() string { return "worker" }
func (svc) Configure() (env.Config, []debugserver.Endpoint) {
return shared.LoadConfig(additionalJobs, migrations.RegisterEnterpriseMigrators), nil
}
func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
go setAuthzProviders(ctx, observationCtx)
return shared.Start(ctx, observationCtx, ready, config.(*shared.Config), getEnterpriseInit(observationCtx.Logger))
}
var Service service.Service = svc{}

24
enterprise/cmd/worker/shared/shared.go
View File

@ -4,6 +4,7 @@ import (
"context"
"time"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/cmd/frontend/globals"
"github.com/sourcegraph/sourcegraph/cmd/worker/job"
workerdb "github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/db"
@ -16,13 +17,16 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/internal/permissions"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/internal/telemetry"
eiauthz "github.com/sourcegraph/sourcegraph/enterprise/internal/authz"
srp "github.com/sourcegraph/sourcegraph/enterprise/internal/authz/subrepoperms"
edb "github.com/sourcegraph/sourcegraph/enterprise/internal/database"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/extsvc/versions"
"github.com/sourcegraph/sourcegraph/internal/observation"
)
var AdditionalJobs = map[string]job.Job{
var additionalJobs = map[string]job.Job{
"codehost-version-syncing": versions.NewSyncingJob(),
"insights-job": workerinsights.NewInsightsJob(),
"insights-query-runner-job": workerinsights.NewInsightsQueryRunnerJob(),
@ -61,9 +65,8 @@ var AdditionalJobs = map[string]job.Job{
// current actor stored in an operation's context, which is likely an internal actor for many of
// the jobs configured in this service. This also enables repository update operations to fetch
// permissions from code hosts.
func SetAuthzProviders(observationCtx *observation.Context) {
func setAuthzProviders(ctx context.Context, observationCtx *observation.Context) {
observationCtx = observation.ContextWithLogger(observationCtx.Logger.Scoped("authz-provider", ""), observationCtx)
db, err := workerdb.InitDB(observationCtx)
if err != nil {
return
@ -72,10 +75,21 @@ func SetAuthzProviders(observationCtx *observation.Context) {
// authz also relies on UserMappings being setup.
globals.WatchPermissionsUserMapping()
ctx := context.Background()
for range time.NewTicker(eiauthz.RefreshInterval()).C {
allowAccessByDefault, authzProviders, _, _, _ := eiauthz.ProvidersFromConfig(ctx, conf.Get(), db.ExternalServices(), db)
authz.SetProviders(allowAccessByDefault, authzProviders)
}
}
func getEnterpriseInit(logger log.Logger) func(database.DB) {
return func(ossDB database.DB) {
enterpriseDB := edb.NewEnterpriseDB(ossDB)
var err error
authz.DefaultSubRepoPermsChecker, err = srp.NewSubRepoPermsClient(enterpriseDB.SubRepoPerms())
if err != nil {
logger.Fatal("Failed to create sub-repo client", log.Error(err))
}
}
}

132
enterprise/dev/app/goreleaser.yaml Normal file
View File

@ -0,0 +1,132 @@
project_name: sourcegraph
env:
- CGO_ENABLED=1
before:
hooks:
- go mod tidy
builds:
- id: build_linux
goos:
- linux
goarch:
- amd64
main: &gomain ./enterprise/cmd/sourcegraph
tags: &tags
- dist
ldflags: &ldflags
- -X github.com/sourcegraph/sourcegraph/internal/version.version={{.Version}}
- -X github.com/sourcegraph/sourcegraph/internal/version.timestamp={{.Timestamp}}
- -X github.com/sourcegraph/sourcegraph/internal/conf/deploy.forceType=single-program
flags: &goflags
- -trimpath
- -v
- id: build_macos
goos:
- darwin
goarch:
- amd64
- arm64
env:
- CC=o64-clang
- CXX=o64-clang++
main: *gomain
tags: *tags
ldflags: *ldflags
flags: *goflags
# TODO(sqs): Windows builds are broken due to compilation errors in: github.com/sourcegraph/mountinfo, github.com/coreos/go-iptables, github.com/sourcegraph/zoekt
#
# - id: build_windows
# goos:
# - windows
# goarch:
# - amd64
# env:
# - CC=x86_64-w64-mingw32-gcc
# - CXX=x86_64-w64-mingw32-g++
# main: *gomain
# tags: *tags
# ldflags: *ldflags
# flags: *goflags
universal_binaries:
- id: build_macos
archives:
- id: zip_archives
builds: ['build_linux', 'build_macos']
format: zip
# Just include the binary file. The only way to do this is to specify a glob that matches
# nothing.
files: ['NO_FILES_*']
checksum:
name_template: 'checksums.txt'
changelog:
skip: true
release:
github:
owner: sourcegraph
# TODO(sqs): use just sourcegraph/sourcegraph?
name: sourcegraph-app
draft: true
prerelease: auto
blobs:
- provider: gs
bucket: sourcegraph-app-releases
folder: "{{.Version}}"
ids:
- build_linux
- build_macos
- zip_archives
- linux_packages
nfpms:
- id: linux_packages
builds: ['build_linux']
formats:
- deb
- rpm
dependencies:
- git
- redis
vendor: "sourcegraph"
homepage: "https://github.com/sourcegraph/sourcegraph"
maintainer: "dev@sourcegraph.com"
description: "Code intelligence and search"
license: "Sourcegraph Enterprise License (portions licensed under Apache 2)"
brews:
- homepage: "https://github.com/sourcegraph/sourcegraph"
description: "Code intelligence and search"
license: "Sourcegraph Enterprise License (portions licensed under Apache 2)"
tap:
owner: sourcegraph
# TODO(sqs): use just sourcegraph/homebrew-sourcegraph
name: homebrew-sourcegraph-app
url_template: https://storage.googleapis.com/sourcegraph-app-releases/{{ .Tag }}/{{ .ArtifactName }}
commit_author:
name: sourcegraph-buildkite
email: 71296199+sourcegraph-buildkite@users.noreply.github.com
install: |
bin.install "sourcegraph"
test: |
system "#{bin}/sourcegraph --help"
dependencies:
- name: git
- name: redis
announce:
slack:
enabled: true
message_template: |
New version `{{.Version}}` (<https://sourcegraph.com/github.com/sourcegraph/sourcegraph/-/commit/{{.FullCommit}}|{{.ShortCommit}}>): `brew install sourcegraph/sourcegraph-app/sourcegraph`, <https://storage.googleapis.com/sourcegraph-app-releases/{{.Version}}/sourcegraph_{{.Version}}_darwin_all.zip|macOS .zip>, <https://storage.googleapis.com/sourcegraph-app-releases/{{.Version}}/sourcegraph_{{.Version}}_linux_amd64.zip|Linux .zip>, <https://storage.googleapis.com/sourcegraph-app-releases/{{.Version}}/sourcegraph_{{.Version}}_linux_amd64.deb|Linux .deb>. Winners ship, shippers win! :ship: :ship: :ship:
# TODO(sqs): add back `dockers`, `scoop`, `snapcraft` as needed

88
enterprise/dev/app/release.sh Executable file
View File

@ -0,0 +1,88 @@
#!/usr/bin/env bash
set -eu
ROOTDIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/../../..)"
GORELEASER_CROSS_VERSION=v1.19.5
GCLOUD_APP_CREDENTIALS_FILE=${GCLOUD_APP_CREDENTIALS_FILE-$HOME/.config/gcloud/application_default_credentials.json}
if [ -z "${SKIP_BUILD_WEB-}" ]; then
# Use esbuild because it's faster. This is just a personal preference by me (@sqs); if there is a
# good reason to change it, feel free to do so.
ENTERPRISE=1 DEV_WEB_BUILDER=esbuild pnpm run build-web
fi
if [ -z "${GITHUB_TOKEN-}" ]; then
echo "Error: GITHUB_TOKEN must be set."
exit 1
fi
if [ ! -f "$GCLOUD_APP_CREDENTIALS_FILE" ]; then
echo "Error: no gcloud application default credentials found. To obtain these credentials, first run:"
echo
echo " gcloud auth application-default login"
echo
echo "Or set GCLOUD_APP_CREDENTIALS_FILE to a file containing the credentials."
exit 1
fi
if [ -z "${VERSION-}" ]; then
echo "Error: VERSION must be set."
exit 1
fi
# Manually set the version because `git describe` (which goreleaser otherwise uses) prints the wrong
# version number because of how we use release branches
# (https://github.com/sourcegraph/sourcegraph/issues/46404).
GORELEASER_CURRENT_TAG=$VERSION
DOCKER_ARGS=()
if [ -z "${BUILDKITE-}" ]; then
DOCKER_VOLUME_SOURCE="$ROOTDIR"
else
# In Buildkite, we're running in a Docker container, so `docker run -v` needs to refer to a
# directory on our Docker host, not in our container. Use the /mnt/tmp directory, which is shared
# between `dind` (the Docker-in-Docker host) and our container.
TMPDIR=$(mktemp -d --tmpdir=/mnt/tmp -t sourcegraph.XXXXXXXX)
cleanup() {
rm -rf "$TMPDIR"
}
trap cleanup EXIT
# goreleaser expects a tag that corresponds to the version. When running in local dev, you can
# pass --skip-validate to skip this check, but in CI we want to run the other validations (such as
# checking that the Git checkout is not dirty).
git tag "$VERSION"
# Copy the ROOTDIR and GCLOUD_APP_CREDENTIALS_FILE to /mnt/tmp so they can be volume-mounted.
cp -R "$ROOTDIR" "$TMPDIR"
DOCKER_VOLUME_SOURCE="$TMPDIR/$(basename "$ROOTDIR")"
GCLOUD_APP_CREDENTIALS_TMP="$TMPDIR"/application_default_credentials.json
cp "$GCLOUD_APP_CREDENTIALS_FILE" "$GCLOUD_APP_CREDENTIALS_TMP"
GCLOUD_APP_CREDENTIALS_FILE="$GCLOUD_APP_CREDENTIALS_TMP"
# In Buildkite, we need to mount /buildkite-git-references because our .git directory refers to
# it. TODO(sqs): This is probably slow and undoes the optimization gained by using `git clone
# --reference`.
mkdir "$TMPDIR"/buildkite-git-references
cp -R /buildkite-git-references/sourcegraph.reference "$TMPDIR"/buildkite-git-references
DOCKER_ARGS+=(-v "$TMPDIR"/buildkite-git-references:/buildkite-git-references)
fi
GORELEASER_ARGS=()
if [ -z "${SLACK_APP_RELEASE_WEBHOOK-}" ]; then
GORELEASER_ARGS+=(--skip-announce)
else
DOCKER_ARGS+=(-e "SLACK_WEBHOOK=$SLACK_APP_RELEASE_WEBHOOK")
fi
# shellcheck disable=SC2086
exec docker run --rm \
${DOCKER_ARGS[*]} \
-v "$DOCKER_VOLUME_SOURCE":/go/src/github.com/sourcegraph/sourcegraph \
-w /go/src/github.com/sourcegraph/sourcegraph \
-v "$GCLOUD_APP_CREDENTIALS_FILE":/root/.config/gcloud/application_default_credentials.json \
-e "GITHUB_TOKEN=$GITHUB_TOKEN" \
-e "GORELEASER_CURRENT_TAG=$GORELEASER_CURRENT_TAG" \
goreleaser/goreleaser-cross:$GORELEASER_CROSS_VERSION \
--config enterprise/dev/app/goreleaser.yaml --parallelism 1 --debug --rm-dist ${GORELEASER_ARGS[*]} "$@"

1
enterprise/dev/ci/integration/executors/config/site-config.json
View File

@ -1,5 +1,4 @@
{
"executors.accessToken": "$EXECUTOR_FRONTEND_PASSWORD",
"executors.batcheshelperImage": "us.gcr.io/sourcegraph-dev/batcheshelper",
"executors.batcheshelperImageTag": "$CANDIDATE_VERSION",
"auth.providers": [

1
enterprise/dev/ci/integration/executors/docker-compose.yml
View File

@ -33,6 +33,7 @@ services:
SOURCEGRAPH_LICENSE_GENERATION_KEY: '${SOURCEGRAPH_LICENSE_GENERATION_KEY}'
SITE_CONFIG_FILE: /e2e/site-config.json
PGDATASOURCE: postgres://sg@postgres:5432/sg
EXECUTOR_QUEUE_DISABLE_ACCESS_TOKEN_INSECURE: 'true'
volumes:
- '${DATA}/config:/etc/sourcegraph'
- '${DATA}/data:/var/opt/sourcegraph'

2
enterprise/dev/ci/integration/executors/run.sh
View File

@ -27,7 +27,7 @@ trap cleanup EXIT
export POSTGRES_IMAGE="us.gcr.io/sourcegraph-dev/postgres-12-alpine:${CANDIDATE_VERSION}"
export SERVER_IMAGE="us.gcr.io/sourcegraph-dev/server:${CANDIDATE_VERSION}"
export EXECUTOR_IMAGE="us.gcr.io/sourcegraph-dev/executor:${CANDIDATE_VERSION}"
export EXECUTOR_FRONTEND_PASSWORD="hunter2hunter2hunter2"
export EXECUTOR_FRONTEND_PASSWORD=none
export SOURCEGRAPH_LICENSE_GENERATION_KEY="${SOURCEGRAPH_LICENSE_GENERATION_KEY:-""}"
export TMP_DIR
export DATA

19
enterprise/dev/ci/internal/ci/operations.go
View File

@ -569,6 +569,25 @@ func addVsceReleaseSteps(pipeline *bk.Pipeline) {
bk.Cmd("pnpm --filter @sourcegraph/vscode run release"))
}
// Release a snapshot of App.
func addAppSnapshotReleaseSteps(c Config) operations.Operation {
// TODO(sqs): Use goreleaser-pro nightly feature? Blocked on
// https://github.com/goreleaser/goreleaser-cross/issues/22.
// goreleaser requires that the version is semver-compatible
// (https://goreleaser.com/limitations/semver/). This is fine for now in alpha.
version := fmt.Sprintf("0.0.%d-snapshot+%s-%.6s", c.BuildNumber, c.Time.Format("20060102"), c.Commit)
return func(pipeline *bk.Pipeline) {
// Release App (.zip/.deb/.rpm to Google Cloud Storage, new tap for Homebrew, etc.).
pipeline.AddStep(":desktop_computer: App release",
withPnpmCache(),
bk.Cmd("pnpm install --frozen-lockfile --fetch-timeout 60000"),
bk.Env("VERSION", version),
bk.Cmd("enterprise/dev/ci/scripts/release-app.sh"))
}
}
// Adds a Buildkite pipeline "Wait".
func wait(pipeline *bk.Pipeline) {
pipeline.AddWait()

4
enterprise/dev/ci/internal/ci/pipeline.go
View File

@ -180,6 +180,10 @@ func GeneratePipeline(c Config) (*bk.Pipeline, error) {
// addVsceIntegrationTests,
)
case runtype.AppSnapshotRelease:
// If this is an App snapshot build, release a snapshot.
ops = operations.NewSet(addAppSnapshotReleaseSteps(c))
case runtype.ImagePatch:
// only build image for the specified image in the branch name
// see https://handbook.sourcegraph.com/engineering/deployments#building-docker-images-for-a-specific-branch

23
enterprise/dev/ci/scripts/release-app.sh Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -eu
# Calls the enterprise/dev/app/release.sh script with the right env vars.
GITHUB_TOKEN=$(gcloud secrets versions access latest --secret=BUILDKITE_GITHUBDOTCOM_TOKEN --quiet --project=sourcegraph-ci)
export GITHUB_TOKEN
# TODO(sqs): Make this non-optional (by removing ` || echo -n`) when https://github.com/sourcegraph/infrastructure/pull/4481 is merged.
SLACK_APP_RELEASE_WEBHOOK=$(gcloud secrets versions access latest --secret=SLACK_APP_RELEASE_WEBHOOK --quiet --project=sourcegraph-ci || echo -n)
export SLACK_APP_RELEASE_WEBHOOK
TMPFILE=$(mktemp)
gcloud secrets versions access latest --secret=BUILDKITE_GCLOUD_SERVICE_ACCOUNT --quiet --project=sourcegraph-ci > "$TMPFILE"
export GCLOUD_APP_CREDENTIALS_FILE=$TMPFILE
cleanup() {
rm "$TMPFILE"
}
trap cleanup EXIT
ROOTDIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/../../../..)"
exec "$ROOTDIR"/enterprise/dev/app/release.sh

4
go.mod
View File

@ -205,6 +205,7 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/prometheus v0.40.5 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/zenazn/goji v1.0.1 // indirect
go.opentelemetry.io/otel/metric v0.33.0 // indirect
@ -219,6 +220,7 @@ require (
github.com/coreos/go-iptables v0.6.0
github.com/crewjam/saml/samlidp v0.0.0-20221211125903-d951aa2d145a
github.com/dcadenas/pagerank v0.0.0-20171013173705-af922e3ceea8
github.com/fergusstrange/embedded-postgres v1.19.0
github.com/frankban/quicktest v1.14.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-github/v47 v47.1.0
@ -232,6 +234,8 @@ require (
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a
)
replace github.com/fergusstrange/embedded-postgres => github.com/sourcegraph/embedded-postgres v1.19.1-0.20230113234230-bb62ad58a1e1
require (
github.com/sourcegraph/zoekt v0.0.0-20230112115613-e0cf62d238b9
github.com/stretchr/objx v0.5.0 // indirect

4
go.sum
View File

@ -2084,6 +2084,8 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d h1:yKm7XZV6j9
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
github.com/sourcegraph/conc v0.1.0 h1:9GeYVmWWa1jeOq3zGq17m10d9pjYZpiGTj/N4hQFl58=
github.com/sourcegraph/conc v0.1.0/go.mod h1:sEXGtKMpRbfGhShfObhgMyxDpdu/5ABGrzSGYaigx5A=
github.com/sourcegraph/embedded-postgres v1.19.1-0.20230113234230-bb62ad58a1e1 h1:NbyS/m5kyBsaxynmY18st03pL9ZSOdEEC/B839vNNRA=
github.com/sourcegraph/embedded-postgres v1.19.1-0.20230113234230-bb62ad58a1e1/go.mod h1:0B+3bPsMvcNgR9nN+bdM2x9YaNYDnf3ksUqYp1OAub0=
github.com/sourcegraph/go-ctags v0.0.0-20230111110657-c27675da7f71 h1:tsWE3F3StWvnwLnC4JWb0zX0UHY9GULQtu/aoQvLJvI=
github.com/sourcegraph/go-ctags v0.0.0-20230111110657-c27675da7f71/go.mod h1:ZYjpRXoJrRlxjU9ZfpaUKJkk62AjhJPffN3rlw2aqxM=
github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
@ -2293,6 +2295,7 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xeonx/timeago v1.0.0-rc4 h1:9rRzv48GlJC0vm+iBpLcWAr8YbETyN9Vij+7h2ammz4=
github.com/xeonx/timeago v1.0.0-rc4/go.mod h1:qDLrYEFynLO7y5Ho7w3GwgtYgpy5UfhcXIIQvMKVDkA=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
@ -2428,6 +2431,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk=
go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=

4
internal/conf/computed.go
View File

@ -21,7 +21,7 @@ import (
func init() {
deployType := deploy.Type()
if !deploy.IsValidDeployType(deployType) {
log.Fatalf("The 'DEPLOY_TYPE' environment variable is invalid. Expected one of: %q, %q, %q, %q, %q, %q. Got: %q", deploy.Kubernetes, deploy.DockerCompose, deploy.PureDocker, deploy.SingleDocker, deploy.Dev, deploy.Helm, deployType)
log.Fatalf("The 'DEPLOY_TYPE' environment variable is invalid. Expected one of: %q, %q, %q, %q, %q, %q, %q. Got: %q", deploy.Kubernetes, deploy.DockerCompose, deploy.PureDocker, deploy.SingleDocker, deploy.Dev, deploy.Helm, deploy.SingleProgram, deployType)
}
confdefaults.Default = defaultConfigForDeployment()
@ -36,6 +36,8 @@ func defaultConfigForDeployment() conftypes.RawUnified {
return confdefaults.DockerContainer
case deploy.IsDeployTypeKubernetes(deployType), deploy.IsDeployTypeDockerCompose(deployType), deploy.IsDeployTypePureDocker(deployType):
return confdefaults.KubernetesOrDockerComposeOrPureDocker
case deploy.IsDeployTypeSingleProgram(deployType):
return confdefaults.SingleProgram
default:
panic("deploy type did not register default configuration")
}

11
internal/conf/conf.go
View File

@ -14,6 +14,7 @@ import (
sglog "github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/schema"
)
@ -66,6 +67,12 @@ func getMode() configurationMode {
}
func getModeUncached() configurationMode {
if deploy.IsDeployTypeSingleProgram(deploy.Type()) {
// Single-program always uses the server mode because everything is running in the same
// process.
return modeServer
}
mode := os.Getenv("CONFIGURATION_MODE")
switch mode {
@ -206,6 +213,10 @@ var siteConfigEscapeHatchPath = env.Get("SITE_CONFIG_ESCAPE_HATCH_PATH", "$HOME/
// cannot access the UI (for example by configuring auth in a way that locks them out)
// they can simply edit this file in any of the frontend containers to undo the change.
func startSiteConfigEscapeHatchWorker(c ConfigurationSource) {
if os.Getenv("NO_SITE_CONFIG_ESCAPE_HATCH") == "1" {
return
}
siteConfigEscapeHatchPath = os.ExpandEnv(siteConfigEscapeHatchPath)
var (

15
internal/conf/confdefaults/confdefaults.go
View File

@ -71,6 +71,21 @@ var KubernetesOrDockerComposeOrPureDocker = conftypes.RawUnified{
}`,
}
// SingleProgram is the default configuration for the single-program (Go static binary)
// distribution.
var SingleProgram = conftypes.RawUnified{
Site: `{
"auth.providers": [
{ "type": "builtin" }
],
"externalURL": "http://localhost:3080",
"codeIntelAutoIndexing.enabled": true,
"codeIntelAutoIndexing.allowGlobalPolicies": true,
"executors.frontendURL": "http://host.docker.internal:3080",
}`,
}
// Default is the default for *this* deployment type. It is populated by
// pkg/conf at init time.
var Default conftypes.RawUnified

14
internal/conf/deploy/deploytype.go
View File

@ -11,12 +11,18 @@ const (
PureDocker = "pure-docker"
Dev = "dev"
Helm = "helm"
SingleProgram = "single-program"
)
var mock string
var forceType string // force a deploy type (can be injected with `go build -ldflags "-X ..."`)
// Type tells the deployment type.
func Type() string {
if forceType != "" {
return forceType
}
if mock != "" {
return mock
}
@ -62,6 +68,11 @@ func IsDeployTypeSingleDockerContainer(deployType string) bool {
return deployType == SingleDocker
}
// IsDeployTypeSingleProgram tells if the given deployment type is a single Go program.
func IsDeployTypeSingleProgram(deployType string) bool {
return deployType == SingleProgram
}
// IsDev tells if the given deployment type is "dev".
func IsDev(deployType string) bool {
return deployType == Dev
@ -74,5 +85,6 @@ func IsValidDeployType(deployType string) bool {
IsDeployTypeDockerCompose(deployType) ||
IsDeployTypePureDocker(deployType) ||
IsDeployTypeSingleDockerContainer(deployType) ||
IsDev(deployType)
IsDev(deployType) ||
IsDeployTypeSingleProgram(deployType)
}

11
internal/env/env.go vendored
View File

@ -218,3 +218,14 @@ func HandleHelpFlag() {
}
}
}
// HackClearEnvironCache can be used to clear the environ cache if os.Setenv was called and you want
// subsequent env.Get calls to return the new value. It is a hack but useful because some env.Get
// calls are hard to remove from static init time, and the ones we've moved to post-init we want to
// be able to use the default values we set in package singleprogram.
//
// TODO(sqs): TODO(single-binary): this indicates our initialization order could be better, hence this
// is labeled as a hack.
func HackClearEnvironCache() {
environ = nil
}

11
internal/extsvc/github/common.go
View File

@ -22,6 +22,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/encryption"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
@ -1463,6 +1464,14 @@ func ExternalRepoSpec(repo *Repository, baseURL *url.URL) api.ExternalRepoSpec {
}
}
func githubBaseURLDefault() string {
isSingleProgram := deploy.IsDeployTypeSingleProgram(deploy.Type())
if isSingleProgram {
return ""
}
return "http://github-proxy"
}
var (
gitHubDisable, _ = strconv.ParseBool(env.Get("SRC_GITHUB_DISABLE", "false", "disables communication with GitHub instances. Used to test GitHub service degradation"))
@ -1470,7 +1479,7 @@ var (
requestCounter = metrics.NewRequestMeter("github", "Total number of requests sent to the GitHub API.")
// Get raw proxy URL at service startup, but only get parsed URL at runtime with getGithubProxyURL
githubProxyRawURL = env.Get("GITHUB_BASE_URL", "http://github-proxy", "base URL for GitHub.com API (used for github-proxy)")
githubProxyRawURL = env.Get("GITHUB_BASE_URL", githubBaseURLDefault(), "base URL for GitHub.com API (used for github-proxy)")
)
func getGithubProxyURL() (*url.URL, bool) {

10
internal/metrics/operation.go
View File

@ -111,7 +111,7 @@ func NewREDMetrics(r prometheus.Registerer, metricPrefix string, fns ...REDMetri
},
options.labels,
)
duration = mustRegisterIgnoreDuplicate(r, duration)
duration = MustRegisterIgnoreDuplicate(r, duration)
count := prometheus.NewCounterVec(
prometheus.CounterOpts{
@ -122,7 +122,7 @@ func NewREDMetrics(r prometheus.Registerer, metricPrefix string, fns ...REDMetri
},
options.labels,
)
count = mustRegisterIgnoreDuplicate(r, count)
count = MustRegisterIgnoreDuplicate(r, count)
errors := prometheus.NewCounterVec(
prometheus.CounterOpts{
@ -133,7 +133,7 @@ func NewREDMetrics(r prometheus.Registerer, metricPrefix string, fns ...REDMetri
},
options.labels,
)
errors = mustRegisterIgnoreDuplicate(r, errors)
errors = MustRegisterIgnoreDuplicate(r, errors)
return &REDMetrics{
Duration: duration,
@ -142,10 +142,10 @@ func NewREDMetrics(r prometheus.Registerer, metricPrefix string, fns ...REDMetri
}
}
// mustRegisterIgnoreDuplicate is like registerer.MustRegister(collector), except that it returns
// MustRegisterIgnoreDuplicate is like registerer.MustRegister(collector), except that it returns
// the already registered collector with the same ID if a duplicate collector is attempted to be
// registered.
func mustRegisterIgnoreDuplicate[T prometheus.Collector](registerer prometheus.Registerer, collector T) T {
func MustRegisterIgnoreDuplicate[T prometheus.Collector](registerer prometheus.Registerer, collector T) T {
if err := registerer.Register(collector); err != nil {
if e, ok := err.(prometheus.AlreadyRegisteredError); ok {
return e.ExistingCollector.(T)

34
internal/service/service.go Normal file
View File

@ -0,0 +1,34 @@
// Package service defines a service that runs as part of the Sourcegraph application. Examples
// include frontend, gitserver, and repo-updater.
package service
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
)
// A Service provides independent functionality in the Sourcegraph application. Examples include
// frontend, gitserver, and repo-updater. A service may run in the same process as any other
// service, in a separate process, in a separate container, or on a separate host.
type Service interface {
// Name is the name of the service.
Name() string
// Configure reads from env vars, runs very quickly, and has no side effects. All services'
// Configure methods are run before any service's Start method.
//
// The returned env.Config will be passed to the service's Start method.
//
// The returned debugserver endpoints will be added to the global debugserver.
Configure() (env.Config, []debugserver.Endpoint)
// Start starts the service.
// TODO(sqs): TODO(single-binary): make it monitorable with goroutine.Whatever interfaces.
Start(context.Context, *observation.Context, ReadyFunc, env.Config) error
}
// ReadyFunc is called in (Service).Start when the service is ready to start serving clients.
type ReadyFunc func()

155
internal/service/svcmain/svcmain.go Normal file
View File

@ -0,0 +1,155 @@
// Package svcmain runs one or more services.
package svcmain
import (
"context"
"sync"
"github.com/getsentry/sentry-go"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/debugserver"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/logging"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/profiler"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/singleprogram"
"github.com/sourcegraph/sourcegraph/internal/tracer"
"github.com/sourcegraph/sourcegraph/internal/version"
)
type Config struct {
AfterConfigure func() // run after all services' Configure hooks are called
}
// Main is called from the `main` function of the `sourcegraph-oss` and `sourcegraph` commands.
func Main(services []service.Service, config Config) {
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
},
// Experimental: DevX is observing how sampling affects the errors signal.
log.NewSentrySinkWith(
log.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
),
)
logger := log.Scoped("sourcegraph", "Sourcegraph")
singleprogram.Init(logger)
run(liblog, logger, services, config, true, true)
}
// DeprecatedSingleServiceMain is called from the `main` function of a command to start a single
// service (such as frontend or gitserver).
//
// DEPRECATED: Building per-service commands (i.e., a separate binary for frontend, gitserver, etc.)
// is deprecated.
func DeprecatedSingleServiceMain(svc service.Service, config Config, validateConfig, useConfPackage bool) {
liblog := log.Init(log.Resource{
Name: env.MyName,
Version: version.Version(),
InstanceID: hostname.Get(),
},
// Experimental: DevX is observing how sampling affects the errors signal.
log.NewSentrySinkWith(
log.SentrySink{
ClientOptions: sentry.ClientOptions{SampleRate: 0.2},
},
),
)
logger := log.Scoped("sourcegraph", "Sourcegraph")
run(liblog, logger, []service.Service{svc}, config, validateConfig, useConfPackage)
}
func run(
liblog *log.PostInitCallbacks,
logger log.Logger,
services []service.Service,
config Config,
validateConfig bool,
useConfPackage bool,
) {
defer liblog.Sync()
// Initialize log15. Even though it's deprecated, it's still fairly widely used.
logging.Init() //nolint:staticcheck // Deprecated, but logs unmigrated to sourcegraph/log look really bad without this.
if useConfPackage {
conf.Init()
go conf.Watch(liblog.Update(conf.GetLogSinks))
tracer.Init(log.Scoped("tracer", "internal tracer package"), conf.DefaultClient())
}
profiler.Init()
obctx := observation.NewContext(logger)
ctx := context.Background()
allReady := make(chan struct{})
// Run the services' Configure funcs before env vars are locked.
var (
serviceConfigs = make([]env.Config, len(services))
allDebugserverEndpoints []debugserver.Endpoint
)
for i, s := range services {
var debugserverEndpoints []debugserver.Endpoint
serviceConfigs[i], debugserverEndpoints = s.Configure()
allDebugserverEndpoints = append(allDebugserverEndpoints, debugserverEndpoints...)
}
// Validate each service's configuration.
//
// This cannot be done for executor, see the executorcmd package for details.
if validateConfig {
for i, c := range serviceConfigs {
if c == nil {
continue
}
if err := c.Validate(); err != nil {
logger.Fatal("invalid configuration", log.String("service", services[i].Name()), log.Error(err))
}
}
}
env.Lock()
env.HandleHelpFlag()
if config.AfterConfigure != nil {
config.AfterConfigure()
}
// Start the debug server. The ready boolean state it publishes will become true when *all*
// services report ready.
var allReadyWG sync.WaitGroup
go debugserver.NewServerRoutine(allReady, allDebugserverEndpoints...).Start()
// Start the services.
for i := range services {
service := services[i]
serviceConfig := serviceConfigs[i]
allReadyWG.Add(1)
go func() {
// TODO(sqs): TODO(single-binary): Consider using the goroutine package and/or the errgroup package to report
// errors and listen to signals to initiate cleanup in a consistent way across all
// services.
obctx := observation.ScopedContext("", service.Name(), "", obctx)
err := service.Start(ctx, obctx, allReadyWG.Done, serviceConfig)
if err != nil {
logger.Fatal("failed to start service", log.String("service", service.Name()), log.Error(err))
}
}()
}
// Pass along the signal to the debugserver that all started services are ready.
go func() {
allReadyWG.Wait()
close(allReady)
}()
select {}
}

141
internal/singleprogram/postgresql.go Normal file
View File

@ -0,0 +1,141 @@
package singleprogram
import (
"context"
"fmt"
golog "log"
"os"
"os/user"
"path/filepath"
"time"
embeddedpostgres "github.com/fergusstrange/embedded-postgres"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
var useEmbeddedPostgreSQL = env.MustGetBool("USE_EMBEDDED_POSTGRESQL", true, "use an embedded PostgreSQL server (to use an existing PostgreSQL server and database, set the PG* env vars)")
type postgresqlEnvVars struct {
PGPORT, PGHOST, PGUSER, PGPASSWORD, PGDATABASE, PGSSLMODE, PGDATASOURCE string
}
func initPostgreSQL(logger log.Logger, embeddedPostgreSQLRootDir string) error {
var vars *postgresqlEnvVars
if useEmbeddedPostgreSQL {
var err error
vars, err = startEmbeddedPostgreSQL(embeddedPostgreSQLRootDir)
if err != nil {
return err
}
os.Setenv("PGPORT", vars.PGPORT)
os.Setenv("PGHOST", vars.PGHOST)
os.Setenv("PGUSER", vars.PGUSER)
os.Setenv("PGPASSWORD", vars.PGPASSWORD)
os.Setenv("PGDATABASE", vars.PGDATABASE)
os.Setenv("PGSSLMODE", vars.PGSSLMODE)
os.Setenv("PGDATASOURCE", vars.PGDATASOURCE)
} else {
vars = &postgresqlEnvVars{
PGPORT: os.Getenv("PGPORT"),
PGHOST: os.Getenv("PGHOST"),
PGUSER: os.Getenv("PGUSER"),
PGPASSWORD: os.Getenv("PGPASSWORD"),
PGDATABASE: os.Getenv("PGDATABASE"),
PGSSLMODE: os.Getenv("PGSSLMODE"),
PGDATASOURCE: os.Getenv("PGDATASOURCE"),
}
}
useSinglePostgreSQLDatabase(logger, vars)
// Migration on startup is ideal for the single-program deployment because there are no other
// simultaneously running services at startup that might interfere with a migration.
//
// TODO(sqs): TODO(single-binary): make this behavior more official and not just for "dev"
setDefaultEnv(logger, "SG_DEV_MIGRATE_ON_APPLICATION_STARTUP", "1")
return nil
}
func startEmbeddedPostgreSQL(pgRootDir string) (*postgresqlEnvVars, error) {
// Note: on macOS unix socket paths must be <103 bytes, so we place them in the home directory.
current, err := user.Current()
if err != nil {
return nil, errors.Wrap(err, "user.Current")
}
unixSocketDir := filepath.Join(current.HomeDir, ".sourcegraph-psql")
err = os.MkdirAll(unixSocketDir, os.ModePerm)
if err != nil {
return nil, errors.Wrap(err, "creating unix socket dir")
}
vars := &postgresqlEnvVars{
PGPORT: "",
PGHOST: unixSocketDir,
PGUSER: "sourcegraph",
PGPASSWORD: "",
PGDATABASE: "sourcegraph",
PGSSLMODE: "disable",
PGDATASOURCE: "postgresql:///sourcegraph?host=" + unixSocketDir,
}
db := embeddedpostgres.NewDatabase(
embeddedpostgres.DefaultConfig().
Version(embeddedpostgres.V14).
BinariesPath(filepath.Join(pgRootDir, "bin")).
DataPath(filepath.Join(pgRootDir, "data")).
RuntimePath(filepath.Join(pgRootDir, "runtime")).
Username(vars.PGUSER).
Database(vars.PGDATABASE).
UseUnixSocket(unixSocketDir).
Logger(golog.Writer()),
)
if err := db.Start(); err != nil {
return nil, err
}
go goroutine.MonitorBackgroundRoutines(context.Background(), &embeddedPostgreSQLBackgroundJob{db})
return vars, nil
}
type embeddedPostgreSQLBackgroundJob struct {
db *embeddedpostgres.EmbeddedPostgres // must be already started
}
func (db *embeddedPostgreSQLBackgroundJob) Start() {
// Noop. We start it synchronously on purpose because everything else following it requires it.
}
func (db *embeddedPostgreSQLBackgroundJob) Stop() {
// Sleep a short amount of time to give other services time to write to the database during their cleanup.
time.Sleep(1000 * time.Millisecond)
if err := db.db.Stop(); err != nil {
fmt.Fprintln(os.Stderr, "error stopping embedded PostgreSQL:", err)
}
}
func useSinglePostgreSQLDatabase(logger log.Logger, vars *postgresqlEnvVars) {
// Use a single PostgreSQL DB.
//
// For code intel:
setDefaultEnv(logger, "CODEINTEL_PGPORT", vars.PGPORT)
setDefaultEnv(logger, "CODEINTEL_PGHOST", vars.PGHOST)
setDefaultEnv(logger, "CODEINTEL_PGUSER", vars.PGUSER)
setDefaultEnv(logger, "CODEINTEL_PGPASSWORD", vars.PGPASSWORD)
setDefaultEnv(logger, "CODEINTEL_PGDATABASE", vars.PGDATABASE)
setDefaultEnv(logger, "CODEINTEL_PGSSLMODE", vars.PGSSLMODE)
setDefaultEnv(logger, "CODEINTEL_PGDATASOURCE", vars.PGDATASOURCE)
setDefaultEnv(logger, "CODEINTEL_PG_ALLOW_SINGLE_DB", "true")
// And for code insights.
setDefaultEnv(logger, "CODEINSIGHTS_PGPORT", vars.PGPORT)
setDefaultEnv(logger, "CODEINSIGHTS_PGHOST", vars.PGHOST)
setDefaultEnv(logger, "CODEINSIGHTS_PGUSER", vars.PGUSER)
setDefaultEnv(logger, "CODEINSIGHTS_PGPASSWORD", vars.PGPASSWORD)
setDefaultEnv(logger, "CODEINSIGHTS_PGDATABASE", vars.PGDATABASE)
setDefaultEnv(logger, "CODEINSIGHTS_PGSSLMODE", vars.PGSSLMODE)
setDefaultEnv(logger, "CODEINSIGHTS_PGDATASOURCE", vars.PGDATASOURCE)
}

160
internal/singleprogram/singleprogram.go Normal file
View File

@ -0,0 +1,160 @@
// Package singleprogram contains runtime utilities for the single-program (Go static binary)
// distribution of Sourcegraph.
package singleprogram
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf/confdefaults"
"github.com/sourcegraph/sourcegraph/internal/env"
)
func Init(logger log.Logger) {
// TODO(sqs) TODO(single-binary): see the env.HackClearEnvironCache docstring, we should be able to remove this
// eventually.
env.HackClearEnvironCache()
// INDEXED_SEARCH_SERVERS is empty (but defined) so that indexed search is disabled.
setDefaultEnv(logger, "INDEXED_SEARCH_SERVERS", "")
// Need to set this to avoid trying to look up gitservers via k8s service discovery.
// TODO(sqs) TODO(single-binary): Make this not require the hostname.
hostname, err := os.Hostname()
if err != nil {
fmt.Fprintln(os.Stderr, "unable to determine hostname:", err)
os.Exit(1)
}
setDefaultEnv(logger, "SRC_GIT_SERVERS", hostname+":3178")
setDefaultEnv(logger, "SYMBOLS_URL", "http://127.0.0.1:3184")
setDefaultEnv(logger, "SEARCHER_URL", "http://127.0.0.1:3181")
setDefaultEnv(logger, "REPO_UPDATER_URL", "http://127.0.0.1:3182")
// The syntax-highlighter might not be running, but this is a better default than an internal
// hostname.
setDefaultEnv(logger, "SRC_SYNTECT_SERVER", "http://localhost:9238")
// Jaeger might not be running, but this is a better default than an internal hostname.
//
// TODO(sqs) TODO(single-binary): this isnt taking effect
//
// setDefaultEnv(logger, "JAEGER_SERVER_URL", "http://localhost:16686")
// The s3proxy blobstore does need to be running. TODO(sqs): TODO(single-binary): bundle this somehow?
setDefaultEnv(logger, "PRECISE_CODE_INTEL_UPLOAD_AWS_ENDPOINT", "http://localhost:9000")
setDefaultEnv(logger, "PRECISE_CODE_INTEL_UPLOAD_BACKEND", "blobstore")
// Need to override this because without a host (eg ":3080") it listens only on localhost, which
// is not accessible from the containers
setDefaultEnv(logger, "SRC_HTTP_ADDR", "0.0.0.0:3080")
// This defaults to an internal hostname.
setDefaultEnv(logger, "SRC_FRONTEND_INTERNAL", "localhost:3090")
cacheDir, err := os.UserCacheDir()
if err == nil {
cacheDir = filepath.Join(cacheDir, "sourcegraph-sp")
err = os.MkdirAll(cacheDir, 0700)
}
if err != nil {
fmt.Fprintln(os.Stderr, "unable to make user cache directory:", err)
os.Exit(1)
}
setDefaultEnv(logger, "SRC_REPOS_DIR", filepath.Join(cacheDir, "repos"))
setDefaultEnv(logger, "CACHE_DIR", filepath.Join(cacheDir, "cache"))
configDir, err := os.UserConfigDir()
if err == nil {
configDir = filepath.Join(configDir, "sourcegraph-sp")
err = os.MkdirAll(configDir, 0700)
}
if err != nil {
fmt.Fprintln(os.Stderr, "unable to make user config directory:", err)
os.Exit(1)
}
embeddedPostgreSQLRootDir := filepath.Join(configDir, "postgresql")
if err := initPostgreSQL(logger, embeddedPostgreSQLRootDir); err != nil {
fmt.Fprintln(os.Stderr, "unable to set up PostgreSQL:", err)
os.Exit(1)
}
writeFileIfNotExists := func(path string, data []byte) {
var err error
if _, err = os.Stat(path); os.IsNotExist(err) {
err = os.WriteFile(path, data, 0600)
}
if err != nil {
fmt.Fprintf(os.Stderr, "unable to write file %s: %s\n", path, err)
os.Exit(1)
}
}
siteConfigPath := filepath.Join(configDir, "site-config.json")
setDefaultEnv(logger, "SITE_CONFIG_FILE", siteConfigPath)
setDefaultEnv(logger, "SITE_CONFIG_ALLOW_EDITS", "true")
writeFileIfNotExists(siteConfigPath, []byte(confdefaults.SingleProgram.Site))
globalSettingsPath := filepath.Join(configDir, "global-settings.json")
setDefaultEnv(logger, "GLOBAL_SETTINGS_FILE", globalSettingsPath)
setDefaultEnv(logger, "GLOBAL_SETTINGS_ALLOW_EDITS", "true")
writeFileIfNotExists(globalSettingsPath, []byte("{}\n"))
// Escape hatch isn't needed in local dev since the site config can always just be a file on disk.
setDefaultEnv(logger, "NO_SITE_CONFIG_ESCAPE_HATCH", "1")
// We disable the use of executors passwords, because executors only listen on `localhost` this
// is safe to do.
setDefaultEnv(logger, "EXECUTOR_FRONTEND_URL", "http://localhost:3080")
setDefaultEnv(logger, "EXECUTOR_FRONTEND_PASSWORD", "none")
setDefaultEnv(logger, "EXECUTOR_QUEUE_DISABLE_ACCESS_TOKEN_INSECURE", "true")
setDefaultEnv(logger, "EXECUTOR_USE_FIRECRACKER", "false")
// TODO(sqs): TODO(single-binary): Make it so we can run multiple executors in single-program mode. Right now, you
// need to change this to "batches" to use batch changes executors.
setDefaultEnv(logger, "EXECUTOR_QUEUE_NAME", "codeintel")
writeFile := func(path string, data []byte, perm fs.FileMode) {
if err := os.WriteFile(path, data, perm); err != nil {
fmt.Fprintf(os.Stderr, "unable to write file %s: %s\n", path, err)
os.Exit(1)
}
}
setDefaultEnv(logger, "CTAGS_PROCESSES", "2")
// Write script that invokes universal-ctags via Docker.
// TODO(sqs): TODO(single-binary): stop relying on a ctags Docker image
ctagsPath := filepath.Join(cacheDir, "universal-ctags-dev")
writeFile(ctagsPath, []byte(universalCtagsDevScript), 0700)
setDefaultEnv(logger, "CTAGS_COMMAND", ctagsPath)
}
// universalCtagsDevScript is copied from cmd/symbols/universal-ctags-dev.
const universalCtagsDevScript = `#!/usr/bin/env bash
# This script is a wrapper around universal-ctags.
exec docker run --rm -i \
-a stdin -a stdout -a stderr \
--user guest \
--name=universal-ctags-$$ \
--entrypoint /usr/local/bin/universal-ctags \
slimsag/ctags:latest@sha256:dd21503a3ae51524ab96edd5c0d0b8326d4baaf99b4238dfe8ec0232050af3c7 "$@"
`
// setDefaultEnv will set the environment variable if it is not set.
func setDefaultEnv(logger log.Logger, k, v string) {
if _, ok := os.LookupEnv(k); ok {
return
}
err := os.Setenv(k, v)
if err != nil {
logger.Fatal("setting default env variable", log.Error(err))
}
}

23
internal/symbols/client.go
View File

@ -32,7 +32,19 @@ import (
"github.com/sourcegraph/sourcegraph/lib/errors"
)
var symbolsURL = env.Get("SYMBOLS_URL", "k8s+http://symbols:3184", "symbols service URL")
func LoadConfig() {
symbolsURL := env.Get("SYMBOLS_URL", "k8s+http://symbols:3184", "symbols service URL")
DefaultClient = &Client{
URL: symbolsURL,
HTTPClient: defaultDoer,
HTTPLimiter: parallel.NewRun(500),
SubRepoPermsChecker: func() authz.SubRepoPermissionChecker { return authz.DefaultSubRepoPermsChecker },
}
}
// DefaultClient is the default Client. Unless overwritten, it is connected to the server specified by the
// SYMBOLS_URL environment variable.
var DefaultClient *Client
var defaultDoer = func() httpcli.Doer {
d, err := httpcli.NewInternalClientFactory("symbols").Doer()
@ -42,15 +54,6 @@ var defaultDoer = func() httpcli.Doer {
return d
}()
// DefaultClient is the default Client. Unless overwritten, it is connected to the server specified by the
// SYMBOLS_URL environment variable.
var DefaultClient = &Client{
URL: symbolsURL,
HTTPClient: defaultDoer,
HTTPLimiter: parallel.NewRun(500),
SubRepoPermsChecker: func() authz.SubRepoPermissionChecker { return authz.DefaultSubRepoPermsChecker },
}
// Client is a symbols service client.
type Client struct {
// URL to symbols service.

4
internal/symbols/client_test.go
View File

@ -14,6 +14,10 @@ import (
"github.com/sourcegraph/sourcegraph/internal/types"
)
func init() {
LoadConfig()
}
func TestSearchWithFiltering(t *testing.T) {
ctx := context.Background()
fixture := search.SymbolsResponse{

5
internal/workerutil/observability.go
View File

@ -93,7 +93,10 @@ func NewMetrics(observationCtx *observation.Context, prefix string, opts ...Obse
Help: help,
}, keys)
observationCtx.Registerer.MustRegister(gaugeVec)
// TODO(sqs): TODO(single-binary): Ideally we would be using MustRegister here, not the
// IgnoreDuplicate variant. This is a bit of a hack to allow 2 executor instances to run in a
// single binary deployment.
gaugeVec = metrics.MustRegisterIgnoreDuplicate(observationCtx.Registerer, gaugeVec)
return gaugeVec.WithLabelValues(values...)
}

81
sg.config.yaml
View File

@ -99,8 +99,8 @@ env:
# Required for frontend and executor to communicate
EXECUTOR_FRONTEND_URL: http://localhost:3080
# Must match the secret defined in the site config.
EXECUTOR_FRONTEND_PASSWORD: hunter2hunter2hunter2
EXECUTOR_FRONTEND_PASSWORD: none
EXECUTOR_QUEUE_DISABLE_ACCESS_TOKEN_INSECURE: true
# Disable firecracker inside executor in dev
EXECUTOR_USE_FIRECRACKER: false
@ -817,6 +817,55 @@ commands:
cmd: pnpm --filter @sourcegraph/browser dev
install: pnpm install
sourcegraph:
description: Single program (Go static binary) distribution
cmd: |
unset SRC_GIT_SERVERS INDEXED_SEARCH_SERVERS
# TODO: This should be fixed
export SOURCEGRAPH_LICENSE_GENERATION_KEY=$(cat ../dev-private/enterprise/dev/test-license-generation-key.pem)
# If EXTSVC_CONFIG_FILE is *unset*, set a default.
export EXTSVC_CONFIG_FILE=${EXTSVC_CONFIG_FILE-'../dev-private/enterprise/dev/external-services-config.json'}
.bin/sourcegraph
install: |
if [ -n "$DELVE" ]; then
export GCFLAGS='all=-N -l'
fi
go build -gcflags="$GCFLAGS" -ldflags="-X github.com/sourcegraph/sourcegraph/internal/conf/deploy.forceType=single-program" -o .bin/sourcegraph github.com/sourcegraph/sourcegraph/enterprise/cmd/sourcegraph
checkBinary: .bin/sourcegraph
env:
USE_EMBEDDED_POSTGRESQL: false
ENTERPRISE: 1
SITE_CONFIG_FILE: '../dev-private/enterprise/dev/site-config.json'
SITE_CONFIG_ESCAPE_HATCH_PATH: '$HOME/.sourcegraph/site-config.json'
WEBPACK_DEV_SERVER: 1
watch:
- cmd
- enterprise
- internal
- lib
- schema
sourcegraph-oss:
description: Single program (Go static binary) distribution, OSS variant
cmd: |
unset SRC_GIT_SERVERS INDEXED_SEARCH_SERVERS
.bin/sourcegraph-oss
install: |
if [ -n "$DELVE" ]; then
export GCFLAGS='all=-N -l'
fi
go build -gcflags="$GCFLAGS" -ldflags="-X github.com/sourcegraph/sourcegraph/internal/conf/deploy.forceType=single-program" -o .bin/sourcegraph-oss github.com/sourcegraph/sourcegraph/cmd/sourcegraph-oss
checkBinary: .bin/sourcegraph-oss
env:
USE_EMBEDDED_POSTGRESQL: false
WEBPACK_DEV_SERVER: 1
watch:
- cmd
- internal
- schema
defaultCommandset: enterprise
commandsets:
oss:
@ -1071,6 +1120,34 @@ commandsets:
- otel-collector
- jaeger
enterprise-single-program:
requiresDevPrivate: true
checks:
- docker
- redis
- postgres
- git
commands:
- sourcegraph
- docsite
- web
- syntax-highlighter
- blobstore
- caddy
oss-single-program:
checks:
- docker
- redis
- postgres
- git
commands:
- sourcegraph-oss
- docsite
- oss-web
- syntax-highlighter
- caddy
tests:
# These can be run with `sg test [name]`
backend: