From e91e94e12ae147a3285a7883df3f591998b0974c Mon Sep 17 00:00:00 2001 From: Robert Lin Date: Fri, 29 Apr 2022 12:13:57 -0700 Subject: [PATCH] dev: listen on 127.0.0.1 in dev (INSECURE_DEV=true) (#34714) This PR sets INSECURE_DEV (maybe rename in a followup?) in sg.config.yaml by default, which has most services listening on 127.0.0.1 instead of all interfaces. Any services not yet configured on INSECURE_DEV has been updated to listen on 127.0.0.1 if the envvar is set. Also removes all the MacOS firewall stuff in sg See https://github.com/sourcegraph/sourcegraph/issues/29908 for more details --- cmd/frontend/envvar/envvar.go | 7 +- cmd/frontend/internal/cli/serve_cmd.go | 12 ++- dev/sg/internal/run/run.go | 85 +--------------------- dev/sg/macos.go | 17 ----- dev/sg/sg_run.go | 8 +- dev/sg/sg_start.go | 8 +- dev/sg/sg_start_test.go | 67 ++++++++--------- doc/dev/background-information/sg/index.md | 3 - go.mod | 4 - go.sum | 8 -- sg.config.yaml | 1 + 11 files changed, 56 insertions(+), 164 deletions(-) delete mode 100644 dev/sg/macos.go diff --git a/cmd/frontend/envvar/envvar.go b/cmd/frontend/envvar/envvar.go index 88c839dbd3b..20b6c504395 100644 --- a/cmd/frontend/envvar/envvar.go +++ b/cmd/frontend/envvar/envvar.go @@ -9,7 +9,12 @@ import ( var HTTPAddrInternal = env.Get( "SRC_HTTP_ADDR_INTERNAL", - "0.0.0.0:3090", + func() string { + if env.InsecureDev { + return "127.0.0.1:3090" + } + return "0.0.0.0:3090" + }(), "HTTP listen address for internal HTTP API. This should never be exposed externally, as it lacks certain authz checks.", ) diff --git a/cmd/frontend/internal/cli/serve_cmd.go b/cmd/frontend/internal/cli/serve_cmd.go index 094f270ddc0..df08759fa34 100644 --- a/cmd/frontend/internal/cli/serve_cmd.go +++ b/cmd/frontend/internal/cli/serve_cmd.go @@ -63,7 +63,12 @@ var ( printLogo, _ = strconv.ParseBool(env.Get("LOGO", "false", "print Sourcegraph logo upon startup")) - httpAddr = env.Get("SRC_HTTP_ADDR", ":3080", "HTTP listen address for app and HTTP API") + httpAddr = env.Get("SRC_HTTP_ADDR", func() string { + if env.InsecureDev { + return "127.0.0.1:3080" + } + return ":3080" + }(), "HTTP listen address for app and HTTP API") httpAddrInternal = envvar.HTTPAddrInternal nginxAddr = env.Get("SRC_NGINX_HTTP_ADDR", "", "HTTP listen address for nginx reverse proxy to SRC_HTTP_ADDR. Has preference over SRC_HTTP_ADDR for ExternalURL.") @@ -289,9 +294,10 @@ func Main(enterpriseSetupHook func(db database.DB, c conftypes.UnifiedWatchable) } if printLogo { - logger.Info(fmt.Sprintf("\n\n%s\n\n", logoColor)) + // This is not a log entry and is usually disabled + println(fmt.Sprintf("\n\n%s\n\n", logoColor)) } - logger.Info(fmt.Sprintf("✱ Sourcegraph is ready at: %s\n", globals.ExternalURL())) + logger.Info(fmt.Sprintf("✱ Sourcegraph is ready at: %s", globals.ExternalURL())) close(ready) goroutine.MonitorBackgroundRoutines(context.Background(), routines...) diff --git a/dev/sg/internal/run/run.go b/dev/sg/internal/run/run.go index bbbf6ec7af6..d974dc30340 100644 --- a/dev/sg/internal/run/run.go +++ b/dev/sg/internal/run/run.go @@ -12,8 +12,6 @@ import ( "sync" "time" - "bitbucket.org/creachadair/shell" - "github.com/bitfield/script" "github.com/grafana/regexp" "github.com/rjeczalik/notify" @@ -23,7 +21,7 @@ import ( "github.com/sourcegraph/sourcegraph/lib/output" ) -func Commands(ctx context.Context, globalEnv map[string]string, addToMacOSFirewall bool, verbose bool, cmds ...Command) error { +func Commands(ctx context.Context, globalEnv map[string]string, verbose bool, cmds ...Command) error { chs := make([]<-chan struct{}, 0, len(cmds)) monitor := &changeMonitor{} for _, cmd := range cmds { @@ -78,11 +76,7 @@ func Commands(ctx context.Context, globalEnv map[string]string, addToMacOSFirewa }(cmd, chs[i]) } - postInstall := func() error { return nil } - if addToMacOSFirewall { - postInstall = addToMacosFirewall(cmds) - } - err = waitForInstallation(cmdNames, installed, failures, okayToStart, postInstall) + err = waitForInstallation(cmdNames, installed, failures, okayToStart) if err != nil { return err } @@ -98,76 +92,7 @@ func Commands(ctx context.Context, globalEnv map[string]string, addToMacOSFirewa } } -// addToMacosFirewall returns a callback that is used to add binaries used by the given -// commands to the MacOS firewall. -func addToMacosFirewall(cmds []Command) func() error { - return func() error { - root, err := root.RepositoryRoot() - if err != nil { - return err - } - - stdout.Out.WriteLine(output.Linef(output.EmojiWarningSign, output.StyleWarning, "You may be prompted to enter your password to add exceptions to the firewall.")) - - // http://www.manpagez.com/man/8/socketfilterfw/ - firewallCmdPath := "/usr/libexec/ApplicationFirewall/socketfilterfw" - var needsFirewallRestart bool - - // Add binaries in '.bin' to firewall - for _, cmd := range cmds { - // Some commands use env variables that may be from command env or global env, - // so do substitutions and get the binary we want to work with. - args, ok := shell.Split(os.Expand(cmd.Cmd, func(key string) string { - if v, exists := cmd.Env[key]; exists { - return v - } - return os.Getenv(key) - })) - if !ok || len(args) == 0 { - stdout.Out.WriteLine(output.Linef(output.EmojiFailure, output.StyleSuggestion, "%s: invalid command", cmd.Cmd)) - continue - } - - for _, binary := range args { - if strings.HasPrefix(binary, ".bin/") || strings.HasPrefix(binary, "./.bin/") { - addException := script.Exec(shell.Join([]string{"sudo", firewallCmdPath, "--add", filepath.Join(root, binary)})) - msg, err := addException.String() - if err != nil { - stdout.Out.WriteLine(output.Linef(output.EmojiFailure, output.StyleBold, "%s: %s", binary, err.Error())) - continue - } - - // socketfilterfw helpfully always returns status 0, so we need to check - // the output to determine whether things worked or not. In all cases we - // don't error out becasue we want other commands to go through the firewall - // updates regardless. - switch { - case strings.Contains(msg, "does not exist"): - stdout.Out.WriteLine(output.Linef(output.EmojiFailure, output.StyleWarning, "%s: %s", binary, strings.TrimSpace(msg))) - - case strings.Contains(msg, "added to firewall"): - stdout.Out.WriteLine(output.Linef(output.EmojiSuccess, output.StyleSuccess, "%s: added to firewall", binary)) - needsFirewallRestart = true - - default: - stdout.Out.WriteLine(output.Linef("", output.StyleSuggestion, "%s: %s", binary, strings.TrimSpace(msg))) - } - } - } - } - - if needsFirewallRestart { - restartFirewall := script. - Exec(shell.Join([]string{"sudo", firewallCmdPath, "--setglobalstate", "off"})). - Exec(shell.Join([]string{"sudo", firewallCmdPath, "--setglobalstate", "on"})) - return restartFirewall.Error() - } - - return nil - } -} - -func waitForInstallation(cmdNames map[string]struct{}, installed chan string, failures chan failedRun, okayToStart chan struct{}, postInstallCallback func() error) error { +func waitForInstallation(cmdNames map[string]struct{}, installed chan string, failures chan failedRun, okayToStart chan struct{}) error { stdout.Out.Write("") stdout.Out.WriteLine(output.Linef(output.EmojiLightbulb, output.StyleBold, "Installing %d commands...", len(cmdNames))) stdout.Out.Write("") @@ -209,10 +134,6 @@ func waitForInstallation(cmdNames map[string]struct{}, installed chan string, fa // Everything installed! if len(cmdNames) == 0 { progress.Complete() - err := postInstallCallback() - if err != nil { - return err - } stdout.Out.Write("") stdout.Out.WriteLine(output.Linef(output.EmojiSuccess, output.StyleSuccess, "Everything installed! Booting up the system!")) diff --git a/dev/sg/macos.go b/dev/sg/macos.go deleted file mode 100644 index ad6088954dd..00000000000 --- a/dev/sg/macos.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "runtime" - - "github.com/urfave/cli/v2" -) - -var ( - addToMacOSFirewall bool - addToMacOSFirewallFlag = &cli.BoolFlag{ - Name: "add-to-macos-firewall", - Usage: "OSX only; Add required exceptions to the firewall", - Value: runtime.GOOS == "darwin", - Destination: &addToMacOSFirewall, - } -) diff --git a/dev/sg/sg_run.go b/dev/sg/sg_run.go index 27926d04bb2..3bbf1215c74 100644 --- a/dev/sg/sg_run.go +++ b/dev/sg/sg_run.go @@ -28,10 +28,8 @@ var runCommand = &cli.Command{ Usage: "Run the given commands", ArgsUsage: "[command]", Category: CategoryDev, - Flags: []cli.Flag{ - addToMacOSFirewallFlag, - }, - Action: execAdapter(runExec), + Flags: []cli.Flag{}, + Action: execAdapter(runExec), BashComplete: completeOptions(func() (options []string) { config, _ := sgconf.Get(configFile, configOverwriteFile) if config == nil { @@ -66,7 +64,7 @@ func runExec(ctx context.Context, args []string) error { cmds = append(cmds, cmd) } - return run.Commands(ctx, config.Env, addToMacOSFirewall, verbose, cmds...) + return run.Commands(ctx, config.Env, verbose, cmds...) } func constructRunCmdLongHelp() string { diff --git a/dev/sg/sg_start.go b/dev/sg/sg_start.go index aafd546c54b..f0949d8414f 100644 --- a/dev/sg/sg_start.go +++ b/dev/sg/sg_start.go @@ -69,8 +69,6 @@ var ( Usage: "Services to set at info crit level.", Destination: &critStartServices, }, - - addToMacOSFirewallFlag, }, BashComplete: completeOptions(func() (options []string) { config, _ := sgconf.Get(configFile, configOverwriteFile) @@ -185,10 +183,10 @@ func startExec(ctx context.Context, args []string) error { } } - return startCommandSet(ctx, set, config, addToMacOSFirewall) + return startCommandSet(ctx, set, config) } -func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.Config, addToMacOSFirewall bool) error { +func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.Config) error { if err := runChecksWithName(ctx, set.Checks); err != nil { return err } @@ -218,7 +216,7 @@ func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.C env[k] = v } - return run.Commands(ctx, env, addToMacOSFirewall, verbose, cmds...) + return run.Commands(ctx, env, verbose, cmds...) } // logLevelOverrides builds a map of commands -> log level that should be overridden in the environment. diff --git a/dev/sg/sg_start_test.go b/dev/sg/sg_start_test.go index 312f9c3f4f9..800d5a76c11 100644 --- a/dev/sg/sg_start_test.go +++ b/dev/sg/sg_start_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "strings" "testing" @@ -33,7 +32,7 @@ func TestStartCommandSet(t *testing.T) { Commandsets: map[string]*sgconf.Commandset{"test-set": commandSet}, } - if err := startCommandSet(ctx, commandSet, testConf, false); err != nil { + if err := startCommandSet(ctx, commandSet, testConf); err != nil { t.Errorf("failed to start: %s", err) } @@ -54,43 +53,39 @@ func TestStartCommandSet(t *testing.T) { } func TestStartCommandSet_InstallError(t *testing.T) { - for _, withPostInstallCallback := range []bool{false, true} { - t.Run(fmt.Sprintf("WithPostInstallCallback:%t", withPostInstallCallback), func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - buf := useOutputBuffer(t) + buf := useOutputBuffer(t) - commandSet := &sgconf.Commandset{Name: "test-set", Commands: []string{"test-cmd-1"}} - command := run.Command{ - Name: "test-cmd-1", - Install: "echo 'booting up horsegraph' && exit 1", - Cmd: "echo 'never appears'", - } - - testConf := &sgconf.Config{ - Commands: map[string]run.Command{"test-cmd-1": command}, - Commandsets: map[string]*sgconf.Commandset{"test-set": commandSet}, - } - - err := startCommandSet(ctx, commandSet, testConf, withPostInstallCallback) - if err == nil { - t.Fatalf("err is nil unexpectedly") - } - if !strings.Contains(err.Error(), "failed to run test-cmd-1") { - t.Errorf("err contains wrong message: %s", err.Error()) - } - - expectOutput(t, buf, []string{ - "", - "💡 Installing 1 commands...", - "--------------------------------------------------------------------------------", - "Failed to build test-cmd-1: 'bash -c echo 'booting up horsegraph' && exit 1' failed: booting up horsegraph: exit status 1:", - "booting up horsegraph", - "--------------------------------------------------------------------------------", - }) - }) + commandSet := &sgconf.Commandset{Name: "test-set", Commands: []string{"test-cmd-1"}} + command := run.Command{ + Name: "test-cmd-1", + Install: "echo 'booting up horsegraph' && exit 1", + Cmd: "echo 'never appears'", } + + testConf := &sgconf.Config{ + Commands: map[string]run.Command{"test-cmd-1": command}, + Commandsets: map[string]*sgconf.Commandset{"test-set": commandSet}, + } + + err := startCommandSet(ctx, commandSet, testConf) + if err == nil { + t.Fatalf("err is nil unexpectedly") + } + if !strings.Contains(err.Error(), "failed to run test-cmd-1") { + t.Errorf("err contains wrong message: %s", err.Error()) + } + + expectOutput(t, buf, []string{ + "", + "💡 Installing 1 commands...", + "--------------------------------------------------------------------------------", + "Failed to build test-cmd-1: 'bash -c echo 'booting up horsegraph' && exit 1' failed: booting up horsegraph: exit status 1:", + "booting up horsegraph", + "--------------------------------------------------------------------------------", + }) } func useOutputBuffer(t *testing.T) *outputtest.Buffer { diff --git a/doc/dev/background-information/sg/index.md b/doc/dev/background-information/sg/index.md index e7a5b0b5eee..3aec407274c 100644 --- a/doc/dev/background-information/sg/index.md +++ b/doc/dev/background-information/sg/index.md @@ -148,9 +148,6 @@ sg start web- # Run default environment, Sourcegraph enterprise: sg start -# (macOs only) Automatically add exceptions to the system firewall -sg start -add-to-macos-firewall - # List available environments (defined under `commandSets` in `sg.config.yaml`): sg start -help diff --git a/go.mod b/go.mod index 5a1013aa885..2efc279aab4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/sourcegraph/sourcegraph go 1.17 require ( - bitbucket.org/creachadair/shell v0.0.7 cloud.google.com/go/kms v1.1.0 cloud.google.com/go/monitoring v1.2.0 cloud.google.com/go/profiler v0.2.0 @@ -24,7 +23,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.24.1 github.com/aws/smithy-go v1.11.0 github.com/beevik/etree v1.1.0 - github.com/bitfield/script v0.20.0 github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff github.com/buildkite/go-buildkite/v3 v3.0.1 github.com/cespare/xxhash v1.1.0 @@ -206,8 +204,6 @@ require ( github.com/envoyproxy/protoc-gen-validate v0.6.3 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.3.2 // indirect - github.com/itchyny/gojq v0.12.7 // indirect - github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect diff --git a/go.sum b/go.sum index 0b2c3e238e1..a901fd54bcc 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk= -bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U= cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.25.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -380,8 +378,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitfield/script v0.20.0 h1:dqeNh8LKf3MfKN21fpuuXNNZPzLlrG6WnmScHqOt13I= -github.com/bitfield/script v0.20.0/go.mod h1:l3AZPVAtKQrL03bwh7nlNTUtgrgSWurpJSbtqspYrOA= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= @@ -1450,10 +1446,6 @@ github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0Gqw github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= -github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= -github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= -github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= -github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= diff --git a/sg.config.yaml b/sg.config.yaml index dd4ce3667b3..29cf704ffbe 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -6,6 +6,7 @@ env: PGDATABASE: sourcegraph PGSSLMODE: disable SG_DEV_MIGRATE_ON_APPLICATION_STARTUP: 'true' + INSECURE_DEV: true SRC_REPOS_DIR: $HOME/.sourcegraph/repos SRC_LOG_LEVEL: info