diff --git a/dev/sg/internal/run/BUILD.bazel b/dev/sg/internal/run/BUILD.bazel index b726eca1a97..377c76eb028 100644 --- a/dev/sg/internal/run/BUILD.bazel +++ b/dev/sg/internal/run/BUILD.bazel @@ -6,6 +6,7 @@ go_library( srcs = [ "bazel_command.go", "command.go", + "docker_commmand.go", "helpers.go", "ibazel.go", "installer.go", @@ -14,6 +15,7 @@ go_library( "prefix_suffix_saver.go", "run.go", "sgconfig_command.go", + "sgconfig_command_options.go", ], importpath = "github.com/sourcegraph/sourcegraph/dev/sg/internal/run", visibility = ["//dev/sg:__subpackages__"], @@ -37,7 +39,14 @@ go_library( go_test( name = "run_test", timeout = "short", - srcs = ["logger_test.go"], + srcs = [ + "docker_command_test.go", + "logger_test.go", + ], embed = [":run"], - deps = ["@com_github_stretchr_testify//assert"], + deps = [ + "@com_github_google_go_cmp//cmp", + "@com_github_stretchr_testify//assert", + "@in_gopkg_yaml_v2//:yaml_v2", + ], ) diff --git a/dev/sg/internal/run/bazel_command.go b/dev/sg/internal/run/bazel_command.go index def1737266a..c45fd3c421b 100644 --- a/dev/sg/internal/run/bazel_command.go +++ b/dev/sg/internal/run/bazel_command.go @@ -7,76 +7,39 @@ import ( "strings" "github.com/rjeczalik/notify" - - "github.com/sourcegraph/sourcegraph/dev/sg/internal/secrets" ) // A BazelCommand is a command definition for sg run/start that uses // bazel under the hood. It will handle restarting itself autonomously, // as long as iBazel is running and watch that specific target. type BazelCommand struct { - Name string - Description string `yaml:"description"` - Target string `yaml:"target"` - Args string `yaml:"args"` - PreCmd string `yaml:"precmd"` - Env map[string]string `yaml:"env"` - IgnoreStdout bool `yaml:"ignoreStdout"` - IgnoreStderr bool `yaml:"ignoreStderr"` - ContinueWatchOnExit bool `yaml:"continueWatchOnExit"` - // Preamble is a short and visible message, displayed when the command is launched. - Preamble string `yaml:"preamble"` - ExternalSecrets map[string]secrets.ExternalSecret `yaml:"external_secrets"` - + Config SGConfigCommandOptions + Target string `yaml:"target"` // RunTarget specifies a target that should be run via `bazel run $RunTarget` instead of directly executing the binary. RunTarget string `yaml:"runTarget"` } -func (bc BazelCommand) GetName() string { - return bc.Name -} +// UnmarshalYAML implements the Unmarshaler interface for BazelCommand. +// This allows us to parse the flat YAML configuration into nested struct. +func (bc *BazelCommand) UnmarshalYAML(unmarshal func(any) error) error { + // In order to not recurse infinitely (calling UnmarshalYAML over and over) we create a + // temporary type alias. + // First parse the BazelCommand specific options + type rawBazel BazelCommand + if err := unmarshal((*rawBazel)(bc)); err != nil { + return err + } -func (bc BazelCommand) GetContinueWatchOnExit() bool { - return bc.ContinueWatchOnExit -} - -func (bc BazelCommand) GetEnv() map[string]string { - return bc.Env -} - -func (bc BazelCommand) GetIgnoreStdout() bool { - return bc.IgnoreStdout -} - -func (bc BazelCommand) GetIgnoreStderr() bool { - return bc.IgnoreStderr -} - -func (bc BazelCommand) GetPreamble() string { - return bc.Preamble + // Then parse the common options from the same list into a nested struct + return unmarshal(&bc.Config) } func (bc BazelCommand) GetBinaryLocation() (string, error) { - baseOutput, err := outputPath() - if err != nil { - return "", err - } - // Trim "bazel-out" because the next bazel query will include it. - outputPath := strings.TrimSuffix(strings.TrimSpace(string(baseOutput)), "bazel-out") - - // Get the binary from the specific target. - cmd := exec.Command("bazel", "cquery", bc.Target, "--output=files") - baseOutput, err = cmd.Output() - if err != nil { - return "", err - } - binPath := strings.TrimSpace(string(baseOutput)) - - return fmt.Sprintf("%s%s", outputPath, binPath), nil + return binaryLocation(bc.Target) } -func (bc BazelCommand) GetExternalSecrets() map[string]secrets.ExternalSecret { - return bc.ExternalSecrets +func (bc BazelCommand) GetConfig() SGConfigCommandOptions { + return bc.Config } func (bc BazelCommand) watchPaths() ([]string, error) { @@ -114,12 +77,25 @@ func (bc BazelCommand) GetExecCmd(ctx context.Context) (*exec.Cmd, error) { } } - return exec.CommandContext(ctx, "bash", "-c", fmt.Sprintf("%s\n%s", bc.PreCmd, cmd)), nil + return exec.CommandContext(ctx, "bash", "-c", fmt.Sprintf("%s\n%s", bc.Config.PreCmd, cmd)), nil } -func outputPath() ([]byte, error) { +func binaryLocation(target string) (string, error) { // Get the output directory from Bazel, which varies depending on which OS // we're running against. - cmd := exec.Command("bazel", "info", "output_path") - return cmd.Output() + baseOutput, err := exec.Command("bazel", "info", "output_path").Output() + if err != nil { + return "", err + } + // Trim "bazel-out" because the next bazel query will include it. + outputPath := strings.TrimSuffix(strings.TrimSpace(string(baseOutput)), "bazel-out") + + // Get the binary from the specific target. + bin, err := exec.Command("bazel", "cquery", target, "--output=files").Output() + if err != nil { + return "", err + } + binPath := strings.TrimSpace(string(bin)) + + return fmt.Sprintf("%s%s", outputPath, binPath), nil } diff --git a/dev/sg/internal/run/command.go b/dev/sg/internal/run/command.go index 0b778ee31ae..77e0bf9d1b0 100644 --- a/dev/sg/internal/run/command.go +++ b/dev/sg/internal/run/command.go @@ -6,8 +6,10 @@ import ( "fmt" "io" "net" + "os" "os/exec" "path/filepath" + "strings" "syscall" "github.com/grafana/regexp" @@ -16,71 +18,52 @@ import ( "github.com/sourcegraph/sourcegraph/dev/sg/internal/secrets" "github.com/sourcegraph/sourcegraph/dev/sg/internal/std" "github.com/sourcegraph/sourcegraph/dev/sg/interrupt" - "github.com/sourcegraph/sourcegraph/dev/sg/root" "github.com/sourcegraph/sourcegraph/lib/errors" "github.com/sourcegraph/sourcegraph/lib/output" "github.com/sourcegraph/sourcegraph/lib/process" ) type Command struct { - Name string - Cmd string `yaml:"cmd"` - Install string `yaml:"install"` - InstallFunc string `yaml:"install_func"` - CheckBinary string `yaml:"checkBinary"` - Env map[string]string `yaml:"env"` - Watch []string `yaml:"watch"` - IgnoreStdout bool `yaml:"ignoreStdout"` - IgnoreStderr bool `yaml:"ignoreStderr"` - DefaultArgs string `yaml:"defaultArgs"` - ContinueWatchOnExit bool `yaml:"continueWatchOnExit"` - // Preamble is a short and visible message, displayed when the command is launched. - Preamble string `yaml:"preamble"` - - ExternalSecrets map[string]secrets.ExternalSecret `yaml:"external_secrets"` - Description string `yaml:"description"` + Config SGConfigCommandOptions + Cmd string `yaml:"cmd"` + DefaultArgs string `yaml:"defaultArgs"` + Install string `yaml:"install"` + InstallFunc string `yaml:"install_func"` + CheckBinary string `yaml:"checkBinary"` + Watch []string `yaml:"watch"` // ATTENTION: If you add a new field here, be sure to also handle that // field in `Merge` (below). } -func (cmd Command) GetName() string { - return cmd.Name +// UnmarshalYAML implements the Unmarshaler interface for Command. +// This allows us to parse the flat YAML configuration into nested struct. +func (cmd *Command) UnmarshalYAML(unmarshal func(any) error) error { + // In order to not recurse infinitely (calling UnmarshalYAML over and over) we create a + // temporary type alias. + // First parse the Command specific options + type rawCommand Command + if err := unmarshal((*rawCommand)(cmd)); err != nil { + return err + } + + // Then parse the common options from the same list into a nested struct + return unmarshal(&cmd.Config) } -func (cmd Command) GetContinueWatchOnExit() bool { - return cmd.ContinueWatchOnExit +func (cmd Command) GetConfig() SGConfigCommandOptions { + return cmd.Config +} + +func (cmd Command) GetName() string { + return cmd.Config.Name } func (cmd Command) GetBinaryLocation() (string, error) { if cmd.CheckBinary != "" { - repoRoot, err := root.RepositoryRoot() - if err != nil { - return "", err - } - return filepath.Join(repoRoot, cmd.CheckBinary), nil + return filepath.Join(cmd.Config.RepositoryRoot, cmd.CheckBinary), nil } - return "", noBinaryError{name: cmd.Name} -} - -func (cmd Command) GetExternalSecrets() map[string]secrets.ExternalSecret { - return cmd.ExternalSecrets -} - -func (cmd Command) GetIgnoreStdout() bool { - return cmd.IgnoreStdout -} - -func (cmd Command) GetIgnoreStderr() bool { - return cmd.IgnoreStderr -} - -func (cmd Command) GetPreamble() string { - return cmd.Preamble -} - -func (cmd Command) GetEnv() map[string]string { - return cmd.Env + return "", noBinaryError{name: cmd.Config.Name} } func (cmd Command) GetExecCmd(ctx context.Context) (*exec.Cmd, error) { @@ -115,9 +98,9 @@ func (cmd Command) hasBashInstaller() bool { } func (cmd Command) bashInstall(ctx context.Context, parentEnv map[string]string) error { - output, err := BashInRoot(ctx, cmd.Install, makeEnv(parentEnv, cmd.Env)) + output, err := BashInRoot(ctx, cmd.Install, makeEnv(parentEnv, cmd.Config.Env)) if err != nil { - return installErr{cmdName: cmd.Name, output: output, originalErr: err} + return installErr{cmdName: cmd.Config.Name, output: output, originalErr: err} } return nil } @@ -125,43 +108,33 @@ func (cmd Command) bashInstall(ctx context.Context, parentEnv map[string]string) func (cmd Command) functionInstall(ctx context.Context, parentEnv map[string]string) error { fn, ok := installFuncs[cmd.InstallFunc] if !ok { - return installErr{cmdName: cmd.Name, originalErr: errors.Newf("no install func with name %q found", cmd.InstallFunc)} + return installErr{cmdName: cmd.Config.Name, originalErr: errors.Newf("no install func with name %q found", cmd.InstallFunc)} } - if err := fn(ctx, makeEnvMap(parentEnv, cmd.Env)); err != nil { - return installErr{cmdName: cmd.Name, originalErr: err} + if err := fn(ctx, makeEnvMap(parentEnv, cmd.Config.Env)); err != nil { + return installErr{cmdName: cmd.Config.Name, originalErr: err} } return nil } -func (cmd Command) getWatchPaths() ([]string, error) { - root, err := root.RepositoryRoot() - if err != nil { - return nil, err - } - +func (cmd Command) getWatchPaths() []string { fullPaths := make([]string, len(cmd.Watch)) for i, path := range cmd.Watch { - fullPaths[i] = filepath.Join(root, path) + fullPaths[i] = filepath.Join(cmd.Config.RepositoryRoot, path) } - return fullPaths, nil + return fullPaths } func (cmd Command) StartWatch(ctx context.Context) (<-chan struct{}, error) { - if watchPaths, err := cmd.getWatchPaths(); err != nil { - return nil, err - } else { - return WatchPaths(ctx, watchPaths) - } + return WatchPaths(ctx, cmd.getWatchPaths()) } func (c Command) Merge(other Command) Command { merged := c - if other.Name != merged.Name && other.Name != "" { - merged.Name = other.Name - } + merged.Config = c.Config.Merge(other.Config) + if other.Cmd != merged.Cmd && other.Cmd != "" { merged.Cmd = other.Cmd } @@ -171,36 +144,6 @@ func (c Command) Merge(other Command) Command { if other.InstallFunc != merged.InstallFunc && other.InstallFunc != "" { merged.InstallFunc = other.InstallFunc } - if other.IgnoreStdout != merged.IgnoreStdout && !merged.IgnoreStdout { - merged.IgnoreStdout = other.IgnoreStdout - } - if other.IgnoreStderr != merged.IgnoreStderr && !merged.IgnoreStderr { - merged.IgnoreStderr = other.IgnoreStderr - } - if other.DefaultArgs != merged.DefaultArgs && other.DefaultArgs != "" { - merged.DefaultArgs = other.DefaultArgs - } - if other.Preamble != merged.Preamble && other.Preamble != "" { - merged.Preamble = other.Preamble - } - if other.Description != merged.Description && other.Description != "" { - merged.Description = other.Description - } - merged.ContinueWatchOnExit = other.ContinueWatchOnExit || merged.ContinueWatchOnExit - - for k, v := range other.Env { - if merged.Env == nil { - merged.Env = make(map[string]string) - } - merged.Env[k] = v - } - - for k, v := range other.ExternalSecrets { - if merged.ExternalSecrets == nil { - merged.ExternalSecrets = make(map[string]secrets.ExternalSecret) - } - merged.ExternalSecrets[k] = v - } if !equal(merged.Watch, other.Watch) && len(other.Watch) != 0 { merged.Watch = other.Watch @@ -276,6 +219,9 @@ type outputOptions struct { // When true, output will be ignored and not written to any writers ignore bool + // When non-nil, all output will be flushed to this file and not to the terminal + logfile io.Writer + // when enabled, output will not be streamed to the writers until // after the process is begun, only captured for later retrieval buffer bool @@ -291,34 +237,60 @@ type outputOptions struct { start chan struct{} } -func startSgCmd(ctx context.Context, cmd SGConfigCommand, dir string, parentEnv map[string]string) (*startedCmd, error) { +func startSgCmd(ctx context.Context, cmd SGConfigCommand, parentEnv map[string]string) (*startedCmd, error) { exec, err := cmd.GetExecCmd(ctx) if err != nil { return nil, err } - secretsEnv, err := getSecrets(ctx, cmd.GetName(), cmd.GetExternalSecrets()) + conf := cmd.GetConfig() + + secretsEnv, err := getSecrets(ctx, conf.Name, conf.ExternalSecrets) if err != nil { std.Out.WriteLine(output.Styledf(output.StyleWarning, "[%s] %s %s", - cmd.GetName(), output.EmojiFailure, err.Error())) + conf.Name, output.EmojiFailure, err.Error())) } opts := commandOptions{ - name: cmd.GetName(), + name: conf.Name, exec: exec, - env: makeEnv(parentEnv, secretsEnv, cmd.GetEnv()), - dir: dir, - stdout: outputOptions{ignore: cmd.GetIgnoreStdout()}, - stderr: outputOptions{ignore: cmd.GetIgnoreStderr()}, + env: makeEnv(parentEnv, secretsEnv, conf.Env), + dir: conf.RepositoryRoot, + stdout: outputOptions{ignore: conf.IgnoreStdout}, + stderr: outputOptions{ignore: conf.IgnoreStderr}, + } + if conf.Logfile != "" { + if logfile, err := initLogFile(conf.Logfile); err != nil { + return nil, err + } else { + opts.stdout.logfile = logfile + opts.stderr.logfile = logfile + } } - if cmd.GetPreamble() != "" { - std.Out.WriteLine(output.Styledf(output.StyleOrange, "[%s] %s %s", cmd.GetName(), output.EmojiInfo, cmd.GetPreamble())) + if conf.Preamble != "" { + std.Out.WriteLine(output.Styledf(output.StyleOrange, "[%s] %s %s", conf.Name, output.EmojiInfo, conf.Preamble)) } return startCmd(ctx, opts) } +func initLogFile(logfile string) (io.Writer, error) { + if strings.HasPrefix(logfile, "~/") || strings.HasPrefix(logfile, "$HOME") { + home, err := os.UserHomeDir() + if err != nil { + return nil, errors.Wrap(err, "failed to get user home directory") + } + logfile = filepath.Join(home, strings.Replace(strings.Replace(logfile, "~/", "", 1), "$HOME", "", 1)) + } + parent := filepath.Dir(logfile) + if err := os.MkdirAll(parent, os.ModePerm); err != nil { + return nil, err + } + // we don't have to worry about the file existing already and growing large, since this will truncate the file if it exists + return os.Create(logfile) +} + func startCmd(ctx context.Context, opts commandOptions) (*startedCmd, error) { sc := &startedCmd{ opts: opts, @@ -395,6 +367,8 @@ func (sc *startedCmd) getOutputWriter(ctx context.Context, opts *outputOptions, if opts.ignore { std.Out.WriteLine(output.Styledf(output.StyleSuggestion, "Ignoring %s of %s", outputName, sc.opts.name)) + } else if opts.logfile != nil { + return opts.logfile } else { // Create a channel to signal when output should start. If buffering is disabled, close // the channel so output starts immediately. diff --git a/dev/sg/internal/run/docker_command_test.go b/dev/sg/internal/run/docker_command_test.go new file mode 100644 index 00000000000..546f90b0658 --- /dev/null +++ b/dev/sg/internal/run/docker_command_test.go @@ -0,0 +1,144 @@ +package run_test + +import ( + "testing" + + "gopkg.in/yaml.v2" + + "github.com/google/go-cmp/cmp" + + "github.com/sourcegraph/sourcegraph/dev/sg/internal/run" +) + +func parse(t *testing.T, input string) run.DockerCommand { + t.Helper() + + got := run.DockerCommand{} + if err := yaml.Unmarshal([]byte(input), &got); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + return got +} +func TestParseDockerCommand(t *testing.T) { + want := run.DockerCommand{ + Config: run.SGConfigCommandOptions{ + Name: "grafana", + Description: "Runs Grafana", + PreCmd: "echo hello", + Args: "--config /sg_config_grafana", + Env: map[string]string{"CACHE": "false", "GRAFANA_DISK": "$HOME/.sourcegraph-dev/data/grafana"}, + IgnoreStdout: true, + IgnoreStderr: false, + Logfile: "$HOME/.sourcegraph-dev/logs/grafana/grafana.log", + }, + Docker: run.DockerOptions{ + Image: "grafana:candidate", + Volumes: []run.DockerVolume{ + { + From: "$HOME/.sourcegraph-dev/data/grafana", + To: "/var/lib/grafana", + }, + { + From: "$(pwd)/dev/grafana/all", + To: "/sg_config_grafana/provisioning/datasources", + }, + }, + Flags: map[string]string{"cpus": "1", "memory": "1g"}, + Ports: []string{"3370", + "5168", + "9128:9128", + "5432:5678", + }, + Linux: run.DockerLinuxOptions{ + Flags: map[string]string{ + "add-host": "host.docker.internal:host-gateway", + "user": "$UID"}, + Env: map[string]string{"FOO": "bar"}}}, + Target: "//docker-images/grafana:image_tarball", + } + + got := parse(t, grafana) + + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("wrong cmd. (-want +got):\n%s", diff) + } +} + +func TestCompileGrafanaCommand(t *testing.T) { + want := `docker inspect grafana > /dev/null 2>&1 && docker rm -f grafana +docker load -i ./fake_img.tar + +mkdir -p $HOME/.sourcegraph-dev/data/grafana +mkdir -p $(pwd)/dev/grafana/all + +echo hello +docker run --rm --name grafana ` + + `-v $HOME/.sourcegraph-dev/data/grafana:/var/lib/grafana ` + + `-v $(pwd)/dev/grafana/all:/sg_config_grafana/provisioning/datasources ` + + `-p 3370:3370 -p 5168:5168 -p 9128:9128 -p 5432:5678 ` + + `--cpus=1 --memory=1g ` + + `-e CACHE="false" -e GRAFANA_DISK="$HOME/.sourcegraph-dev/data/grafana" ` + + `grafana:candidate --config /sg_config_grafana` + got := parse(t, grafana).GetCmd("./fake_img.tar", false) + + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("wrong cmd. (-want +got):\n%s", diff) + } +} + +func TestCompileGrafanaCommand_Linux(t *testing.T) { + want := `docker inspect grafana > /dev/null 2>&1 && docker rm -f grafana +docker load -i ./fake_img.tar + +mkdir -p $HOME/.sourcegraph-dev/data/grafana +mkdir -p $(pwd)/dev/grafana/all + +echo hello +docker run --rm --name grafana ` + + `-v $HOME/.sourcegraph-dev/data/grafana:/var/lib/grafana ` + + `-v $(pwd)/dev/grafana/all:/sg_config_grafana/provisioning/datasources ` + + `-p 3370:3370 -p 5168:5168 -p 9128:9128 -p 5432:5678 ` + + `--add-host=host.docker.internal:host-gateway --cpus=1 --memory=1g --user=$UID ` + + `-e CACHE="false" -e FOO="bar" -e GRAFANA_DISK="$HOME/.sourcegraph-dev/data/grafana" ` + + `grafana:candidate --config /sg_config_grafana` + + got := parse(t, grafana).GetCmd("./fake_img.tar", true) + + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("wrong cmd. (-want +got):\n%s", diff) + } +} + +var grafana = ` +name: grafana +target: //docker-images/grafana:image_tarball +description: Runs Grafana +precmd: echo hello +ignoreStdout: true +args: "--config /sg_config_grafana" +logfile: "$HOME/.sourcegraph-dev/logs/grafana/grafana.log" +env: + GRAFANA_DISK: "$HOME/.sourcegraph-dev/data/grafana" + CACHE: false +docker: + image: grafana:candidate + ports: + - 3370 + - 5168 + - 9128:9128 + - 5432:5678 + flags: + cpus: 1 + memory: 1g + volumes: + - from: $HOME/.sourcegraph-dev/data/grafana + to: /var/lib/grafana + - from: $(pwd)/dev/grafana/all + to: /sg_config_grafana/provisioning/datasources + linux: + flags: + add-host: host.docker.internal:host-gateway + user: $UID + env: + FOO: bar` diff --git a/dev/sg/internal/run/docker_commmand.go b/dev/sg/internal/run/docker_commmand.go new file mode 100644 index 00000000000..02aca99e8b7 --- /dev/null +++ b/dev/sg/internal/run/docker_commmand.go @@ -0,0 +1,230 @@ +package run + +import ( + "cmp" + "context" + "fmt" + "os/exec" + "runtime" + "sort" + "strings" + + "github.com/rjeczalik/notify" +) + +// A DockerCommand is a command definition for sg run/start that uses +// bazel under the hood. It will handle restarting itself autonomously, +// as long as iBazel is running and watch that specific target. +type DockerCommand struct { + Config SGConfigCommandOptions + Docker DockerOptions `yaml:"docker"` + // Optional bazel target to build and watch which provides a docker image tarball + // if not provided, the DockerOptions::Image will simply be run directly + // if Pull=true, it will be pulled first + Target string `yaml:"target"` +} + +type DockerOptions struct { + Image string `yaml:"image"` + // If true, the image will be pulled before running the container + Pull bool `yaml:"pull"` + Volumes []DockerVolume `yaml:"volumes"` + // Additional flags to pass to the docker run command + // e.g. cpus: 1 would be converted to --cpus=1 + Flags map[string]string `yaml:"flags"` + // Ports is a list of ports to expose from the container to the host. + // If only a single value is given it will be assumed to map that port from + // the container to the same port on the host + Ports []string `yaml:"ports"` + Linux DockerLinuxOptions `yaml:"linux"` +} + +// DockerLinuxOptions is a struct that holds linux specific modifications to +// DockerEngine parameters for the DockerCommand +type DockerLinuxOptions struct { + Flags map[string]string `yaml:"flags"` + Env map[string]string `yaml:"env"` +} + +// Details for a docker volume to mount into the container +type DockerVolume struct { + From string `yaml:"from"` + To string `yaml:"to"` +} + +// UnmarshalYAML implements the Unmarshaler interface for DockerCommand. +// This allows us to parse the flat YAML configuration into nested struct. +func (dc *DockerCommand) UnmarshalYAML(unmarshal func(any) error) error { + // In order to not recurse infinitely (calling UnmarshalYAML over and over) we create a + // temporary type alias. + // First parse the DockerCommand specific options + type rawDocker DockerCommand + if err := unmarshal((*rawDocker)(dc)); err != nil { + return err + } + + // Then parse the common options from the same list into a nested struct + return unmarshal(&dc.Config) +} + +func (dc DockerCommand) GetConfig() SGConfigCommandOptions { + config := dc.Config + // Add a custom preamble for docker listing ports + config.Preamble += dc.GetDockerPreamble() + // Add any platform specific environment overrides + config.Env = makeEnvMap(config.Env, dc.GetDockerEnv(runtime.GOOS == "linux")) + return config +} + +func (dc DockerCommand) GetBinaryLocation() (string, error) { + if dc.Target == "" { + return "", nil + } + return binaryLocation(dc.Target) +} + +func (dc DockerCommand) StartWatch(ctx context.Context) (<-chan struct{}, error) { + if watchPaths, err := dc.watchPaths(); err != nil { + return nil, err + } else { + // skip remove events as we don't care about files being removed, we only + // want to know when the binary has been rebuilt + return WatchPaths(ctx, watchPaths, notify.Remove) + } +} + +func (dc DockerCommand) watchPaths() ([]string, error) { + // If no target is defined, there is nothing to be built and watched + if dc.Target == "" { + return nil, nil + } + // Grab the location of the binary in bazel-out. + binLocation, err := dc.GetBinaryLocation() + if err != nil { + return nil, err + } + return []string{binLocation}, nil +} + +// GetDockerEnv returns the environment variables to be passed to the docker run command +func (dc DockerCommand) GetDockerEnv(isLinux bool) map[string]string { + env := dc.Config.Env + if isLinux { + merge(env, dc.Docker.Linux.Env) + } + return env +} + +// GetFlags returns the flags (i.e. --something) to be passed to the docker run command +func (opts DockerOptions) GetFlags(isLinux bool) map[string]string { + if isLinux { + merge(opts.Flags, opts.Linux.Flags) + } + return opts.Flags +} + +// CreateDockerVolumes returns bash commands that will ensure that all of the local volumes +// exist before the docker run command is executed +func (dc DockerCommand) CreateDockerVolumes() string { + var cmd strings.Builder + for _, volume := range dc.Docker.Volumes { + fmt.Fprintf(&cmd, "mkdir -p %s\n", volume.From) + } + return cmd.String() +} + +func (dc DockerCommand) GetDockerImage(bin string) string { + if bin != "" { + return fmt.Sprintf("docker load -i %s\n", bin) + } + if dc.Docker.Pull { + return fmt.Sprintf("docker pull %s\n", dc.Docker.Image) + } + + return "" +} + +func (dc DockerCommand) GetDockerPreamble() string { + var preamble strings.Builder + if dc.Config.Logfile != "" { + fmt.Fprintf(&preamble, "Writing log output to %s\n", dc.Config.Logfile) + } + + if len(dc.Docker.Ports) > 0 { + var localports []string + for _, port := range dc.Docker.Ports { + localports = append(localports, strings.Split(port, ":")[0]) + } + fmt.Fprintf(&preamble, "Listening on local ports: %s\n", strings.Join(localports, ", ")) + } + return preamble.String() +} + +// Constructs the actual docker run command to be executed +func (dc DockerCommand) GetDockerCommand(isLinux bool) string { + var cmd strings.Builder + fmt.Fprintf(&cmd, "docker run --rm --name %s", dc.Config.Name) + for _, volume := range dc.Docker.Volumes { + fmt.Fprintf(&cmd, " -v %s:%s", volume.From, volume.To) + } + for _, port := range dc.Docker.Ports { + if strings.Contains(port, ":") { + fmt.Fprintf(&cmd, " -p %s", port) + } else { + fmt.Fprintf(&cmd, " -p %s:%s", port, port) + } + } + for _, flag := range toSortedPairs(dc.Docker.GetFlags(isLinux)) { + fmt.Fprintf(&cmd, " --%s=%s", flag.Key, flag.Value) + } + for _, env := range toSortedPairs(dc.GetDockerEnv(isLinux)) { + fmt.Fprintf(&cmd, ` -e %s="%s"`, env.Key, env.Value) + } + fmt.Fprintf(&cmd, " %s %s", dc.Docker.Image, dc.Config.Args) + return cmd.String() + +} + +func (dc DockerCommand) GetCmd(bin string, isLinux bool) string { + cleanup := fmt.Sprintf("docker inspect %s > /dev/null 2>&1 && docker rm -f %s", dc.Config.Name, dc.Config.Name) + load := dc.GetDockerImage(bin) + docker := dc.GetDockerCommand(isLinux) + volumes := dc.CreateDockerVolumes() + + return strings.Join([]string{cleanup, load, volumes, dc.Config.PreCmd, docker}, "\n") +} + +func (dc DockerCommand) GetExecCmd(ctx context.Context) (*exec.Cmd, error) { + bin, err := dc.GetBinaryLocation() + if err != nil { + return nil, err + } + cmd := dc.GetCmd(bin, runtime.GOOS == "linux") + return exec.CommandContext(ctx, "bash", "-c", cmd), nil +} + +type Entry[K, V any] struct { + Key K + Value V +} + +func toSortedPairs[K cmp.Ordered, V any](m map[K]V) []Entry[K, V] { + keys := make([]K, len(m)) + pairs := make([]Entry[K, V], len(m)) + i := 0 + for k := range m { + keys[i] = k + i++ + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + for i, k := range keys { + pairs[i] = Entry[K, V]{k, m[k]} + } + return pairs +} + +func merge(base, overrides map[string]string) { + for k, v := range overrides { + base[k] = v + } +} diff --git a/dev/sg/internal/run/ibazel.go b/dev/sg/internal/run/ibazel.go index 1250f41b54c..316141849b9 100644 --- a/dev/sg/internal/run/ibazel.go +++ b/dev/sg/internal/run/ibazel.go @@ -40,7 +40,7 @@ type IBazel struct { } // returns a runner to interact with ibazel. -func NewIBazel(cmds []BazelCommand, dir string) (*IBazel, error) { +func NewIBazel(targets []string, dir string) (*IBazel, error) { logsDir, err := initLogsDir() if err != nil { return nil, err @@ -51,15 +51,8 @@ func NewIBazel(cmds []BazelCommand, dir string) (*IBazel, error) { return nil, err } - targets := make([]string, 0, len(cmds)) - for _, cmd := range cmds { - if cmd.Target != "" && !slices.Contains(targets, cmd.Target) { - targets = append(targets, cmd.Target) - } - } - return &IBazel{ - targets: targets, + targets: cleanTargets(targets), events: newIBazelEventHandler(profileEventsPath(logsDir)), logsDir: logsDir, logFile: logFile, @@ -67,6 +60,17 @@ func NewIBazel(cmds []BazelCommand, dir string) (*IBazel, error) { }, nil } +func cleanTargets(targets []string) []string { + output := []string{} + + for _, target := range targets { + if target != "" && !slices.Contains(output, target) { + output = append(output, target) + } + } + return output +} + func initLogsDir() (string, error) { sghomedir, err := root.GetSGHomePath() if err != nil { diff --git a/dev/sg/internal/run/run.go b/dev/sg/internal/run/run.go index f99b17177f1..5eaede64397 100644 --- a/dev/sg/internal/run/run.go +++ b/dev/sg/internal/run/run.go @@ -13,17 +13,15 @@ import ( "github.com/sourcegraph/conc/pool" "github.com/sourcegraph/sourcegraph/dev/sg/internal/std" - "github.com/sourcegraph/sourcegraph/dev/sg/root" "github.com/sourcegraph/sourcegraph/lib/errors" "github.com/sourcegraph/sourcegraph/lib/output" ) type cmdRunner struct { *std.Output - cmds []SGConfigCommand - repositoryRoot string - parentEnv map[string]string - verbose bool + cmds []SGConfigCommand + parentEnv map[string]string + verbose bool } func Commands(ctx context.Context, parentEnv map[string]string, verbose bool, cmds ...SGConfigCommand) (err error) { @@ -33,11 +31,7 @@ func Commands(ctx context.Context, parentEnv map[string]string, verbose bool, cm } std.Out.WriteLine(output.Styled(output.StylePending, fmt.Sprintf("Starting %d cmds", len(cmds)))) - repoRoot, err := root.RepositoryRoot() - if err != nil { - return err - } - + repoRoot := cmds[0].GetConfig().RepositoryRoot // binaries get installed to /.bin. If the binary is installed with go build, then go // will create .bin directory. Some binaries (like docsite) get downloaded instead of built and therefore // need the directory to exist before hand. @@ -53,7 +47,6 @@ func Commands(ctx context.Context, parentEnv map[string]string, verbose bool, cm runner := cmdRunner{ std.Out, cmds, - repoRoot, parentEnv, verbose, } @@ -67,7 +60,8 @@ func (runner *cmdRunner) run(ctx context.Context) error { for _, cmd := range runner.cmds { cmd := cmd p.Go(func(ctx context.Context) error { - std.Out.WriteLine(output.Styledf(output.StylePending, "Running %s...", cmd.GetName())) + config := cmd.GetConfig() + std.Out.WriteLine(output.Styledf(output.StylePending, "Running %s...", config.Name)) // Start watching the commands dependencies wantRestart, err := cmd.StartWatch(ctx) @@ -80,7 +74,7 @@ func (runner *cmdRunner) run(ctx context.Context) error { proc, err := runner.start(ctx, cmd) if err != nil { runner.printError(cmd, err) - return errors.Wrapf(err, "failed to start command %q", cmd.GetName()) + return errors.Wrapf(err, "failed to start command %q", config.Name) } defer proc.cancel() @@ -98,17 +92,17 @@ func (runner *cmdRunner) run(ctx context.Context) error { return err } - runner.WriteLine(output.Styledf(output.StyleSuccess, "%s%s exited without error%s", output.StyleBold, cmd.GetName(), output.StyleReset)) + runner.WriteLine(output.Styledf(output.StyleSuccess, "%s%s exited without error%s", output.StyleBold, config.Name, output.StyleReset)) // If we shouldn't restart when the process exits, return - if !cmd.GetContinueWatchOnExit() { + if !config.ContinueWatchOnExit { return nil } // handle file watcher triggered case <-wantRestart: // If the command has an installer, re-run the install and determine if we should restart - runner.WriteLine(output.Styledf(output.StylePending, "Change detected. Reloading %s...", cmd.GetName())) + runner.WriteLine(output.Styledf(output.StylePending, "Change detected. Reloading %s...", config.Name)) shouldRestart, err := runner.reinstall(ctx, cmd) if err != nil { runner.printError(cmd, err) @@ -116,7 +110,7 @@ func (runner *cmdRunner) run(ctx context.Context) error { } if shouldRestart { - runner.WriteLine(output.Styledf(output.StylePending, "Restarting %s...", cmd.GetName())) + runner.WriteLine(output.Styledf(output.StylePending, "Restarting %s...", config.Name)) proc.cancel() proc, err = runner.start(ctx, cmd) if err != nil { @@ -124,7 +118,7 @@ func (runner *cmdRunner) run(ctx context.Context) error { } defer proc.cancel() } else { - runner.WriteLine(output.Styledf(output.StylePending, "Binary for %s did not change. Not restarting.", cmd.GetName())) + runner.WriteLine(output.Styledf(output.StylePending, "Binary for %s did not change. Not restarting.", config.Name)) } } } @@ -135,7 +129,7 @@ func (runner *cmdRunner) run(ctx context.Context) error { } func (runner *cmdRunner) printError(cmd SGConfigCommand, err error) { - printCmdError(runner.Output.Output, cmd.GetName(), err) + printCmdError(runner.Output.Output, cmd.GetConfig().Name, err) } func (runner *cmdRunner) debug(msg string, args ...any) { //nolint currently unused but a handy tool for debugginlg @@ -146,7 +140,7 @@ func (runner *cmdRunner) debug(msg string, args ...any) { //nolint currently unu } func (runner *cmdRunner) start(ctx context.Context, cmd SGConfigCommand) (*startedCmd, error) { - return startSgCmd(ctx, cmd, runner.repositoryRoot, runner.parentEnv) + return startSgCmd(ctx, cmd, runner.parentEnv) } func (runner *cmdRunner) reinstall(ctx context.Context, cmd SGConfigCommand) (bool, error) { @@ -349,15 +343,12 @@ func md5HashFile(filename string) (string, error) { } func Test(ctx context.Context, cmd SGConfigCommand, parentEnv map[string]string) error { - repoRoot, err := root.RepositoryRoot() - if err != nil { - return err - } + name := cmd.GetConfig().Name - std.Out.WriteLine(output.Styledf(output.StylePending, "Starting testsuite %q.", cmd.GetName())) - proc, err := startSgCmd(ctx, cmd, repoRoot, parentEnv) + std.Out.WriteLine(output.Styledf(output.StylePending, "Starting testsuite %q.", name)) + proc, err := startSgCmd(ctx, cmd, parentEnv) if err != nil { - printCmdError(std.Out.Output, cmd.GetName(), err) + printCmdError(std.Out.Output, name, err) } return proc.Wait() } diff --git a/dev/sg/internal/run/sgconfig_command.go b/dev/sg/internal/run/sgconfig_command.go index 26bdfa8e321..33ca9b86a1e 100644 --- a/dev/sg/internal/run/sgconfig_command.go +++ b/dev/sg/internal/run/sgconfig_command.go @@ -6,20 +6,12 @@ import ( "os/exec" "github.com/rjeczalik/notify" - - "github.com/sourcegraph/sourcegraph/dev/sg/internal/secrets" ) type SGConfigCommand interface { - // Getters for common fields - GetName() string - GetContinueWatchOnExit() bool - GetIgnoreStdout() bool - GetIgnoreStderr() bool - GetPreamble() string - GetEnv() map[string]string + // Extracts common config and options, allowing the implementation any final overrides + GetConfig() SGConfigCommandOptions GetBinaryLocation() (string, error) - GetExternalSecrets() map[string]secrets.ExternalSecret GetExecCmd(context.Context) (*exec.Cmd, error) // Start a file watcher on the relevant filesystem sub-tree for this command diff --git a/dev/sg/internal/run/sgconfig_command_options.go b/dev/sg/internal/run/sgconfig_command_options.go new file mode 100644 index 00000000000..91415ac85f4 --- /dev/null +++ b/dev/sg/internal/run/sgconfig_command_options.go @@ -0,0 +1,79 @@ +package run + +import "github.com/sourcegraph/sourcegraph/dev/sg/internal/secrets" + +// Common sg command parameters shared by all command types +type SGConfigCommandOptions struct { + Name string + Description string `yaml:"description"` + // A command to be run before the command is run but after installation + PreCmd string `yaml:"precmd"` + // A list of additional arguments to be passed to the command + Args string `yaml:"args"` + Env map[string]string `yaml:"env"` + IgnoreStdout bool `yaml:"ignoreStdout"` + IgnoreStderr bool `yaml:"ignoreStderr"` + // If true, the runner will continue watching this commands dependencies + // even if the command exits with a zero status code. + ContinueWatchOnExit bool `yaml:"continueWatchOnExit"` + // Preamble is a short and visible message, displayed when the command is launched. + Preamble string `yaml:"preamble"` + + // Output all logs to a file instead of to stdout/stderr + Logfile string `yaml:"logfile"` + ExternalSecrets map[string]secrets.ExternalSecret `yaml:"external_secrets"` + + RepositoryRoot string +} + +func (opts SGConfigCommandOptions) Merge(other SGConfigCommandOptions) SGConfigCommandOptions { + merged := opts + + if other.Name != merged.Name && other.Name != "" { + merged.Name = other.Name + } + if other.Description != merged.Description && other.Description != "" { + merged.Description = other.Description + } + if other.Description != merged.Description && other.Description != "" { + merged.Description = other.Description + } + if other.PreCmd != merged.PreCmd && other.PreCmd != "" { + merged.PreCmd = other.PreCmd + } + if other.Args != merged.Args && other.Args != "" { + merged.Args = other.Args + } + if other.IgnoreStdout != merged.IgnoreStdout && !merged.IgnoreStdout { + merged.IgnoreStdout = other.IgnoreStdout + } + if other.IgnoreStderr != merged.IgnoreStderr && !merged.IgnoreStderr { + merged.IgnoreStderr = other.IgnoreStderr + } + merged.ContinueWatchOnExit = other.ContinueWatchOnExit || merged.ContinueWatchOnExit + if other.Preamble != merged.Preamble && other.Preamble != "" { + merged.Preamble = other.Preamble + } + if other.Logfile != merged.Logfile && other.Logfile != "" { + merged.Logfile = other.Logfile + } + if other.RepositoryRoot != merged.RepositoryRoot && other.RepositoryRoot != "" { + merged.RepositoryRoot = other.RepositoryRoot + } + + for k, v := range other.Env { + if merged.Env == nil { + merged.Env = make(map[string]string) + } + merged.Env[k] = v + } + + for k, v := range other.ExternalSecrets { + if merged.ExternalSecrets == nil { + merged.ExternalSecrets = make(map[string]secrets.ExternalSecret) + } + merged.ExternalSecrets[k] = v + } + + return merged +} diff --git a/dev/sg/internal/sgconf/BUILD.bazel b/dev/sg/internal/sgconf/BUILD.bazel index fb80335cff9..47736c0dfc0 100644 --- a/dev/sg/internal/sgconf/BUILD.bazel +++ b/dev/sg/internal/sgconf/BUILD.bazel @@ -1,5 +1,5 @@ -load("//dev:go_defs.bzl", "go_test") load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//dev:go_defs.bzl", "go_test") go_library( name = "sgconf", @@ -22,6 +22,12 @@ go_test( timeout = "short", srcs = ["config_test.go"], embed = [":sgconf"], + env = { + # This allows calls to root.RepositoryRoot() to return a fake root. + # Otherwise it will fail because it is being run not within the repository + # but within the bazel tree + "SG_FORCE_REPO_ROOT": "./fake_root", + }, deps = [ "//dev/sg/internal/run", "@com_github_google_go_cmp//cmp", diff --git a/dev/sg/internal/sgconf/config.go b/dev/sg/internal/sgconf/config.go index 8de378f76ef..31a49609201 100644 --- a/dev/sg/internal/sgconf/config.go +++ b/dev/sg/internal/sgconf/config.go @@ -8,6 +8,7 @@ import ( "gopkg.in/yaml.v2" "github.com/sourcegraph/sourcegraph/dev/sg/internal/run" + "github.com/sourcegraph/sourcegraph/dev/sg/root" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -32,15 +33,25 @@ func parseConfig(data []byte) (*Config, error) { return nil, err } + root, err := root.RepositoryRoot() + if err != nil { + return nil, err + } + for name, cmd := range conf.BazelCommands { - cmd.Name = name - conf.BazelCommands[name] = cmd + cmd.Config.Name = name + cmd.Config.RepositoryRoot = root + } + + for name, cmd := range conf.DockerCommands { + cmd.Config.Name = name + cmd.Config.RepositoryRoot = root } for name, cmd := range conf.Commands { - cmd.Name = name - normalizeCmd(&cmd) - conf.Commands[name] = cmd + cmd.Config.Name = name + cmd.Config.RepositoryRoot = root + normalizeCmd(cmd) } for name, cmd := range conf.Commandsets { @@ -49,8 +60,9 @@ func parseConfig(data []byte) (*Config, error) { } for name, cmd := range conf.Tests { - cmd.Name = name - normalizeCmd(&cmd) + cmd.Config.Name = name + cmd.Config.RepositoryRoot = root + normalizeCmd(cmd) conf.Tests[name] = cmd } @@ -64,11 +76,12 @@ func normalizeCmd(cmd *run.Command) { } type Commandset struct { - Name string `yaml:"-"` - Commands []string `yaml:"commands"` - BazelCommands []string `yaml:"bazelCommands"` - Checks []string `yaml:"checks"` - Env map[string]string `yaml:"env"` + Name string `yaml:"-"` + Commands []string `yaml:"commands"` + BazelCommands []string `yaml:"bazelCommands"` + DockerCommands []string `yaml:"dockerCommands"` + Checks []string `yaml:"checks"` + Env map[string]string `yaml:"env"` // If this is set to true, then the commandset requires the dev-private // repository to be cloned at the same level as the sourcegraph repository. @@ -114,6 +127,10 @@ func (c *Commandset) Merge(other *Commandset) *Commandset { merged.BazelCommands = other.BazelCommands } + if !equal(merged.DockerCommands, other.DockerCommands) && len(other.DockerCommands) != 0 { + merged.DockerCommands = other.DockerCommands + } + for k, v := range other.Env { merged.Env[k] = v } @@ -124,12 +141,13 @@ func (c *Commandset) Merge(other *Commandset) *Commandset { } type Config struct { - Env map[string]string `yaml:"env"` - Commands map[string]run.Command `yaml:"commands"` - BazelCommands map[string]run.BazelCommand `yaml:"bazelCommands"` - Commandsets map[string]*Commandset `yaml:"commandsets"` - DefaultCommandset string `yaml:"defaultCommandset"` - Tests map[string]run.Command `yaml:"tests"` + Env map[string]string `yaml:"env"` + Commands map[string]*run.Command `yaml:"commands"` + BazelCommands map[string]*run.BazelCommand `yaml:"bazelCommands"` + DockerCommands map[string]*run.DockerCommand `yaml:"dockerCommands"` + Commandsets map[string]*Commandset `yaml:"commandsets"` + DefaultCommandset string `yaml:"defaultCommandset"` + Tests map[string]*run.Command `yaml:"tests"` } // Merges merges the top-level entries of two Config objects, with the receiver @@ -141,7 +159,8 @@ func (c *Config) Merge(other *Config) { for k, v := range other.Commands { if original, ok := c.Commands[k]; ok { - c.Commands[k] = original.Merge(v) + merged := original.Merge(*v) + c.Commands[k] = &merged } else { c.Commands[k] = v } @@ -161,7 +180,8 @@ func (c *Config) Merge(other *Config) { for k, v := range other.Tests { if original, ok := c.Tests[k]; ok { - c.Tests[k] = original.Merge(v) + merged := original.Merge(*v) + c.Tests[k] = &merged } else { c.Tests[k] = v } diff --git a/dev/sg/internal/sgconf/config_test.go b/dev/sg/internal/sgconf/config_test.go index 9d9b24318f1..b2377a29a88 100644 --- a/dev/sg/internal/sgconf/config_test.go +++ b/dev/sg/internal/sgconf/config_test.go @@ -1,6 +1,7 @@ package sgconf import ( + "os" "testing" "github.com/google/go-cmp/cmp" @@ -47,13 +48,16 @@ commandsets: want := &Config{ Env: map[string]string{"SRC_REPOS_DIR": "$HOME/.sourcegraph/repos"}, - Commands: map[string]run.Command{ + Commands: map[string]*run.Command{ "frontend": { - Name: "frontend", + Config: run.SGConfigCommandOptions{ + Name: "frontend", + Env: map[string]string{"CONFIGURATION_MODE": "server"}, + RepositoryRoot: os.Getenv("SG_FORCE_REPO_ROOT"), + }, Cmd: "ulimit -n 10000 && .bin/frontend", Install: "go build -o .bin/frontend github.com/sourcegraph/sourcegraph/cmd/frontend", CheckBinary: ".bin/frontend", - Env: map[string]string{"CONFIGURATION_MODE": "server"}, Watch: []string{"lib"}, }, }, @@ -113,12 +117,15 @@ commands: t.Fatalf("command not found") } - want := run.Command{ - Name: "frontend", + want := &run.Command{ + Config: run.SGConfigCommandOptions{ + Name: "frontend", + Env: map[string]string{"EXTSVC_CONFIG_FILE": ""}, + RepositoryRoot: os.Getenv("SG_FORCE_REPO_ROOT"), + }, Cmd: ".bin/frontend", Install: "go build .bin/frontend github.com/sourcegraph/sourcegraph/cmd/frontend", CheckBinary: ".bin/frontend", - Env: map[string]string{"EXTSVC_CONFIG_FILE": ""}, Watch: []string{ "lib", "internal", diff --git a/dev/sg/sg_run.go b/dev/sg/sg_run.go index a6d52df6c5a..6aac3abce65 100644 --- a/dev/sg/sg_run.go +++ b/dev/sg/sg_run.go @@ -110,7 +110,7 @@ func runExec(ctx *cli.Context) error { if err != nil { return err } - if err = std.Out.WriteMarkdown(fmt.Sprintf("# %s\n\n```yaml\n%s\n```\n\n", cmd.GetName(), string(out))); err != nil { + if err = std.Out.WriteMarkdown(fmt.Sprintf("# %s\n\n```yaml\n%s\n```\n\n", cmd.GetConfig().Name, string(out))); err != nil { return err } } @@ -144,8 +144,8 @@ func constructRunCmdLongHelp() string { var names []string for name, command := range config.Commands { - if command.Description != "" { - name = fmt.Sprintf("%s: %s", name, command.Description) + if command.Config.Description != "" { + name = fmt.Sprintf("%s: %s", name, command.Config.Description) } names = append(names, name) } diff --git a/dev/sg/sg_start.go b/dev/sg/sg_start.go index fd82eacf55b..0bb5b242152 100644 --- a/dev/sg/sg_start.go +++ b/dev/sg/sg_start.go @@ -343,14 +343,14 @@ func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.C return err } - if len(cmds) == 0 && len(bcmds) == 0 { - std.Out.WriteLine(output.Styled(output.StyleWarning, "WARNING: no commands to run")) - return nil + dcmds, err := getCommands(set.DockerCommands, set, conf.DockerCommands) + if err != nil { + return err } - levelOverrides := logLevelOverrides() - for _, cmd := range cmds { - enrichWithLogLevels(&cmd, levelOverrides) + if len(cmds)+len(bcmds)+len(dcmds) == 0 { + std.Out.WriteLine(output.Styled(output.StyleWarning, "WARNING: no commands to run")) + return nil } env := conf.Env @@ -364,8 +364,16 @@ func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.C } var ibazel *run.IBazel - if len(bcmds) > 0 { - ibazel, err = run.NewIBazel(bcmds, repoRoot) + if len(bcmds)+len(dcmds) > 0 { + var targets []string + for _, cmd := range bcmds { + targets = append(targets, cmd.Target) + } + for _, cmd := range dcmds { + targets = append(targets, cmd.Target) + } + + ibazel, err = run.NewIBazel(targets, repoRoot) if err != nil { return err } @@ -380,14 +388,22 @@ func startCommandSet(ctx context.Context, set *sgconf.Commandset, conf *sgconf.C ibazel.StartOutput() } + levelOverrides := logLevelOverrides() configCmds := make([]run.SGConfigCommand, 0, len(bcmds)+len(cmds)) for _, cmd := range bcmds { + enrichWithLogLevels(&cmd.Config, levelOverrides) configCmds = append(configCmds, cmd) } for _, cmd := range cmds { + enrichWithLogLevels(&cmd.Config, levelOverrides) configCmds = append(configCmds, cmd) } + for _, cmd := range dcmds { + enrichWithLogLevels(&cmd.Config, levelOverrides) + configCmds = append(configCmds, cmd) + } + return run.Commands(ctx, env, verbose, configCmds...) } @@ -412,7 +428,7 @@ func getCommands[T run.SGConfigCommand](commands []string, set *sgconf.Commandse } if _, excluded := exceptSet[name]; excluded { - std.Out.WriteLine(output.Styledf(output.StylePending, "Skipping command %s since it's in --except.", cmd.GetName())) + std.Out.WriteLine(output.Styledf(output.StylePending, "Skipping command %s since it's in --except.", name)) continue } @@ -423,7 +439,7 @@ func getCommands[T run.SGConfigCommand](commands []string, set *sgconf.Commandse if _, inSet := onlySet[name]; inSet { cmds = append(cmds, cmd) } else { - std.Out.WriteLine(output.Styledf(output.StylePending, "Skipping command %s since it's not included in --only.", cmd.GetName())) + std.Out.WriteLine(output.Styledf(output.StylePending, "Skipping command %s since it's not included in --only.", name)) } } @@ -451,16 +467,16 @@ func logLevelOverrides() map[string]string { } // enrichWithLogLevels will add any logger level overrides to a given command if they have been specified. -func enrichWithLogLevels(cmd *run.Command, overrides map[string]string) { +func enrichWithLogLevels(config *run.SGConfigCommandOptions, overrides map[string]string) { logLevelVariable := "SRC_LOG_LEVEL" - if level, ok := overrides[cmd.Name]; ok { - std.Out.WriteLine(output.Styledf(output.StylePending, "Setting log level: %s for command %s.", level, cmd.Name)) - if cmd.Env == nil { - cmd.Env = make(map[string]string, 1) - cmd.Env[logLevelVariable] = level + if level, ok := overrides[config.Name]; ok { + std.Out.WriteLine(output.Styledf(output.StylePending, "Setting log level: %s for command %s.", level, config.Name)) + if config.Env == nil { + config.Env = make(map[string]string, 1) + config.Env[logLevelVariable] = level } - cmd.Env[logLevelVariable] = level + config.Env[logLevelVariable] = level } } diff --git a/dev/sg/sg_start_test.go b/dev/sg/sg_start_test.go index 1f952057f9d..1aa64b50d54 100644 --- a/dev/sg/sg_start_test.go +++ b/dev/sg/sg_start_test.go @@ -21,14 +21,16 @@ func TestStartCommandSet(t *testing.T) { buf := useOutputBuffer(t) commandSet := &sgconf.Commandset{Name: "test-set", Commands: []string{"test-cmd-1"}} - command := run.Command{ - Name: "test-cmd-1", + command := &run.Command{ + Config: run.SGConfigCommandOptions{ + Name: "test-cmd-1", + }, Install: "echo 'booting up horsegraph'", Cmd: "echo 'horsegraph booted up. mount your horse.' && echo 'quitting. not horsing around anymore.'", } testConf := &sgconf.Config{ - Commands: map[string]run.Command{"test-cmd-1": command}, + Commands: map[string]*run.Command{"test-cmd-1": command}, Commandsets: map[string]*sgconf.Commandset{"test-set": commandSet}, } @@ -61,14 +63,16 @@ func TestStartCommandSet_InstallError(t *testing.T) { buf := useOutputBuffer(t) commandSet := &sgconf.Commandset{Name: "test-set", Commands: []string{"test-cmd-1"}} - command := run.Command{ - Name: "test-cmd-1", + command := &run.Command{ + Config: run.SGConfigCommandOptions{ + Name: "test-cmd-1", + }, Install: "echo 'booting up horsegraph' && exit 1", Cmd: "echo 'never appears'", } testConf := &sgconf.Config{ - Commands: map[string]run.Command{"test-cmd-1": command}, + Commands: map[string]*run.Command{"test-cmd-1": command}, Commandsets: map[string]*sgconf.Commandset{"test-set": commandSet}, } diff --git a/dev/sg/sg_tests.go b/dev/sg/sg_tests.go index 06a95c7eb68..519ce2eae63 100644 --- a/dev/sg/sg_tests.go +++ b/dev/sg/sg_tests.go @@ -73,7 +73,7 @@ func testExec(ctx *cli.Context) error { return flag.ErrHelp } - return run.Test(ctx.Context, newSGTestCommand(cmd, args[1:]), config.Env) + return run.Test(ctx.Context, newSGTestCommand(*cmd, args[1:]), config.Env) } func constructTestCmdLongHelp() string { diff --git a/sg.config.yaml b/sg.config.yaml index 87e5116f37e..124c22dd625 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -265,6 +265,7 @@ commands: - internal - cmd/embeddings - internal/embeddings + qdrant: cmd: | docker run -p 6333:6333 -p 6334:6334 \ @@ -637,8 +638,7 @@ commands: sleep 1 done install: | - bazel build //cmd/executor-kubernetes:image_tarball - docker load --input $(bazel cquery //cmd/executor-kubernetes:image_tarball --output=files) + bazel run //cmd/executor-kubernetes:image_tarball env: IMAGE: executor-kubernetes:candidate @@ -1094,6 +1094,185 @@ bazelCommands: EXECUTOR_QUEUE_NAME: codeintel TMPDIR: $HOME/.sourcegraph/indexer-temp +dockerCommands: + batcheshelper-builder: + # Nothing to run for this, we just want to re-run the install script every time. + cmd: exit 0 + target: //cmd/batcheshelper:image_tarball + image: batcheshelper:candidate + env: + # TODO: This is required but should only be set on M1 Macs. + PLATFORM: linux/arm64 + continueWatchOnExit: true + + grafana: + target: //docker-images/grafana:image_tarball + docker: + image: grafana:candidate + ports: + - 3370 + flags: + cpus: 1 + memory: 1g + volumes: + - from: $HOME/.sourcegraph-dev/data/grafana + to: /var/lib/grafana + - from: $(pwd)/dev/grafana/all + to: /sg_config_grafana/provisioning/datasources + + linux: + flags: + # Linux needs an extra arg to support host.internal.docker, which is how grafana connects + # to the prometheus backend. + add-host: host.docker.internal:host-gateway + + # Docker users on Linux will generally be using direct user mapping, which + # means that they'll want the data in the volume mount to be owned by the + # same user as is running this script. Fortunately, the Grafana container + # doesn't really care what user it runs as, so long as it can write to + # /var/lib/grafana. + user: $UID + # Log file location: since we log outside of the Docker container, we should + # log somewhere that's _not_ ~/.sourcegraph-dev/data/grafana, since that gets + # volume mounted into the container and therefore has its own ownership + # semantics. + # Now for the actual logging. Grafana's output gets sent to stdout and stderr. + # We want to capture that output, but because it's fairly noisy, don't want to + # display it in the normal case. + logfile: $HOME/.sourcegraph-dev/logs/grafana/grafana.log + + env: + # docker containers must access things via docker host on non-linux platforms + CACHE: false + + loki: + logfile: $HOME/.sourcegraph-dev/logs/loki/loki.log + docker: + image: index.docker.io/grafana/loki:2.3.0 + pull: true + ports: + - 3100 + volumes: + - from: $HOME/.sourcegraph-dev/data/loki + to: /loki + + otel-collector: + target: //docker-images/opentelemetry-collector:image_tarball + description: OpenTelemetry collector + args: '--config "/etc/otel-collector/$CONFIGURATION_FILE"' + docker: + image: opentelemetry-collector:candidate + ports: + - 4317 + - 4318 + - 55679 + - 55670 + - 8888 + linux: + flags: + # Jaeger generally runs outside of Docker, so to access it we need to be + # able to access ports on the host, because the Docker host only exists on + # MacOS. --net=host is a very dirty way of enabling this. + net: host + env: + JAEGER_HOST: localhost + env: + JAEGER_HOST: host.docker.internal + # Overwrite the following in sg.config.overwrite.yaml, based on which collector + # config you are using - see docker-images/opentelemetry-collector for more details. + CONFIGURATION_FILE: 'configs/jaeger.yaml' + + postgres_exporter: + target: //docker-images/postgres_exporter:image_tarball + docker: + image: postgres-exporter:candidate + flags: + cpus: 1 + memory: 1g + ports: + - 9187 + linux: + flags: + # Linux needs an extra arg to support host.internal.docker, which is how + # postgres_exporter connects to the prometheus backend. + add-host: host.docker.internal:host-gateway + net: host + precmd: | + # Use psql to read the effective values for PG* env vars (instead of, e.g., hardcoding the default + # values). + get_pg_env() { psql -c '\set' | grep "$1" | cut -f 2 -d "'"; } + PGHOST=${PGHOST-$(get_pg_env HOST)} + PGUSER=${PGUSER-$(get_pg_env USER)} + PGPORT=${PGPORT-$(get_pg_env PORT)} + # we need to be able to query migration_logs table + PGDATABASE=${PGDATABASE-$(get_pg_env DBNAME)} + + ADJUSTED_HOST=${PGHOST:-127.0.0.1} + if [[ ("$ADJUSTED_HOST" == "localhost" || "$ADJUSTED_HOST" == "127.0.0.1" || -f "$ADJUSTED_HOST") && "$OSTYPE" != "linux-gnu" ]]; then + ADJUSTED_HOST="host.docker.internal" + fi + env: + DATA_SOURCE_NAME: postgresql://${PGUSER}:${PGPASSWORD}@${ADJUSTED_HOST}:${PGPORT}/${PGDATABASE}?sslmode=${PGSSLMODE:-disable} + + prometheus: + target: //docker-images/prometheus:image_tarball + logfile: $HOME/.sourcegraph-dev/logs/prometheus/prometheus.log + docker: + image: prometheus:candidate + volumes: + - from: $HOME/.sourcegraph-dev/data/prometheus + to: /prometheus + - from: $(pwd)/$CONFIG_DIR + to: /sg_prometheus_add_ons + flags: + cpus: 1 + memory: 4g + ports: + - 9090 + linux: + flags: + net: host + user: $UID + env: + PROM_TARGETS: dev/prometheus/linux/prometheus_targets.yml + SRC_FRONTEND_INTERNAL: localhost:3090 + precmd: cp ${PROM_TARGETS} "${CONFIG_DIR}"/prometheus_targets.yml + env: + CONFIG_DIR: docker-images/prometheus/config + PROM_TARGETS: dev/prometheus/all/prometheus_targets.yml + SRC_FRONTEND_INTERNAL: host.docker.internal:3090 + DISABLE_SOURCEGRAPH_CONFIG: false + DISABLE_ALERTMANAGER: false + PROMETHEUS_ADDITIONAL_FLAGS: '--web.enable-lifecycle --web.enable-admin-api' + + syntax-highlighter: + ignoreStdout: true + ignoreStderr: true + docker: + image: sourcegraph/syntax-highlighter:insiders + pull: true + ports: + - 9238 + env: + WORKERS: 1 + ROCKET_ADDRESS: 0.0.0.0 + + qdrant: + docker: + image: sourcegraph/qdrant:insiders + ports: + - 6333 + - 6334 + volumes: + - from: $HOME/.sourcegraph-dev/data/qdrant_data + to: /data + env: + QDRANT__SERVICE__GRPC_PORT: 6334 + QDRANT__LOG_LEVEL: INFO + QDRANT__STORAGE__STORAGE_PATH: /data + QDRANT__STORAGE__SNAPSHOTS_PATH: /data + QDRANT_INIT_FILE_PATH: /data/.qdrant-initialized + # # CommandSets ################################################################ # @@ -1493,6 +1672,17 @@ commandsets: - caddy monitoring: + checks: + - docker + commands: + - jaeger + dockerCommands: + - otel-collector + - prometheus + - grafana + - postgres_exporter + + monitoring-og: checks: - docker commands: