diff --git a/doc/dev/background-information/sg/reference.md b/doc/dev/background-information/sg/reference.md index 0e2f2652ce7..e7cc4874c61 100644 --- a/doc/dev/background-information/sg/reference.md +++ b/doc/dev/background-information/sg/reference.md @@ -86,6 +86,7 @@ Available commands in `sg.config.yaml`: * bext * caddy * codeintel-executor +* codeintel-executor-firecracker * codeintel-worker * debug-env: Debug env vars * docsite: Docsite instance serving the docs diff --git a/enterprise/cmd/executor/config.go b/enterprise/cmd/executor/config.go index c6bf20095c5..c884bea152a 100644 --- a/enterprise/cmd/executor/config.go +++ b/enterprise/cmd/executor/config.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/c2h5oh/datasize" "github.com/google/uuid" "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/apiclient" @@ -46,7 +47,7 @@ type Config struct { func defaultFirecrackerImageTag() string { // In dev, just use latest for convenience. if version.IsDev(version.Version()) { - return "latest" + return "insiders" } return version.Version() } @@ -80,9 +81,16 @@ func (c *Config) Load() { } func (c *Config) Validate() error { - if c.JobNumCPUs != 1 && c.JobNumCPUs%2 != 0 && c.UseFirecracker { - // Required by Firecracker: The vCPU number is invalid! The vCPU number can only be 1 or an even number when hyperthreading is enabled - c.AddError(errors.Newf("EXECUTOR_JOB_NUM_CPUS must be 1 or an even number")) + if c.UseFirecracker { + if c.JobNumCPUs != 1 && c.JobNumCPUs%2 != 0 { + // Required by Firecracker: The vCPU number is invalid! The vCPU number can only be 1 or an even number when hyperthreading is enabled + c.AddError(errors.Newf("EXECUTOR_JOB_NUM_CPUS must be 1 or an even number")) + } + + _, err := datasize.ParseString(c.FirecrackerDiskSpace) + if err != nil { + c.AddError(errors.Wrapf(err, "invalid disk size provided for EXECUTOR_FIRECRACKER_DISK_SPACE: %q", c.FirecrackerDiskSpace)) + } } if c.QueueName != "batches" && c.QueueName != "codeintel" { c.AddError(errors.Newf("EXECUTOR_QUEUE_NAME must be set to 'batches' or 'codeintel'")) diff --git a/enterprise/cmd/executor/internal/command/firecracker.go b/enterprise/cmd/executor/internal/command/firecracker.go index c6812030de6..477d9af36d1 100644 --- a/enterprise/cmd/executor/internal/command/firecracker.go +++ b/enterprise/cmd/executor/internal/command/firecracker.go @@ -53,8 +53,8 @@ func formatFirecrackerCommand(spec CommandSpec, name string, options Options) co // setupFirecracker invokes a set of commands to provision and prepare a Firecracker virtual // machine instance. If a startup script path (an executable file on the host) is supplied, // it will be mounted into the new virtual machine instance and executed. -func setupFirecracker(ctx context.Context, runner commandRunner, logger Logger, name, repoDir string, options Options, operations *Operations) error { - // Start the VM and wait for the SSH server to become available +func setupFirecracker(ctx context.Context, runner commandRunner, logger Logger, name, workspaceDevice string, options Options, operations *Operations) error { + // Start the VM and wait for the SSH server to become available. startCommand := command{ Key: "setup.firecracker.start", Command: flatten( @@ -62,7 +62,8 @@ func setupFirecracker(ctx context.Context, runner commandRunner, logger Logger, "--runtime", "docker", "--network-plugin", "cni", firecrackerResourceFlags(options.ResourceOptions), - firecrackerCopyfileFlags(repoDir, options.FirecrackerOptions.VMStartupScriptPath), + firecrackerCopyfileFlags(options.FirecrackerOptions.VMStartupScriptPath), + firecrackerVolumeFlags(workspaceDevice, firecrackerContainerDir), "--ssh", "--name", name, "--kernel-image", sanitizeImage(options.FirecrackerOptions.KernelImage), @@ -112,11 +113,8 @@ func firecrackerResourceFlags(options ResourceOptions) []string { } } -func firecrackerCopyfileFlags(dir, vmStartupScriptPath string) []string { - copyfiles := make([]string, 0, 2) - if dir != "" { - copyfiles = append(copyfiles, fmt.Sprintf("%s:%s", dir, firecrackerContainerDir)) - } +func firecrackerCopyfileFlags(vmStartupScriptPath string) []string { + copyfiles := make([]string, 0, 1) if vmStartupScriptPath != "" { copyfiles = append(copyfiles, fmt.Sprintf("%s:%s", vmStartupScriptPath, vmStartupScriptPath)) } @@ -125,6 +123,10 @@ func firecrackerCopyfileFlags(dir, vmStartupScriptPath string) []string { return intersperse("--copy-files", copyfiles) } +func firecrackerVolumeFlags(workspaceDevice, firecrackerContainerDir string) []string { + return []string{"--volumes", fmt.Sprintf("%s:%s", workspaceDevice, firecrackerContainerDir)} +} + var imagePattern = lazyregexp.New(`([^:@]+)(?::([^@]+))?(?:@sha256:([a-z0-9]{64}))?`) // sanitizeImage sanitizes the given docker image for use by ignite. The ignite utility diff --git a/enterprise/cmd/executor/internal/command/firecracker_test.go b/enterprise/cmd/executor/internal/command/firecracker_test.go index ccc18957592..282a01812b3 100644 --- a/enterprise/cmd/executor/internal/command/firecracker_test.go +++ b/enterprise/cmd/executor/internal/command/firecracker_test.go @@ -127,7 +127,7 @@ func TestSetupFirecracker(t *testing.T) { operations := NewOperations(&observation.TestContext) logger := NewMockLogger() - if err := setupFirecracker(context.Background(), runner, logger, "deadbeef", "/proj", options, operations); err != nil { + if err := setupFirecracker(context.Background(), runner, logger, "deadbeef", "/dev/loopX", options, operations); err != nil { t.Fatalf("unexpected error tearing down virtual machine: %s", err) } @@ -141,8 +141,8 @@ func TestSetupFirecracker(t *testing.T) { "ignite run", "--runtime docker --network-plugin cni", "--cpus 4 --memory 20G --size 1T", - "--copy-files /proj:/work", "--copy-files /vm-startup.sh:/vm-startup.sh", + "--volumes /dev/loopX:/work", "--ssh --name deadbeef", "--kernel-image", "ignite-kernel:5.10.135", "sourcegraph/executor-vm:3.43.1", diff --git a/enterprise/cmd/executor/internal/command/runner.go b/enterprise/cmd/executor/internal/command/runner.go index d03621bd749..3472bdc7307 100644 --- a/enterprise/cmd/executor/internal/command/runner.go +++ b/enterprise/cmd/executor/internal/command/runner.go @@ -86,11 +86,11 @@ func NewRunner(dir string, logger Logger, options Options, operations *Operation } return &firecrackerRunner{ - name: options.ExecutorName, - dir: dir, - logger: logger, - options: options, - operations: operations, + name: options.ExecutorName, + workspaceDevice: dir, + logger: logger, + options: options, + operations: operations, } } @@ -115,17 +115,17 @@ func (r *dockerRunner) Run(ctx context.Context, command CommandSpec) error { } type firecrackerRunner struct { - name string - dir string - logger Logger - options Options - operations *Operations + name string + workspaceDevice string + logger Logger + options Options + operations *Operations } var _ Runner = &firecrackerRunner{} func (r *firecrackerRunner) Setup(ctx context.Context) error { - return setupFirecracker(ctx, defaultRunner, r.logger, r.name, r.dir, r.options, r.operations) + return setupFirecracker(ctx, defaultRunner, r.logger, r.name, r.workspaceDevice, r.options, r.operations) } func (r *firecrackerRunner) Teardown(ctx context.Context) error { diff --git a/enterprise/cmd/executor/internal/worker/handler.go b/enterprise/cmd/executor/internal/worker/handler.go index 57088984541..587b46528e3 100644 --- a/enterprise/cmd/executor/internal/worker/handler.go +++ b/enterprise/cmd/executor/internal/worker/handler.go @@ -3,9 +3,6 @@ package worker import ( "context" "fmt" - "os" - "path/filepath" - "strings" "time" "github.com/google/uuid" @@ -100,26 +97,11 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut logger.Info("Creating workspace") hostRunner := h.runnerFactory("", commandLogger, command.Options{}, h.operations) - workspaceRoot, err := h.prepareWorkspace(ctx, hostRunner, job.RepositoryName, job.RepositoryDirectory, job.Commit, job.FetchTags, job.ShallowClone, job.SparseCheckout) + workspace, err := h.prepareWorkspace(ctx, hostRunner, job, commandLogger) if err != nil { return errors.Wrap(err, "failed to prepare workspace") } - defer func() { - if !h.options.KeepWorkspaces { - handle := commandLogger.Log("teardown.fs", nil) - - handle.Write([]byte(fmt.Sprintf("Removing %s\n", workspaceRoot))) - - if rmErr := os.RemoveAll(workspaceRoot); rmErr != nil { - handle.Write([]byte(fmt.Sprintf("Operation failed: %s\n", rmErr.Error()))) - } - - // We always finish this with exit code 0 even if it errored, because workspace - // cleanup doesn't fail the execution job. We can deal with it separately. - handle.Finalize(0) - handle.Close() - } - }() + defer workspace.Remove(ctx, h.options.KeepWorkspaces) vmNameSuffix, err := uuid.NewRandom() if err != nil { @@ -143,37 +125,7 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut FirecrackerOptions: h.options.FirecrackerOptions, ResourceOptions: h.options.ResourceOptions, } - runner := h.runnerFactory(workspaceRoot, commandLogger, options, h.operations) - - // Construct a map from filenames to file content that should be accessible to jobs - // within the workspace. This consists of files supplied within the job record itself, - // as well as file-version of each script step. - workspaceFileContentsByPath := map[string][]byte{} - - for relativePath, content := range job.VirtualMachineFiles { - path, err := filepath.Abs(filepath.Join(workspaceRoot, relativePath)) - if err != nil { - return err - } - if !strings.HasPrefix(path, workspaceRoot) { - return errors.Errorf("refusing to write outside of working directory") - } - - workspaceFileContentsByPath[path] = []byte(content) - } - - scriptNames := make([]string, 0, len(job.DockerSteps)) - for i, dockerStep := range job.DockerSteps { - scriptName := scriptNameFromJobStep(job, i) - scriptNames = append(scriptNames, scriptName) - - path := filepath.Join(workspaceRoot, command.ScriptsPath, scriptName) - workspaceFileContentsByPath[path] = buildScript(dockerStep) - } - - if err := writeFiles(workspaceFileContentsByPath, commandLogger); err != nil { - return errors.Wrap(err, "failed to write virtual machine files") - } + runner := h.runnerFactory(workspace.Path(), commandLogger, options, h.operations) logger.Info("Setting up VM") @@ -195,7 +147,7 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut dockerStepCommand := command.CommandSpec{ Key: fmt.Sprintf("step.docker.%d", i), Image: dockerStep.Image, - ScriptPath: scriptNames[i], + ScriptPath: workspace.ScriptFilenames()[i], Dir: dockerStep.Dir, Env: dockerStep.Env, Operation: h.operations.Exec, @@ -228,14 +180,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut return nil } -var scriptPreamble = ` -set -x -` - -func buildScript(dockerStep executor.DockerStep) []byte { - return []byte(strings.Join(append([]string{scriptPreamble, ""}, dockerStep.Commands...), "\n") + "\n") -} - func union(a, b map[string]string) map[string]string { c := make(map[string]string, len(a)+len(b)) @@ -249,44 +193,6 @@ func union(a, b map[string]string) map[string]string { return c } -func scriptNameFromJobStep(job executor.Job, i int) string { - return fmt.Sprintf("%d.%d_%s@%s.sh", job.ID, i, strings.ReplaceAll(job.RepositoryName, "/", "_"), job.Commit) -} - -// writeFiles writes to the filesystem the content in the given map. -func writeFiles(workspaceFileContentsByPath map[string][]byte, logger command.Logger) (err error) { - // Bail out early if nothing to do, we don't need to spawn an empty log group. - if len(workspaceFileContentsByPath) == 0 { - return nil - } - - handle := logger.Log("setup.fs", nil) - defer func() { - if err == nil { - handle.Finalize(0) - } else { - handle.Finalize(1) - } - - handle.Close() - }() - - for path, content := range workspaceFileContentsByPath { - // Ensure the path exists. - if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { - return err - } - - if err := os.WriteFile(path, content, os.ModePerm); err != nil { - return err - } - - handle.Write([]byte(fmt.Sprintf("Wrote %s\n", path))) - } - - return nil -} - func createHoneyEvent(_ context.Context, job executor.Job, err error, duration time.Duration) honey.Event { fields := map[string]any{ "duration_ms": duration.Milliseconds(), diff --git a/enterprise/cmd/executor/internal/worker/handler_test.go b/enterprise/cmd/executor/internal/worker/handler_test.go index 2d5194f0af4..158ffd9c8fa 100644 --- a/enterprise/cmd/executor/internal/worker/handler_test.go +++ b/enterprise/cmd/executor/internal/worker/handler_test.go @@ -12,15 +12,16 @@ import ( "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/janitor" + "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/worker/workspace" "github.com/sourcegraph/sourcegraph/enterprise/internal/executor" "github.com/sourcegraph/sourcegraph/internal/observation" ) func TestHandle(t *testing.T) { testDir := "/tmp/codeintel" - makeTempDir = func() (string, error) { return testDir, nil } + workspace.MakeTempDirectory = func(string) (string, error) { return testDir, nil } t.Cleanup(func() { - makeTempDir = makeTemporaryDirectory + workspace.MakeTempDirectory = workspace.MakeTemporaryDirectory }) if err := os.MkdirAll(filepath.Join(testDir, command.ScriptsPath), os.ModePerm); err != nil { diff --git a/enterprise/cmd/executor/internal/worker/workspace.go b/enterprise/cmd/executor/internal/worker/workspace.go index ecee4eb640f..5215a8af902 100644 --- a/enterprise/cmd/executor/internal/worker/workspace.go +++ b/enterprise/cmd/executor/internal/worker/workspace.go @@ -2,214 +2,49 @@ package worker import ( "context" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" - "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/worker/workspace" + "github.com/sourcegraph/sourcegraph/enterprise/internal/executor" ) -const SchemeExecutorToken = "token-executor" - -// These env vars should be set for git commands. We want to make sure it never hangs on interactive input. -var gitStdEnv = []string{"GIT_TERMINAL_PROMPT=0"} - -// prepareWorkspace creates and returns a temporary director in which acts the workspace +// prepareWorkspace creates and returns a temporary directory in which acts the workspace // while processing a single job. It is up to the caller to ensure that this directory is // removed after the job has finished processing. If a repository name is supplied, then // that repository will be cloned (through the frontend API) into the workspace. -func (h *handler) prepareWorkspace(ctx context.Context, commandRunner command.Runner, repositoryName, repositoryDirectory, commit string, fetchTags bool, shallowClone bool, sparseCheckout []string) (_ string, err error) { - tempDir, err := makeTempDir() - if err != nil { - return "", err - } - defer func() { - if err != nil { - _ = os.RemoveAll(tempDir) - } - }() - - if repositoryName != "" { - repoPath := tempDir - if repositoryDirectory != "" { - repoPath = filepath.Join(tempDir, repositoryDirectory) - - if !strings.HasPrefix(repoPath, tempDir) { - return "", errors.Newf("invalid repo path %q not a subdirectory of %q", repoPath, tempDir) - } - - if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { - return "", errors.Wrap(err, "creating repo directory") - } - } - - cloneURL, err := makeRelativeURL( - h.options.ClientOptions.EndpointOptions.URL, - h.options.GitServicePath, - repositoryName, - ) - if err != nil { - return "", err - } - - authorizationOption := fmt.Sprintf( - "http.extraHeader=Authorization: %s %s", - SchemeExecutorToken, - h.options.ClientOptions.EndpointOptions.Token, - ) - - fetchCommand := []string{ - "git", - "-C", repoPath, - "-c", "protocol.version=2", - "-c", authorizationOption, - "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", - "fetch", - "--progress", - "--no-recurse-submodules", - "origin", - commit, - } - - appendFetchArg := func(arg string) { - l := len(fetchCommand) - insertPos := l - 2 - fetchCommand = append(fetchCommand[:insertPos+1], fetchCommand[insertPos:]...) - fetchCommand[insertPos] = arg - } - - if fetchTags { - appendFetchArg("--tags") - } - - if shallowClone { - if !fetchTags { - appendFetchArg("--no-tags") - } - appendFetchArg("--depth=1") - } - - // For a sparse checkout, we want to add a blob filter so we only fetch the minimum set of files initially. - if len(sparseCheckout) > 0 { - appendFetchArg("--filter=blob:none") - } - - gitCommands := []command.CommandSpec{ - {Key: "setup.git.init", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "init"}, Operation: h.operations.SetupGitInit}, - {Key: "setup.git.add-remote", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "remote", "add", "origin", cloneURL.String()}, Operation: h.operations.SetupAddRemote}, - // Disable gc, this can improve performance and should never run for executor clones. - {Key: "setup.git.disable-gc", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "config", "--local", "gc.auto", "0"}, Operation: h.operations.SetupGitDisableGC}, - {Key: "setup.git.fetch", Env: gitStdEnv, Command: fetchCommand, Operation: h.operations.SetupGitFetch}, - } - - if len(sparseCheckout) > 0 { - gitCommands = append(gitCommands, command.CommandSpec{ - Key: "setup.git.sparse-checkout-config", - Env: gitStdEnv, - Command: []string{"git", "-C", repoPath, "config", "--local", "core.sparseCheckout", "1"}, - Operation: h.operations.SetupGitSparseCheckoutConfig, - }) - gitCommands = append(gitCommands, command.CommandSpec{ - Key: "setup.git.sparse-checkout-set", - Env: gitStdEnv, - Command: append([]string{"git", "-C", repoPath, "sparse-checkout", "set", "--no-cone", "--"}, sparseCheckout...), - Operation: h.operations.SetupGitSparseCheckoutSet, - }) - } - - checkoutCommand := []string{ - "git", - "-C", repoPath, - "checkout", - "--progress", - "--force", - commit, - } - - // Sparse checkouts need to fetch additional blobs, so we need to add - // auth config here. - if len(sparseCheckout) > 0 { - checkoutCommand = []string{ - "git", - "-C", repoPath, - "-c", "protocol.version=2", "-c", authorizationOption, "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", - "checkout", - "--progress", - "--force", - commit, - } - } - - gitCommands = append(gitCommands, command.CommandSpec{ - Key: "setup.git.checkout", - Env: gitStdEnv, - Command: checkoutCommand, - Operation: h.operations.SetupGitCheckout, - }) - - // This is for LSIF, it relies on the origin being set to the upstream repo - // for indexing. - gitCommands = append(gitCommands, command.CommandSpec{ - Key: "setup.git.set-remote", - Env: gitStdEnv, - Command: []string{ - "git", - "-C", repoPath, - "remote", - "set-url", - "origin", - repositoryName, +func (h *handler) prepareWorkspace( + ctx context.Context, + commandRunner command.Runner, + job executor.Job, + commandLogger command.Logger, +) (workspace.Workspace, error) { + if h.options.FirecrackerOptions.Enabled { + return workspace.NewFirecrackerWorkspace( + ctx, + job, + h.options.ResourceOptions.DiskSpace, + h.options.KeepWorkspaces, + commandRunner, + commandLogger, + workspace.CloneOptions{ + EndpointURL: h.options.ClientOptions.EndpointOptions.URL, + GitServicePath: h.options.GitServicePath, + ExecutorToken: h.options.ClientOptions.EndpointOptions.Token, }, - Operation: h.operations.SetupGitSetRemoteUrl, - }) - - for _, spec := range gitCommands { - if err := commandRunner.Run(ctx, spec); err != nil { - return "", errors.Wrap(err, fmt.Sprintf("failed %s", spec.Key)) - } - } + h.operations, + ) } - // Create the scripts path. - if err := os.MkdirAll(filepath.Join(tempDir, command.ScriptsPath), os.ModePerm); err != nil { - return "", errors.Wrap(err, "creating script path") - } - - return tempDir, nil -} - -func makeRelativeURL(base string, path ...string) (*url.URL, error) { - baseURL, err := url.Parse(base) - if err != nil { - return nil, err - } - - urlx, err := baseURL.ResolveReference(&url.URL{Path: filepath.Join(path...)}), nil - if err != nil { - return nil, err - } - - urlx.User = url.User("executor") - return urlx, nil -} - -// makeTempDir defaults to makeTemporaryDirectory and can be replaced for testing -// with determinstic workspace/scripts directories. -var makeTempDir = makeTemporaryDirectory - -func makeTemporaryDirectory() (string, error) { - // TMPDIR is set in the dev Procfile to avoid requiring developers to explicitly - // allow bind mounts of the host's /tmp. If this directory doesn't exist, - // os.MkdirTemp below will fail. - if tempdir := os.Getenv("TMPDIR"); tempdir != "" { - if err := os.MkdirAll(tempdir, os.ModePerm); err != nil { - return "", err - } - return os.MkdirTemp(tempdir, "") - } - - return os.MkdirTemp("", "") + return workspace.NewDockerWorkspace( + ctx, + job, + commandRunner, + commandLogger, + workspace.CloneOptions{ + EndpointURL: h.options.ClientOptions.EndpointOptions.URL, + GitServicePath: h.options.GitServicePath, + ExecutorToken: h.options.ClientOptions.EndpointOptions.Token, + }, + h.operations, + ) } diff --git a/enterprise/cmd/executor/internal/worker/workspace/clone.go b/enterprise/cmd/executor/internal/worker/workspace/clone.go new file mode 100644 index 00000000000..0ea707f40fd --- /dev/null +++ b/enterprise/cmd/executor/internal/worker/workspace/clone.go @@ -0,0 +1,184 @@ +package workspace + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" + "github.com/sourcegraph/sourcegraph/enterprise/internal/executor" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +const SchemeExecutorToken = "token-executor" + +// These env vars should be set for git commands. We want to make sure it never hangs on interactive input. +var gitStdEnv = []string{"GIT_TERMINAL_PROMPT=0"} + +func cloneRepo( + ctx context.Context, + workspaceDir string, + job executor.Job, + commandRunner command.Runner, + options CloneOptions, + operations *command.Operations, +) error { + repoPath := workspaceDir + if job.RepositoryDirectory != "" { + repoPath = filepath.Join(workspaceDir, job.RepositoryDirectory) + + if !strings.HasPrefix(repoPath, workspaceDir) { + return errors.Newf("invalid repo path %q not a subdirectory of %q", repoPath, workspaceDir) + } + + if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { + return errors.Wrap(err, "creating repo directory") + } + } + + cloneURL, err := makeRelativeURL( + options.EndpointURL, + options.GitServicePath, + job.RepositoryName, + ) + if err != nil { + return err + } + + authorizationOption := fmt.Sprintf( + "http.extraHeader=Authorization: %s %s", + SchemeExecutorToken, + options.ExecutorToken, + ) + + fetchCommand := []string{ + "git", + "-C", repoPath, + "-c", "protocol.version=2", + "-c", authorizationOption, + "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", + "fetch", + "--progress", + "--no-recurse-submodules", + "origin", + job.Commit, + } + + appendFetchArg := func(arg string) { + l := len(fetchCommand) + insertPos := l - 2 + fetchCommand = append(fetchCommand[:insertPos+1], fetchCommand[insertPos:]...) + fetchCommand[insertPos] = arg + } + + if job.FetchTags { + appendFetchArg("--tags") + } + + if job.ShallowClone { + if !job.FetchTags { + appendFetchArg("--no-tags") + } + appendFetchArg("--depth=1") + } + + // For a sparse checkout, we want to add a blob filter so we only fetch the minimum set of files initially. + if len(job.SparseCheckout) > 0 { + appendFetchArg("--filter=blob:none") + } + + gitCommands := []command.CommandSpec{ + {Key: "setup.git.init", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "init"}, Operation: operations.SetupGitInit}, + {Key: "setup.git.add-remote", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "remote", "add", "origin", cloneURL.String()}, Operation: operations.SetupAddRemote}, + // Disable gc, this can improve performance and should never run for executor clones. + {Key: "setup.git.disable-gc", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "config", "--local", "gc.auto", "0"}, Operation: operations.SetupGitDisableGC}, + {Key: "setup.git.fetch", Env: gitStdEnv, Command: fetchCommand, Operation: operations.SetupGitFetch}, + } + + if len(job.SparseCheckout) > 0 { + gitCommands = append(gitCommands, command.CommandSpec{ + Key: "setup.git.sparse-checkout-config", + Env: gitStdEnv, + Command: []string{"git", "-C", repoPath, "config", "--local", "core.sparseCheckout", "1"}, + Operation: operations.SetupGitSparseCheckoutConfig, + }) + gitCommands = append(gitCommands, command.CommandSpec{ + Key: "setup.git.sparse-checkout-set", + Env: gitStdEnv, + Command: append([]string{"git", "-C", repoPath, "sparse-checkout", "set", "--no-cone", "--"}, job.SparseCheckout...), + Operation: operations.SetupGitSparseCheckoutSet, + }) + } + + checkoutCommand := []string{ + "git", + "-C", repoPath, + "checkout", + "--progress", + "--force", + job.Commit, + } + + // Sparse checkouts need to fetch additional blobs, so we need to add + // auth config here. + if len(job.SparseCheckout) > 0 { + checkoutCommand = []string{ + "git", + "-C", repoPath, + "-c", "protocol.version=2", "-c", authorizationOption, "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", + "checkout", + "--progress", + "--force", + job.Commit, + } + } + + gitCommands = append(gitCommands, command.CommandSpec{ + Key: "setup.git.checkout", + Env: gitStdEnv, + Command: checkoutCommand, + Operation: operations.SetupGitCheckout, + }) + + // This is for LSIF, it relies on the origin being set to the upstream repo + // for indexing. + gitCommands = append(gitCommands, command.CommandSpec{ + Key: "setup.git.set-remote", + Env: gitStdEnv, + Command: []string{ + "git", + "-C", repoPath, + "remote", + "set-url", + "origin", + job.RepositoryName, + }, + Operation: operations.SetupGitSetRemoteUrl, + }) + + for _, spec := range gitCommands { + if err := commandRunner.Run(ctx, spec); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed %s", spec.Key)) + } + } + + return nil +} + +func makeRelativeURL(base string, path ...string) (*url.URL, error) { + baseURL, err := url.Parse(base) + if err != nil { + return nil, err + } + + urlx, err := baseURL.ResolveReference(&url.URL{Path: filepath.Join(path...)}), nil + if err != nil { + return nil, err + } + + urlx.User = url.User("executor") + return urlx, nil +} diff --git a/enterprise/cmd/executor/internal/worker/workspace/docker.go b/enterprise/cmd/executor/internal/worker/workspace/docker.go new file mode 100644 index 00000000000..e4c17b495e2 --- /dev/null +++ b/enterprise/cmd/executor/internal/worker/workspace/docker.go @@ -0,0 +1,82 @@ +package workspace + +import ( + "context" + "fmt" + "os" + "strconv" + + "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" + "github.com/sourcegraph/sourcegraph/enterprise/internal/executor" +) + +// NewDockerWorkspace creates a new workspace for docker-based execution. A path on +// the host will be used to set up the workspace, clone the repo and put script files. +func NewDockerWorkspace( + ctx context.Context, + job executor.Job, + commandRunner command.Runner, + logger command.Logger, + cloneOpts CloneOptions, + operations *command.Operations, +) (Workspace, error) { + workspaceDir, err := MakeTempDirectory("workspace-" + strconv.Itoa(job.ID)) + if err != nil { + return nil, err + } + + if job.RepositoryName != "" { + if err := cloneRepo(ctx, workspaceDir, job, commandRunner, cloneOpts, operations); err != nil { + _ = os.RemoveAll(workspaceDir) + return nil, err + } + } + + scriptPaths, err := prepareScripts(ctx, job, workspaceDir, commandRunner, logger) + if err != nil { + _ = os.RemoveAll(workspaceDir) + return nil, err + } + + return &dockerWorkspace{ + path: workspaceDir, + scriptFilenames: scriptPaths, + workspaceDir: workspaceDir, + logger: logger, + }, nil +} + +type dockerWorkspace struct { + path string + scriptFilenames []string + workspaceDir string + logger command.Logger +} + +func (w dockerWorkspace) Path() string { + return w.path +} + +func (w dockerWorkspace) ScriptFilenames() []string { + return w.scriptFilenames +} + +func (w dockerWorkspace) Remove(ctx context.Context, keepWorkspace bool) { + handle := w.logger.Log("teardown.fs", nil) + defer func() { + // We always finish this with exit code 0 even if it errored, because workspace + // cleanup doesn't fail the execution job. We can deal with it separately. + handle.Finalize(0) + handle.Close() + }() + + if keepWorkspace { + fmt.Fprintf(handle, "Preserving workspace (%s) as per config", w.workspaceDir) + return + } + + fmt.Fprintf(handle, "Removing %s\n", w.workspaceDir) + if rmErr := os.RemoveAll(w.workspaceDir); rmErr != nil { + fmt.Fprintf(handle, "Operation failed: %s\n", rmErr.Error()) + } +} diff --git a/enterprise/cmd/executor/internal/worker/workspace/files.go b/enterprise/cmd/executor/internal/worker/workspace/files.go new file mode 100644 index 00000000000..030b0a7bfc0 --- /dev/null +++ b/enterprise/cmd/executor/internal/worker/workspace/files.go @@ -0,0 +1,104 @@ +package workspace + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" + "github.com/sourcegraph/sourcegraph/enterprise/internal/executor" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +func prepareScripts( + ctx context.Context, + job executor.Job, + workspaceDir string, + commandRunner command.Runner, + commandLogger command.Logger, +) ([]string, error) { + // Create the scripts path. + if err := os.MkdirAll(filepath.Join(workspaceDir, command.ScriptsPath), os.ModePerm); err != nil { + return nil, errors.Wrap(err, "creating script path") + } + + // Construct a map from filenames to file content that should be accessible to jobs + // within the workspace. This consists of files supplied within the job record itself, + // as well as file-version of each script step. + workspaceFileContentsByPath := map[string][]byte{} + + for relativePath, content := range job.VirtualMachineFiles { + path, err := filepath.Abs(filepath.Join(workspaceDir, relativePath)) + if err != nil { + return nil, err + } + if !strings.HasPrefix(path, workspaceDir) { + return nil, errors.Errorf("refusing to write outside of working directory") + } + + workspaceFileContentsByPath[path] = []byte(content) + } + + scriptNames := make([]string, 0, len(job.DockerSteps)) + for i, dockerStep := range job.DockerSteps { + scriptName := scriptNameFromJobStep(job, i) + scriptNames = append(scriptNames, scriptName) + + path := filepath.Join(workspaceDir, command.ScriptsPath, scriptName) + workspaceFileContentsByPath[path] = buildScript(dockerStep) + } + + if err := writeFiles(workspaceFileContentsByPath, commandLogger); err != nil { + return nil, errors.Wrap(err, "failed to write virtual machine files") + } + + return scriptNames, nil +} + +var scriptPreamble = ` +set -x +` + +func buildScript(dockerStep executor.DockerStep) []byte { + return []byte(strings.Join(append([]string{scriptPreamble, ""}, dockerStep.Commands...), "\n") + "\n") +} + +func scriptNameFromJobStep(job executor.Job, i int) string { + return fmt.Sprintf("%d.%d_%s@%s.sh", job.ID, i, strings.ReplaceAll(job.RepositoryName, "/", "_"), job.Commit) +} + +// writeFiles writes the content of the given map to the filesystem. +func writeFiles(workspaceFileContentsByPath map[string][]byte, logger command.Logger) (err error) { + // Bail out early if nothing to do, we don't want to spawn an empty log group. + if len(workspaceFileContentsByPath) == 0 { + return nil + } + + handle := logger.Log("setup.fs.extras", nil) + defer func() { + if err == nil { + handle.Finalize(0) + } else { + handle.Finalize(1) + } + + handle.Close() + }() + + for path, content := range workspaceFileContentsByPath { + // Ensure the path exists. + if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { + return err + } + + if err := os.WriteFile(path, content, os.ModePerm); err != nil { + return err + } + + fmt.Fprintf(handle, "Wrote %s\n", path) + } + + return nil +} diff --git a/enterprise/cmd/executor/internal/worker/workspace/firecracker.go b/enterprise/cmd/executor/internal/worker/workspace/firecracker.go new file mode 100644 index 00000000000..71be80ce386 --- /dev/null +++ b/enterprise/cmd/executor/internal/worker/workspace/firecracker.go @@ -0,0 +1,234 @@ +package workspace + +import ( + "context" + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + + "github.com/c2h5oh/datasize" + + "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" + "github.com/sourcegraph/sourcegraph/enterprise/internal/executor" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +// NewFirecrackerWorkspace creates a new workspace for firecracker-based execution. +// A block device will be created on the host disk, with an ext4 file system. It +// is exposed through a loopback device. To set up the workspace, this device will +// be mounted and clone the repo and put script files in it. Then, the executor +// VM can mount this loopback device. This prevents host file system access. +func NewFirecrackerWorkspace( + ctx context.Context, + job executor.Job, + diskSpace string, + keepWorkspace bool, + commandRunner command.Runner, + logger command.Logger, + cloneOpts CloneOptions, + operations *command.Operations, +) (Workspace, error) { + blockDeviceFile, tmpMountDir, blockDevice, err := setupLoopDevice( + ctx, + job.ID, + diskSpace, + keepWorkspace, + logger, + ) + if err != nil { + return nil, err + } + + // Unmount the workspace volume when done, we finished writing to it from the host. + defer func() { + if err2 := syscall.Unmount(tmpMountDir, 0); err2 != nil { + err = errors.Append(err, err2) + return + } + if err2 := os.RemoveAll(tmpMountDir); err2 != nil { + err = errors.Append(err, err2) + } + }() + + if job.RepositoryName != "" { + if err := cloneRepo(ctx, tmpMountDir, job, commandRunner, cloneOpts, operations); err != nil { + return nil, err + } + } + + scriptPaths, err := prepareScripts(ctx, job, tmpMountDir, commandRunner, logger) + if err != nil { + return nil, err + } + + return &firecrackerWorkspace{ + scriptFilenames: scriptPaths, + blockDeviceFile: blockDeviceFile, + blockDevice: blockDevice, + logger: logger, + }, err +} + +type firecrackerWorkspace struct { + scriptFilenames []string + blockDeviceFile string + blockDevice string + logger command.Logger +} + +func (w firecrackerWorkspace) Path() string { + return w.blockDevice +} + +func (w firecrackerWorkspace) ScriptFilenames() []string { + return w.scriptFilenames +} + +func (w firecrackerWorkspace) Remove(ctx context.Context, keepWorkspace bool) { + handle := w.logger.Log("teardown.fs", nil) + defer func() { + // We always finish this with exit code 0 even if it errored, because workspace + // cleanup doesn't fail the execution job. We can deal with it separately. + handle.Finalize(0) + handle.Close() + }() + + if keepWorkspace { + fmt.Fprintf(handle, "Preserving workspace files (block device: %s, loop file: %s) as per config", w.blockDevice, w.blockDeviceFile) + // Remount the workspace, so that it can be inspected. + mountDir, err := mountLoopDevice(ctx, w.blockDevice, handle) + if err != nil { + fmt.Fprintf(handle, "Failed to mount workspace device %q, mount manually to inspect the contents: %s\n", w.blockDevice, err) + return + } + fmt.Fprintf(handle, "Inspect the workspace contents at: %s\n", mountDir) + return + } + + fmt.Fprintf(handle, "Removing loop device %s\n", w.blockDevice) + if err := detachLoopDevice(ctx, w.blockDevice, handle); err != nil { + fmt.Fprintf(handle, "stderr: Failed to detach loop device: %s\n", err) + } + + fmt.Fprintf(handle, "Removing block device file %s\n", w.blockDeviceFile) + if err := os.Remove(w.blockDeviceFile); err != nil { + fmt.Fprintf(handle, "stderr: Failed to remove block device: %s\n", err) + } +} + +// setupLoopDevice is used in firecracker mode. It creates a block device on disk, +// creates a loop device pointing to it, and mounts it so that it can be written to. +// The loop device will be given to ignite and mounted into the guest VM. +func setupLoopDevice( + ctx context.Context, + jobID int, + diskSpace string, + keepWorkspace bool, + logger command.Logger, +) (blockDeviceFile, tmpMountDir, blockDevice string, err error) { + handle := logger.Log("setup.fs.workspace", nil) + defer func() { + // add the error to the bottom of the step's log output, + // but only if this isnt from exec.Command, as those get added + // by our logging wrapper + if !errors.HasType(err, &exec.ExitError{}) { + fmt.Fprint(handle, err.Error()) + } + if err != nil { + handle.Finalize(1) + } else { + handle.Finalize(0) + } + handle.Close() + }() + + // Create a temp file to hold the block device on disk. + loopFile, err := MakeTempFile("workspace-loop-" + strconv.Itoa(jobID)) + if err != nil { + return "", "", "", err + } + defer func() { + if err != nil && !keepWorkspace { + os.Remove(loopFile.Name()) + } + }() + blockDeviceFile = loopFile.Name() + fmt.Fprintf(handle, "Created backing workspace file at %q\n", blockDeviceFile) + + // Truncate the file to be of the size of the maximum permissible disk space. + diskSize, err := datasize.ParseString(diskSpace) + if err != nil { + return "", "", "", errors.Wrapf(err, "invalid disk size provided: %q", diskSpace) + } + if err := loopFile.Truncate(int64(diskSize.Bytes())); err != nil { + return "", "", "", errors.Wrapf(err, "failed to make backing file sparse with %d bytes", diskSize.Bytes()) + } + fmt.Fprintf(handle, "Created sparse file of size %s from %q\n", diskSize.HumanReadable(), blockDeviceFile) + if err := loopFile.Close(); err != nil { + return "", "", "", errors.Wrap(err, "failed to close backing file") + } + + // Create an ext4 file system in the device backing file. + out, err := commandLogger(ctx, handle, "mkfs.ext4", blockDeviceFile) + if err != nil { + return "", "", "", errors.Wrapf(err, "failed to create ext4 filesystem in backing file: %q", out) + } + + fmt.Fprintf(handle, "Wrote ext4 filesystem to backing file %q\n", blockDeviceFile) + + // Create a loop device pointing to our block device. + out, err = commandLogger(ctx, handle, "losetup", "--find", "--show", blockDeviceFile) + if err != nil { + return "", "", "", errors.Wrapf(err, "failed to create loop device: %q", out) + } + blockDevice = strings.TrimSpace(out) + defer func() { + // If something further down in this function failed we detach the loop device + // to not hoard them. + if err != nil { + err := detachLoopDevice(ctx, blockDevice, handle) + if err != nil { + fmt.Fprint(handle, "stderr: "+strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", "\nstderr: ")) + } + } + }() + fmt.Fprintf(handle, "Created loop device at %q backed by %q\n", blockDevice, blockDeviceFile) + + // Mount the loop device at a temporary directory so we can write the workspace contents to it. + tmpMountDir, err = mountLoopDevice(ctx, blockDevice, handle) + if err != nil { + // important to set at least blockDevice for the above defer + return blockDeviceFile, "", blockDevice, err + } + fmt.Fprintf(handle, "Created temporary workspace mount location at %q\n", tmpMountDir) + + return blockDeviceFile, tmpMountDir, blockDevice, nil +} + +// detachLoopDevice detaches a loop device by path (/dev/loopX). +func detachLoopDevice(ctx context.Context, blockDevice string, handle command.LogEntry) error { + out, err := commandLogger(ctx, handle, "losetup", "--detach", blockDevice) + if err != nil { + return errors.Wrapf(err, "failed to detach loop device: %s", out) + } + return nil +} + +// mountLoopDevice takes a path to a loop device (/dev/loopX) and mounts it at a +// random temporary mount point. The mount point is returned. +func mountLoopDevice(ctx context.Context, blockDevice string, handle command.LogEntry) (string, error) { + tmpMountDir, err := MakeTempDirectory("workspace-mountpoints") + if err != nil { + return "", err + } + + if out, err := commandLogger(ctx, handle, "mount", blockDevice, tmpMountDir); err != nil { + _ = os.RemoveAll(tmpMountDir) + return "", errors.Wrapf(err, "failed to mount loop device %q to %q: %q", blockDevice, tmpMountDir, out) + } + + return tmpMountDir, nil +} diff --git a/enterprise/cmd/executor/internal/worker/workspace/iface.go b/enterprise/cmd/executor/internal/worker/workspace/iface.go new file mode 100644 index 00000000000..c84ae76289d --- /dev/null +++ b/enterprise/cmd/executor/internal/worker/workspace/iface.go @@ -0,0 +1,21 @@ +package workspace + +import "context" + +type CloneOptions struct { + EndpointURL string + GitServicePath string + ExecutorToken string +} + +type Workspace interface { + // Path represents the block device path when firecracker is enabled and the + // directory when firecracker is disabled where the workspace is configured. + Path() string + // ScriptFilenames holds the ordered set of script filenames to be invoked. + ScriptFilenames() []string + // Remove cleans up the workspace post execution. If keep workspace is true, + // the implementation will only clean up additional resources, while keeping + // the workspace contents on disk for debugging purposes. + Remove(ctx context.Context, keepWorkspace bool) +} diff --git a/enterprise/cmd/executor/internal/worker/workspace/util.go b/enterprise/cmd/executor/internal/worker/workspace/util.go new file mode 100644 index 00000000000..b326588b301 --- /dev/null +++ b/enterprise/cmd/executor/internal/worker/workspace/util.go @@ -0,0 +1,55 @@ +package workspace + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" +) + +// MakeTempFile defaults to makeTemporaryFile and can be replaced for testing +// with determinstic workspace/scripts directories. +var MakeTempFile = makeTemporaryFile + +func makeTemporaryFile(prefix string) (*os.File, error) { + if tempdir := os.Getenv("TMPDIR"); tempdir != "" { + if err := os.MkdirAll(tempdir, os.ModePerm); err != nil { + return nil, err + } + return os.CreateTemp(tempdir, prefix+"-*") + } + + return os.CreateTemp("", prefix+"-*") +} + +// MakeTempDirectory defaults to makeTemporaryDirectory and can be replaced for testing +// with determinstic workspace/scripts directories. +var MakeTempDirectory = MakeTemporaryDirectory + +func MakeTemporaryDirectory(prefix string) (string, error) { + if tempdir := os.Getenv("TMPDIR"); tempdir != "" { + if err := os.MkdirAll(tempdir, os.ModePerm); err != nil { + return "", err + } + return os.MkdirTemp(tempdir, prefix+"-*") + } + + return os.MkdirTemp("", prefix+"-*") +} + +// runs the given command with args and logs the invocation and output to the provided log entry handle. +func commandLogger(ctx context.Context, handle command.LogEntry, command string, args ...string) (string, error) { + fmt.Fprintf(handle, "$ %s %s\n", command, strings.Join(args, " ")) + cmd := exec.CommandContext(ctx, command, args...) + out, err := cmd.CombinedOutput() + if len(out) == 0 { + fmt.Fprint(handle, "stderr: \n") + } else { + fmt.Fprintf(handle, "stderr: %s\n", strings.ReplaceAll(strings.TrimSpace(string(out)), "\n", "\nstderr: ")) + } + + return string(out), err +} diff --git a/enterprise/cmd/executor/internal/worker/workspace_test.go b/enterprise/cmd/executor/internal/worker/workspace_test.go index a8479f60927..982bfeaf80b 100644 --- a/enterprise/cmd/executor/internal/worker/workspace_test.go +++ b/enterprise/cmd/executor/internal/worker/workspace_test.go @@ -10,6 +10,7 @@ import ( "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/apiclient" "github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command" + "github.com/sourcegraph/sourcegraph/enterprise/internal/executor" "github.com/sourcegraph/sourcegraph/internal/observation" ) @@ -29,11 +30,15 @@ func TestPrepareWorkspace_Clone(t *testing.T) { operations: command.NewOperations(&observation.TestContext), } - dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "", "deadbeef", true, false, []string{}) + workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{ + RepositoryName: "torvalds/linux", + Commit: "deadbeef", + FetchTags: true, + }, nil) if err != nil { t.Fatalf("unexpected error preparing workspace: %s", err) } - defer os.RemoveAll(dir) + defer os.RemoveAll(workspace.Path()) if value := len(runner.RunFunc.History()); value != 6 { t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 6, value) @@ -45,12 +50,12 @@ func TestPrepareWorkspace_Clone(t *testing.T) { } expectedCommands := [][]string{ - {"git", "-C", dir, "init"}, - {"git", "-C", dir, "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"}, - {"git", "-C", dir, "config", "--local", "gc.auto", "0"}, - {"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--tags", "origin", "deadbeef"}, - {"git", "-C", dir, "checkout", "--progress", "--force", "deadbeef"}, - {"git", "-C", dir, "remote", "set-url", "origin", "torvalds/linux"}, + {"git", "-C", workspace.Path(), "init"}, + {"git", "-C", workspace.Path(), "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"}, + {"git", "-C", workspace.Path(), "config", "--local", "gc.auto", "0"}, + {"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--tags", "origin", "deadbeef"}, + {"git", "-C", workspace.Path(), "checkout", "--progress", "--force", "deadbeef"}, + {"git", "-C", workspace.Path(), "remote", "set-url", "origin", "torvalds/linux"}, } if diff := cmp.Diff(expectedCommands, commands); diff != "" { t.Errorf("unexpected commands (-want +got):\n%s", diff) @@ -73,13 +78,17 @@ func TestPrepareWorkspace_Clone_Subdirectory(t *testing.T) { operations: command.NewOperations(&observation.TestContext), } - dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "subdirectory", "deadbeef", false, false, []string{}) + workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{ + RepositoryName: "torvalds/linux", + RepositoryDirectory: "subdirectory", + Commit: "deadbeef", + }, nil) if err != nil { t.Fatalf("unexpected error preparing workspace: %s", err) } - defer os.RemoveAll(dir) + defer os.RemoveAll(workspace.Path()) - repoDir := filepath.Join(dir, "subdirectory") + repoDir := filepath.Join(workspace.Path(), "subdirectory") if value := len(runner.RunFunc.History()); value != 6 { t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 6, value) @@ -119,11 +128,15 @@ func TestPrepareWorkspace_ShallowClone(t *testing.T) { operations: command.NewOperations(&observation.TestContext), } - dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "", "deadbeef", false, true, []string{}) + workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{ + RepositoryName: "torvalds/linux", + Commit: "deadbeef", + ShallowClone: true, + }, nil) if err != nil { t.Fatalf("unexpected error preparing workspace: %s", err) } - defer os.RemoveAll(dir) + defer os.RemoveAll(workspace.Path()) if value := len(runner.RunFunc.History()); value != 6 { t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 6, value) @@ -135,12 +148,12 @@ func TestPrepareWorkspace_ShallowClone(t *testing.T) { } expectedCommands := [][]string{ - {"git", "-C", dir, "init"}, - {"git", "-C", dir, "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"}, - {"git", "-C", dir, "config", "--local", "gc.auto", "0"}, - {"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "origin", "deadbeef"}, - {"git", "-C", dir, "checkout", "--progress", "--force", "deadbeef"}, - {"git", "-C", dir, "remote", "set-url", "origin", "torvalds/linux"}, + {"git", "-C", workspace.Path(), "init"}, + {"git", "-C", workspace.Path(), "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"}, + {"git", "-C", workspace.Path(), "config", "--local", "gc.auto", "0"}, + {"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "origin", "deadbeef"}, + {"git", "-C", workspace.Path(), "checkout", "--progress", "--force", "deadbeef"}, + {"git", "-C", workspace.Path(), "remote", "set-url", "origin", "torvalds/linux"}, } if diff := cmp.Diff(expectedCommands, commands); diff != "" { t.Errorf("unexpected commands (-want +got):\n%s", diff) @@ -163,11 +176,16 @@ func TestPrepareWorkspace_SparseCheckout(t *testing.T) { operations: command.NewOperations(&observation.TestContext), } - dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "", "deadbeef", false, true, []string{"kernel"}) + workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{ + RepositoryName: "torvalds/linux", + Commit: "deadbeef", + ShallowClone: true, + SparseCheckout: []string{"kernel"}, + }, nil) if err != nil { t.Fatalf("unexpected error preparing workspace: %s", err) } - defer os.RemoveAll(dir) + defer os.RemoveAll(workspace.Path()) if value := len(runner.RunFunc.History()); value != 8 { t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 8, value) @@ -179,14 +197,14 @@ func TestPrepareWorkspace_SparseCheckout(t *testing.T) { } expectedCommands := [][]string{ - {"git", "-C", dir, "init"}, - {"git", "-C", dir, "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"}, - {"git", "-C", dir, "config", "--local", "gc.auto", "0"}, - {"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "--filter=blob:none", "origin", "deadbeef"}, - {"git", "-C", dir, "config", "--local", "core.sparseCheckout", "1"}, - {"git", "-C", dir, "sparse-checkout", "set", "--no-cone", "--", "kernel"}, - {"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "checkout", "--progress", "--force", "deadbeef"}, - {"git", "-C", dir, "remote", "set-url", "origin", "torvalds/linux"}, + {"git", "-C", workspace.Path(), "init"}, + {"git", "-C", workspace.Path(), "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"}, + {"git", "-C", workspace.Path(), "config", "--local", "gc.auto", "0"}, + {"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "--filter=blob:none", "origin", "deadbeef"}, + {"git", "-C", workspace.Path(), "config", "--local", "core.sparseCheckout", "1"}, + {"git", "-C", workspace.Path(), "sparse-checkout", "set", "--no-cone", "--", "kernel"}, + {"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "checkout", "--progress", "--force", "deadbeef"}, + {"git", "-C", workspace.Path(), "remote", "set-url", "origin", "torvalds/linux"}, } if diff := cmp.Diff(expectedCommands, commands); diff != "" { t.Errorf("unexpected commands (-want +got):\n%s", diff) @@ -201,11 +219,11 @@ func TestPrepareWorkspace_NoRepository(t *testing.T) { operations: command.NewOperations(&observation.TestContext), } - dir, err := handler.prepareWorkspace(context.Background(), runner, "", "", "", false, false, []string{}) + workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{}, nil) if err != nil { t.Fatalf("unexpected error preparing workspace: %s", err) } - defer os.RemoveAll(dir) + defer os.RemoveAll(workspace.Path()) if value := len(runner.RunFunc.History()); value != 0 { t.Fatalf("unexpected call to Run") diff --git a/go.mod b/go.mod index cf721bd58da..8413b7d540a 100644 --- a/go.mod +++ b/go.mod @@ -215,6 +215,7 @@ require ( require github.com/hmarr/codeowners v0.4.0 require ( + github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b github.com/hashicorp/hcl v1.0.0 github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.13 github.com/prometheus/prometheus v0.37.1 diff --git a/go.sum b/go.sum index 79249dd069c..b43620417eb 100644 --- a/go.sum +++ b/go.sum @@ -386,6 +386,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/buildkite/go-buildkite/v3 v3.0.1 h1:5kX1fFDj3Co7cP6cqZKuW1VoCJz3u4cOx6wfdCeM4ZA= github.com/buildkite/go-buildkite/v3 v3.0.1/go.mod h1:6pweknacVv7He5Lvbf54urp2P0W6/b4Nrcxn718PQrE= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= +github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw= github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo= github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A= @@ -1249,8 +1251,6 @@ github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7 github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/goware/urlx v0.3.1 h1:BbvKl8oiXtJAzOzMqAQ0GfIhf96fKeNEZfm9ocNSUBI= github.com/goware/urlx v0.3.1/go.mod h1:h8uwbJy68o+tQXCGZNa9D73WN8n0r9OBae5bUnLcgjw= -github.com/grafana-tools/sdk v0.0.0-20220203092117-edae16afa87b h1:R9LID2XreyUOQfJ/NKLGuYOF4/Wz6ljmYFAhlOaHVQ4= -github.com/grafana-tools/sdk v0.0.0-20220203092117-edae16afa87b/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc h1:PXZQA2WCxe85Tnn+WEvr8fDpfwibmEPgfgFEaC87G24= github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= github.com/grafana/regexp v0.0.0-20220304100321-149c8afcd6cb h1:wwzNkyaQwcXCzQuKoWz3lwngetmcyg+EhW0fF5lz73M= diff --git a/sg.config.yaml b/sg.config.yaml index 88405f25c28..4d5da5cfd7b 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -485,6 +485,19 @@ commands: EXECUTOR_QUEUE_NAME: codeintel SRC_PROF_HTTP: ':6092' + # If you want to use this, either start it with `sg run batches-executor-firecracker` or + # modify the `commandsets.batches` in your local `sg.config.overwrite.yaml` + codeintel-executor-firecracker: + <<: *executor_template + cmd: | + env TMPDIR="$HOME/.sourcegraph/codeintel-executor-temp" \ + sudo --preserve-env=TMPDIR,EXECUTOR_QUEUE_NAME,SRC_PROF_HTTP,EXECUTOR_FRONTEND_URL,EXECUTOR_FRONTEND_PASSWORD,EXECUTOR_USE_FIRECRACKER,EXECUTOR_IMAGE_ARCHIVE_PATH \ + .bin/executor + env: + EXECUTOR_USE_FIRECRACKER: true + EXECUTOR_QUEUE_NAME: codeintel + SRC_PROF_HTTP: ':6093' + batches-executor: <<: *executor_template cmd: |