artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 15:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

executors: Reduce disk IO by attaching workspace as block device (#41335)

Browse Source 
Co-authored-by: Erik Seliger <erikseliger@me.com>
This commit is contained in:
Noah S-C 2022-09-21 20:39:50 +01:00 committed by GitHub
parent c43e379b6e
commit 8f097574f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 823 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/dev/background-information/sg/reference.md
View File

@ -86,6 +86,7 @@ Available commands in `sg.config.yaml`:
* bext
* caddy
* codeintel-executor
* codeintel-executor-firecracker
* codeintel-worker
* debug-env: Debug env vars
* docsite: Docsite instance serving the docs

16
enterprise/cmd/executor/config.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"time"
"github.com/c2h5oh/datasize"
"github.com/google/uuid"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/apiclient"
@ -46,7 +47,7 @@ type Config struct {
func defaultFirecrackerImageTag() string {
// In dev, just use latest for convenience.
if version.IsDev(version.Version()) {
return "latest"
return "insiders"
}
return version.Version()
}
@ -80,9 +81,16 @@ func (c *Config) Load() {
}
func (c *Config) Validate() error {
if c.JobNumCPUs != 1 && c.JobNumCPUs%2 != 0 && c.UseFirecracker {
// Required by Firecracker: The vCPU number is invalid! The vCPU number can only be 1 or an even number when hyperthreading is enabled
c.AddError(errors.Newf("EXECUTOR_JOB_NUM_CPUS must be 1 or an even number"))
if c.UseFirecracker {
if c.JobNumCPUs != 1 && c.JobNumCPUs%2 != 0 {
// Required by Firecracker: The vCPU number is invalid! The vCPU number can only be 1 or an even number when hyperthreading is enabled
c.AddError(errors.Newf("EXECUTOR_JOB_NUM_CPUS must be 1 or an even number"))
}
_, err := datasize.ParseString(c.FirecrackerDiskSpace)
if err != nil {
c.AddError(errors.Wrapf(err, "invalid disk size provided for EXECUTOR_FIRECRACKER_DISK_SPACE: %q", c.FirecrackerDiskSpace))
}
}
if c.QueueName != "batches" && c.QueueName != "codeintel" {
c.AddError(errors.Newf("EXECUTOR_QUEUE_NAME must be set to 'batches' or 'codeintel'"))

18
enterprise/cmd/executor/internal/command/firecracker.go
View File

@ -53,8 +53,8 @@ func formatFirecrackerCommand(spec CommandSpec, name string, options Options) co
// setupFirecracker invokes a set of commands to provision and prepare a Firecracker virtual
// machine instance. If a startup script path (an executable file on the host) is supplied,
// it will be mounted into the new virtual machine instance and executed.
func setupFirecracker(ctx context.Context, runner commandRunner, logger Logger, name, repoDir string, options Options, operations *Operations) error {
// Start the VM and wait for the SSH server to become available
func setupFirecracker(ctx context.Context, runner commandRunner, logger Logger, name, workspaceDevice string, options Options, operations *Operations) error {
// Start the VM and wait for the SSH server to become available.
startCommand := command{
Key: "setup.firecracker.start",
Command: flatten(
@ -62,7 +62,8 @@ func setupFirecracker(ctx context.Context, runner commandRunner, logger Logger,
"--runtime", "docker",
"--network-plugin", "cni",
firecrackerResourceFlags(options.ResourceOptions),
firecrackerCopyfileFlags(repoDir, options.FirecrackerOptions.VMStartupScriptPath),
firecrackerCopyfileFlags(options.FirecrackerOptions.VMStartupScriptPath),
firecrackerVolumeFlags(workspaceDevice, firecrackerContainerDir),
"--ssh",
"--name", name,
"--kernel-image", sanitizeImage(options.FirecrackerOptions.KernelImage),
@ -112,11 +113,8 @@ func firecrackerResourceFlags(options ResourceOptions) []string {
}
}
func firecrackerCopyfileFlags(dir, vmStartupScriptPath string) []string {
copyfiles := make([]string, 0, 2)
if dir != "" {
copyfiles = append(copyfiles, fmt.Sprintf("%s:%s", dir, firecrackerContainerDir))
}
func firecrackerCopyfileFlags(vmStartupScriptPath string) []string {
copyfiles := make([]string, 0, 1)
if vmStartupScriptPath != "" {
copyfiles = append(copyfiles, fmt.Sprintf("%s:%s", vmStartupScriptPath, vmStartupScriptPath))
}
@ -125,6 +123,10 @@ func firecrackerCopyfileFlags(dir, vmStartupScriptPath string) []string {
return intersperse("--copy-files", copyfiles)
}
func firecrackerVolumeFlags(workspaceDevice, firecrackerContainerDir string) []string {
return []string{"--volumes", fmt.Sprintf("%s:%s", workspaceDevice, firecrackerContainerDir)}
}
var imagePattern = lazyregexp.New(`([^:@]+)(?::([^@]+))?(?:@sha256:([a-z0-9]{64}))?`)
// sanitizeImage sanitizes the given docker image for use by ignite. The ignite utility

4
enterprise/cmd/executor/internal/command/firecracker_test.go
View File

@ -127,7 +127,7 @@ func TestSetupFirecracker(t *testing.T) {
operations := NewOperations(&observation.TestContext)
logger := NewMockLogger()
if err := setupFirecracker(context.Background(), runner, logger, "deadbeef", "/proj", options, operations); err != nil {
if err := setupFirecracker(context.Background(), runner, logger, "deadbeef", "/dev/loopX", options, operations); err != nil {
t.Fatalf("unexpected error tearing down virtual machine: %s", err)
}
@ -141,8 +141,8 @@ func TestSetupFirecracker(t *testing.T) {
"ignite run",
"--runtime docker --network-plugin cni",
"--cpus 4 --memory 20G --size 1T",
"--copy-files /proj:/work",
"--copy-files /vm-startup.sh:/vm-startup.sh",
"--volumes /dev/loopX:/work",
"--ssh --name deadbeef",
"--kernel-image", "ignite-kernel:5.10.135",
"sourcegraph/executor-vm:3.43.1",

22
enterprise/cmd/executor/internal/command/runner.go
View File

@ -86,11 +86,11 @@ func NewRunner(dir string, logger Logger, options Options, operations *Operation
}
return &firecrackerRunner{
name: options.ExecutorName,
dir: dir,
logger: logger,
options: options,
operations: operations,
name: options.ExecutorName,
workspaceDevice: dir,
logger: logger,
options: options,
operations: operations,
}
}
@ -115,17 +115,17 @@ func (r *dockerRunner) Run(ctx context.Context, command CommandSpec) error {
}
type firecrackerRunner struct {
name string
dir string
logger Logger
options Options
operations *Operations
name string
workspaceDevice string
logger Logger
options Options
operations *Operations
}
var _ Runner = &firecrackerRunner{}
func (r *firecrackerRunner) Setup(ctx context.Context) error {
return setupFirecracker(ctx, defaultRunner, r.logger, r.name, r.dir, r.options, r.operations)
return setupFirecracker(ctx, defaultRunner, r.logger, r.name, r.workspaceDevice, r.options, r.operations)
}
func (r *firecrackerRunner) Teardown(ctx context.Context) error {

102
enterprise/cmd/executor/internal/worker/handler.go
View File

@ -3,9 +3,6 @@ package worker
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/google/uuid"
@ -100,26 +97,11 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut
logger.Info("Creating workspace")
hostRunner := h.runnerFactory("", commandLogger, command.Options{}, h.operations)
workspaceRoot, err := h.prepareWorkspace(ctx, hostRunner, job.RepositoryName, job.RepositoryDirectory, job.Commit, job.FetchTags, job.ShallowClone, job.SparseCheckout)
workspace, err := h.prepareWorkspace(ctx, hostRunner, job, commandLogger)
if err != nil {
return errors.Wrap(err, "failed to prepare workspace")
}
defer func() {
if !h.options.KeepWorkspaces {
handle := commandLogger.Log("teardown.fs", nil)
handle.Write([]byte(fmt.Sprintf("Removing %s\n", workspaceRoot)))
if rmErr := os.RemoveAll(workspaceRoot); rmErr != nil {
handle.Write([]byte(fmt.Sprintf("Operation failed: %s\n", rmErr.Error())))
}
// We always finish this with exit code 0 even if it errored, because workspace
// cleanup doesn't fail the execution job. We can deal with it separately.
handle.Finalize(0)
handle.Close()
}
}()
defer workspace.Remove(ctx, h.options.KeepWorkspaces)
vmNameSuffix, err := uuid.NewRandom()
if err != nil {
@ -143,37 +125,7 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut
FirecrackerOptions: h.options.FirecrackerOptions,
ResourceOptions: h.options.ResourceOptions,
}
runner := h.runnerFactory(workspaceRoot, commandLogger, options, h.operations)
// Construct a map from filenames to file content that should be accessible to jobs
// within the workspace. This consists of files supplied within the job record itself,
// as well as file-version of each script step.
workspaceFileContentsByPath := map[string][]byte{}
for relativePath, content := range job.VirtualMachineFiles {
path, err := filepath.Abs(filepath.Join(workspaceRoot, relativePath))
if err != nil {
return err
}
if !strings.HasPrefix(path, workspaceRoot) {
return errors.Errorf("refusing to write outside of working directory")
}
workspaceFileContentsByPath[path] = []byte(content)
}
scriptNames := make([]string, 0, len(job.DockerSteps))
for i, dockerStep := range job.DockerSteps {
scriptName := scriptNameFromJobStep(job, i)
scriptNames = append(scriptNames, scriptName)
path := filepath.Join(workspaceRoot, command.ScriptsPath, scriptName)
workspaceFileContentsByPath[path] = buildScript(dockerStep)
}
if err := writeFiles(workspaceFileContentsByPath, commandLogger); err != nil {
return errors.Wrap(err, "failed to write virtual machine files")
}
runner := h.runnerFactory(workspace.Path(), commandLogger, options, h.operations)
logger.Info("Setting up VM")
@ -195,7 +147,7 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut
dockerStepCommand := command.CommandSpec{
Key: fmt.Sprintf("step.docker.%d", i),
Image: dockerStep.Image,
ScriptPath: scriptNames[i],
ScriptPath: workspace.ScriptFilenames()[i],
Dir: dockerStep.Dir,
Env: dockerStep.Env,
Operation: h.operations.Exec,
@ -228,14 +180,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record workerut
return nil
}
var scriptPreamble = `
set -x
`
func buildScript(dockerStep executor.DockerStep) []byte {
return []byte(strings.Join(append([]string{scriptPreamble, ""}, dockerStep.Commands...), "\n") + "\n")
}
func union(a, b map[string]string) map[string]string {
c := make(map[string]string, len(a)+len(b))
@ -249,44 +193,6 @@ func union(a, b map[string]string) map[string]string {
return c
}
func scriptNameFromJobStep(job executor.Job, i int) string {
return fmt.Sprintf("%d.%d_%s@%s.sh", job.ID, i, strings.ReplaceAll(job.RepositoryName, "/", "_"), job.Commit)
}
// writeFiles writes to the filesystem the content in the given map.
func writeFiles(workspaceFileContentsByPath map[string][]byte, logger command.Logger) (err error) {
// Bail out early if nothing to do, we don't need to spawn an empty log group.
if len(workspaceFileContentsByPath) == 0 {
return nil
}
handle := logger.Log("setup.fs", nil)
defer func() {
if err == nil {
handle.Finalize(0)
} else {
handle.Finalize(1)
}
handle.Close()
}()
for path, content := range workspaceFileContentsByPath {
// Ensure the path exists.
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
return err
}
if err := os.WriteFile(path, content, os.ModePerm); err != nil {
return err
}
handle.Write([]byte(fmt.Sprintf("Wrote %s\n", path)))
}
return nil
}
func createHoneyEvent(_ context.Context, job executor.Job, err error, duration time.Duration) honey.Event {
fields := map[string]any{
"duration_ms": duration.Milliseconds(),

5
enterprise/cmd/executor/internal/worker/handler_test.go
View File

@ -12,15 +12,16 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/janitor"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/worker/workspace"
"github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
"github.com/sourcegraph/sourcegraph/internal/observation"
)
func TestHandle(t *testing.T) {
testDir := "/tmp/codeintel"
makeTempDir = func() (string, error) { return testDir, nil }
workspace.MakeTempDirectory = func(string) (string, error) { return testDir, nil }
t.Cleanup(func() {
makeTempDir = makeTemporaryDirectory
workspace.MakeTempDirectory = workspace.MakeTemporaryDirectory
})
if err := os.MkdirAll(filepath.Join(testDir, command.ScriptsPath), os.ModePerm); err != nil {

235
enterprise/cmd/executor/internal/worker/workspace.go
View File

@ -2,214 +2,49 @@ package worker
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/worker/workspace"
"github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
)
const SchemeExecutorToken = "token-executor"
// These env vars should be set for git commands. We want to make sure it never hangs on interactive input.
var gitStdEnv = []string{"GIT_TERMINAL_PROMPT=0"}
// prepareWorkspace creates and returns a temporary director in which acts the workspace
// prepareWorkspace creates and returns a temporary directory in which acts the workspace
// while processing a single job. It is up to the caller to ensure that this directory is
// removed after the job has finished processing. If a repository name is supplied, then
// that repository will be cloned (through the frontend API) into the workspace.
func (h *handler) prepareWorkspace(ctx context.Context, commandRunner command.Runner, repositoryName, repositoryDirectory, commit string, fetchTags bool, shallowClone bool, sparseCheckout []string) (_ string, err error) {
tempDir, err := makeTempDir()
if err != nil {
return "", err
}
defer func() {
if err != nil {
_ = os.RemoveAll(tempDir)
}
}()
if repositoryName != "" {
repoPath := tempDir
if repositoryDirectory != "" {
repoPath = filepath.Join(tempDir, repositoryDirectory)
if !strings.HasPrefix(repoPath, tempDir) {
return "", errors.Newf("invalid repo path %q not a subdirectory of %q", repoPath, tempDir)
}
if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
return "", errors.Wrap(err, "creating repo directory")
}
}
cloneURL, err := makeRelativeURL(
h.options.ClientOptions.EndpointOptions.URL,
h.options.GitServicePath,
repositoryName,
)
if err != nil {
return "", err
}
authorizationOption := fmt.Sprintf(
"http.extraHeader=Authorization: %s %s",
SchemeExecutorToken,
h.options.ClientOptions.EndpointOptions.Token,
)
fetchCommand := []string{
"git",
"-C", repoPath,
"-c", "protocol.version=2",
"-c", authorizationOption,
"-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal",
"fetch",
"--progress",
"--no-recurse-submodules",
"origin",
commit,
}
appendFetchArg := func(arg string) {
l := len(fetchCommand)
insertPos := l - 2
fetchCommand = append(fetchCommand[:insertPos+1], fetchCommand[insertPos:]...)
fetchCommand[insertPos] = arg
}
if fetchTags {
appendFetchArg("--tags")
}
if shallowClone {
if !fetchTags {
appendFetchArg("--no-tags")
}
appendFetchArg("--depth=1")
}
// For a sparse checkout, we want to add a blob filter so we only fetch the minimum set of files initially.
if len(sparseCheckout) > 0 {
appendFetchArg("--filter=blob:none")
}
gitCommands := []command.CommandSpec{
{Key: "setup.git.init", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "init"}, Operation: h.operations.SetupGitInit},
{Key: "setup.git.add-remote", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "remote", "add", "origin", cloneURL.String()}, Operation: h.operations.SetupAddRemote},
// Disable gc, this can improve performance and should never run for executor clones.
{Key: "setup.git.disable-gc", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "config", "--local", "gc.auto", "0"}, Operation: h.operations.SetupGitDisableGC},
{Key: "setup.git.fetch", Env: gitStdEnv, Command: fetchCommand, Operation: h.operations.SetupGitFetch},
}
if len(sparseCheckout) > 0 {
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.sparse-checkout-config",
Env: gitStdEnv,
Command: []string{"git", "-C", repoPath, "config", "--local", "core.sparseCheckout", "1"},
Operation: h.operations.SetupGitSparseCheckoutConfig,
})
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.sparse-checkout-set",
Env: gitStdEnv,
Command: append([]string{"git", "-C", repoPath, "sparse-checkout", "set", "--no-cone", "--"}, sparseCheckout...),
Operation: h.operations.SetupGitSparseCheckoutSet,
})
}
checkoutCommand := []string{
"git",
"-C", repoPath,
"checkout",
"--progress",
"--force",
commit,
}
// Sparse checkouts need to fetch additional blobs, so we need to add
// auth config here.
if len(sparseCheckout) > 0 {
checkoutCommand = []string{
"git",
"-C", repoPath,
"-c", "protocol.version=2", "-c", authorizationOption, "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal",
"checkout",
"--progress",
"--force",
commit,
}
}
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.checkout",
Env: gitStdEnv,
Command: checkoutCommand,
Operation: h.operations.SetupGitCheckout,
})
// This is for LSIF, it relies on the origin being set to the upstream repo
// for indexing.
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.set-remote",
Env: gitStdEnv,
Command: []string{
"git",
"-C", repoPath,
"remote",
"set-url",
"origin",
repositoryName,
func (h *handler) prepareWorkspace(
ctx context.Context,
commandRunner command.Runner,
job executor.Job,
commandLogger command.Logger,
) (workspace.Workspace, error) {
if h.options.FirecrackerOptions.Enabled {
return workspace.NewFirecrackerWorkspace(
ctx,
job,
h.options.ResourceOptions.DiskSpace,
h.options.KeepWorkspaces,
commandRunner,
commandLogger,
workspace.CloneOptions{
EndpointURL: h.options.ClientOptions.EndpointOptions.URL,
GitServicePath: h.options.GitServicePath,
ExecutorToken: h.options.ClientOptions.EndpointOptions.Token,
},
Operation: h.operations.SetupGitSetRemoteUrl,
})
for _, spec := range gitCommands {
if err := commandRunner.Run(ctx, spec); err != nil {
return "", errors.Wrap(err, fmt.Sprintf("failed %s", spec.Key))
}
}
h.operations,
)
}
// Create the scripts path.
if err := os.MkdirAll(filepath.Join(tempDir, command.ScriptsPath), os.ModePerm); err != nil {
return "", errors.Wrap(err, "creating script path")
}
return tempDir, nil
}
func makeRelativeURL(base string, path ...string) (*url.URL, error) {
baseURL, err := url.Parse(base)
if err != nil {
return nil, err
}
urlx, err := baseURL.ResolveReference(&url.URL{Path: filepath.Join(path...)}), nil
if err != nil {
return nil, err
}
urlx.User = url.User("executor")
return urlx, nil
}
// makeTempDir defaults to makeTemporaryDirectory and can be replaced for testing
// with determinstic workspace/scripts directories.
var makeTempDir = makeTemporaryDirectory
func makeTemporaryDirectory() (string, error) {
// TMPDIR is set in the dev Procfile to avoid requiring developers to explicitly
// allow bind mounts of the host's /tmp. If this directory doesn't exist,
// os.MkdirTemp below will fail.
if tempdir := os.Getenv("TMPDIR"); tempdir != "" {
if err := os.MkdirAll(tempdir, os.ModePerm); err != nil {
return "", err
}
return os.MkdirTemp(tempdir, "")
}
return os.MkdirTemp("", "")
return workspace.NewDockerWorkspace(
ctx,
job,
commandRunner,
commandLogger,
workspace.CloneOptions{
EndpointURL: h.options.ClientOptions.EndpointOptions.URL,
GitServicePath: h.options.GitServicePath,
ExecutorToken: h.options.ClientOptions.EndpointOptions.Token,
},
h.operations,
)
}

184
enterprise/cmd/executor/internal/worker/workspace/clone.go Normal file
View File

@ -0,0 +1,184 @@
package workspace
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
"github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
const SchemeExecutorToken = "token-executor"
// These env vars should be set for git commands. We want to make sure it never hangs on interactive input.
var gitStdEnv = []string{"GIT_TERMINAL_PROMPT=0"}
func cloneRepo(
ctx context.Context,
workspaceDir string,
job executor.Job,
commandRunner command.Runner,
options CloneOptions,
operations *command.Operations,
) error {
repoPath := workspaceDir
if job.RepositoryDirectory != "" {
repoPath = filepath.Join(workspaceDir, job.RepositoryDirectory)
if !strings.HasPrefix(repoPath, workspaceDir) {
return errors.Newf("invalid repo path %q not a subdirectory of %q", repoPath, workspaceDir)
}
if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
return errors.Wrap(err, "creating repo directory")
}
}
cloneURL, err := makeRelativeURL(
options.EndpointURL,
options.GitServicePath,
job.RepositoryName,
)
if err != nil {
return err
}
authorizationOption := fmt.Sprintf(
"http.extraHeader=Authorization: %s %s",
SchemeExecutorToken,
options.ExecutorToken,
)
fetchCommand := []string{
"git",
"-C", repoPath,
"-c", "protocol.version=2",
"-c", authorizationOption,
"-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal",
"fetch",
"--progress",
"--no-recurse-submodules",
"origin",
job.Commit,
}
appendFetchArg := func(arg string) {
l := len(fetchCommand)
insertPos := l - 2
fetchCommand = append(fetchCommand[:insertPos+1], fetchCommand[insertPos:]...)
fetchCommand[insertPos] = arg
}
if job.FetchTags {
appendFetchArg("--tags")
}
if job.ShallowClone {
if !job.FetchTags {
appendFetchArg("--no-tags")
}
appendFetchArg("--depth=1")
}
// For a sparse checkout, we want to add a blob filter so we only fetch the minimum set of files initially.
if len(job.SparseCheckout) > 0 {
appendFetchArg("--filter=blob:none")
}
gitCommands := []command.CommandSpec{
{Key: "setup.git.init", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "init"}, Operation: operations.SetupGitInit},
{Key: "setup.git.add-remote", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "remote", "add", "origin", cloneURL.String()}, Operation: operations.SetupAddRemote},
// Disable gc, this can improve performance and should never run for executor clones.
{Key: "setup.git.disable-gc", Env: gitStdEnv, Command: []string{"git", "-C", repoPath, "config", "--local", "gc.auto", "0"}, Operation: operations.SetupGitDisableGC},
{Key: "setup.git.fetch", Env: gitStdEnv, Command: fetchCommand, Operation: operations.SetupGitFetch},
}
if len(job.SparseCheckout) > 0 {
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.sparse-checkout-config",
Env: gitStdEnv,
Command: []string{"git", "-C", repoPath, "config", "--local", "core.sparseCheckout", "1"},
Operation: operations.SetupGitSparseCheckoutConfig,
})
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.sparse-checkout-set",
Env: gitStdEnv,
Command: append([]string{"git", "-C", repoPath, "sparse-checkout", "set", "--no-cone", "--"}, job.SparseCheckout...),
Operation: operations.SetupGitSparseCheckoutSet,
})
}
checkoutCommand := []string{
"git",
"-C", repoPath,
"checkout",
"--progress",
"--force",
job.Commit,
}
// Sparse checkouts need to fetch additional blobs, so we need to add
// auth config here.
if len(job.SparseCheckout) > 0 {
checkoutCommand = []string{
"git",
"-C", repoPath,
"-c", "protocol.version=2", "-c", authorizationOption, "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal",
"checkout",
"--progress",
"--force",
job.Commit,
}
}
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.checkout",
Env: gitStdEnv,
Command: checkoutCommand,
Operation: operations.SetupGitCheckout,
})
// This is for LSIF, it relies on the origin being set to the upstream repo
// for indexing.
gitCommands = append(gitCommands, command.CommandSpec{
Key: "setup.git.set-remote",
Env: gitStdEnv,
Command: []string{
"git",
"-C", repoPath,
"remote",
"set-url",
"origin",
job.RepositoryName,
},
Operation: operations.SetupGitSetRemoteUrl,
})
for _, spec := range gitCommands {
if err := commandRunner.Run(ctx, spec); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed %s", spec.Key))
}
}
return nil
}
func makeRelativeURL(base string, path ...string) (*url.URL, error) {
baseURL, err := url.Parse(base)
if err != nil {
return nil, err
}
urlx, err := baseURL.ResolveReference(&url.URL{Path: filepath.Join(path...)}), nil
if err != nil {
return nil, err
}
urlx.User = url.User("executor")
return urlx, nil
}

82
enterprise/cmd/executor/internal/worker/workspace/docker.go Normal file
View File

@ -0,0 +1,82 @@
package workspace
import (
"context"
"fmt"
"os"
"strconv"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
"github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
)
// NewDockerWorkspace creates a new workspace for docker-based execution. A path on
// the host will be used to set up the workspace, clone the repo and put script files.
func NewDockerWorkspace(
ctx context.Context,
job executor.Job,
commandRunner command.Runner,
logger command.Logger,
cloneOpts CloneOptions,
operations *command.Operations,
) (Workspace, error) {
workspaceDir, err := MakeTempDirectory("workspace-" + strconv.Itoa(job.ID))
if err != nil {
return nil, err
}
if job.RepositoryName != "" {
if err := cloneRepo(ctx, workspaceDir, job, commandRunner, cloneOpts, operations); err != nil {
_ = os.RemoveAll(workspaceDir)
return nil, err
}
}
scriptPaths, err := prepareScripts(ctx, job, workspaceDir, commandRunner, logger)
if err != nil {
_ = os.RemoveAll(workspaceDir)
return nil, err
}
return &dockerWorkspace{
path: workspaceDir,
scriptFilenames: scriptPaths,
workspaceDir: workspaceDir,
logger: logger,
}, nil
}
type dockerWorkspace struct {
path string
scriptFilenames []string
workspaceDir string
logger command.Logger
}
func (w dockerWorkspace) Path() string {
return w.path
}
func (w dockerWorkspace) ScriptFilenames() []string {
return w.scriptFilenames
}
func (w dockerWorkspace) Remove(ctx context.Context, keepWorkspace bool) {
handle := w.logger.Log("teardown.fs", nil)
defer func() {
// We always finish this with exit code 0 even if it errored, because workspace
// cleanup doesn't fail the execution job. We can deal with it separately.
handle.Finalize(0)
handle.Close()
}()
if keepWorkspace {
fmt.Fprintf(handle, "Preserving workspace (%s) as per config", w.workspaceDir)
return
}
fmt.Fprintf(handle, "Removing %s\n", w.workspaceDir)
if rmErr := os.RemoveAll(w.workspaceDir); rmErr != nil {
fmt.Fprintf(handle, "Operation failed: %s\n", rmErr.Error())
}
}

104
enterprise/cmd/executor/internal/worker/workspace/files.go Normal file
View File

@ -0,0 +1,104 @@
package workspace
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
"github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func prepareScripts(
ctx context.Context,
job executor.Job,
workspaceDir string,
commandRunner command.Runner,
commandLogger command.Logger,
) ([]string, error) {
// Create the scripts path.
if err := os.MkdirAll(filepath.Join(workspaceDir, command.ScriptsPath), os.ModePerm); err != nil {
return nil, errors.Wrap(err, "creating script path")
}
// Construct a map from filenames to file content that should be accessible to jobs
// within the workspace. This consists of files supplied within the job record itself,
// as well as file-version of each script step.
workspaceFileContentsByPath := map[string][]byte{}
for relativePath, content := range job.VirtualMachineFiles {
path, err := filepath.Abs(filepath.Join(workspaceDir, relativePath))
if err != nil {
return nil, err
}
if !strings.HasPrefix(path, workspaceDir) {
return nil, errors.Errorf("refusing to write outside of working directory")
}
workspaceFileContentsByPath[path] = []byte(content)
}
scriptNames := make([]string, 0, len(job.DockerSteps))
for i, dockerStep := range job.DockerSteps {
scriptName := scriptNameFromJobStep(job, i)
scriptNames = append(scriptNames, scriptName)
path := filepath.Join(workspaceDir, command.ScriptsPath, scriptName)
workspaceFileContentsByPath[path] = buildScript(dockerStep)
}
if err := writeFiles(workspaceFileContentsByPath, commandLogger); err != nil {
return nil, errors.Wrap(err, "failed to write virtual machine files")
}
return scriptNames, nil
}
var scriptPreamble = `
set -x
`
func buildScript(dockerStep executor.DockerStep) []byte {
return []byte(strings.Join(append([]string{scriptPreamble, ""}, dockerStep.Commands...), "\n") + "\n")
}
func scriptNameFromJobStep(job executor.Job, i int) string {
return fmt.Sprintf("%d.%d_%s@%s.sh", job.ID, i, strings.ReplaceAll(job.RepositoryName, "/", "_"), job.Commit)
}
// writeFiles writes the content of the given map to the filesystem.
func writeFiles(workspaceFileContentsByPath map[string][]byte, logger command.Logger) (err error) {
// Bail out early if nothing to do, we don't want to spawn an empty log group.
if len(workspaceFileContentsByPath) == 0 {
return nil
}
handle := logger.Log("setup.fs.extras", nil)
defer func() {
if err == nil {
handle.Finalize(0)
} else {
handle.Finalize(1)
}
handle.Close()
}()
for path, content := range workspaceFileContentsByPath {
// Ensure the path exists.
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
return err
}
if err := os.WriteFile(path, content, os.ModePerm); err != nil {
return err
}
fmt.Fprintf(handle, "Wrote %s\n", path)
}
return nil
}

234
enterprise/cmd/executor/internal/worker/workspace/firecracker.go Normal file
View File

@ -0,0 +1,234 @@
package workspace
import (
"context"
"fmt"
"os"
"os/exec"
"strconv"
"strings"
"syscall"
"github.com/c2h5oh/datasize"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
"github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
// NewFirecrackerWorkspace creates a new workspace for firecracker-based execution.
// A block device will be created on the host disk, with an ext4 file system. It
// is exposed through a loopback device. To set up the workspace, this device will
// be mounted and clone the repo and put script files in it. Then, the executor
// VM can mount this loopback device. This prevents host file system access.
func NewFirecrackerWorkspace(
ctx context.Context,
job executor.Job,
diskSpace string,
keepWorkspace bool,
commandRunner command.Runner,
logger command.Logger,
cloneOpts CloneOptions,
operations *command.Operations,
) (Workspace, error) {
blockDeviceFile, tmpMountDir, blockDevice, err := setupLoopDevice(
ctx,
job.ID,
diskSpace,
keepWorkspace,
logger,
)
if err != nil {
return nil, err
}
// Unmount the workspace volume when done, we finished writing to it from the host.
defer func() {
if err2 := syscall.Unmount(tmpMountDir, 0); err2 != nil {
err = errors.Append(err, err2)
return
}
if err2 := os.RemoveAll(tmpMountDir); err2 != nil {
err = errors.Append(err, err2)
}
}()
if job.RepositoryName != "" {
if err := cloneRepo(ctx, tmpMountDir, job, commandRunner, cloneOpts, operations); err != nil {
return nil, err
}
}
scriptPaths, err := prepareScripts(ctx, job, tmpMountDir, commandRunner, logger)
if err != nil {
return nil, err
}
return &firecrackerWorkspace{
scriptFilenames: scriptPaths,
blockDeviceFile: blockDeviceFile,
blockDevice: blockDevice,
logger: logger,
}, err
}
type firecrackerWorkspace struct {
scriptFilenames []string
blockDeviceFile string
blockDevice string
logger command.Logger
}
func (w firecrackerWorkspace) Path() string {
return w.blockDevice
}
func (w firecrackerWorkspace) ScriptFilenames() []string {
return w.scriptFilenames
}
func (w firecrackerWorkspace) Remove(ctx context.Context, keepWorkspace bool) {
handle := w.logger.Log("teardown.fs", nil)
defer func() {
// We always finish this with exit code 0 even if it errored, because workspace
// cleanup doesn't fail the execution job. We can deal with it separately.
handle.Finalize(0)
handle.Close()
}()
if keepWorkspace {
fmt.Fprintf(handle, "Preserving workspace files (block device: %s, loop file: %s) as per config", w.blockDevice, w.blockDeviceFile)
// Remount the workspace, so that it can be inspected.
mountDir, err := mountLoopDevice(ctx, w.blockDevice, handle)
if err != nil {
fmt.Fprintf(handle, "Failed to mount workspace device %q, mount manually to inspect the contents: %s\n", w.blockDevice, err)
return
}
fmt.Fprintf(handle, "Inspect the workspace contents at: %s\n", mountDir)
return
}
fmt.Fprintf(handle, "Removing loop device %s\n", w.blockDevice)
if err := detachLoopDevice(ctx, w.blockDevice, handle); err != nil {
fmt.Fprintf(handle, "stderr: Failed to detach loop device: %s\n", err)
}
fmt.Fprintf(handle, "Removing block device file %s\n", w.blockDeviceFile)
if err := os.Remove(w.blockDeviceFile); err != nil {
fmt.Fprintf(handle, "stderr: Failed to remove block device: %s\n", err)
}
}
// setupLoopDevice is used in firecracker mode. It creates a block device on disk,
// creates a loop device pointing to it, and mounts it so that it can be written to.
// The loop device will be given to ignite and mounted into the guest VM.
func setupLoopDevice(
ctx context.Context,
jobID int,
diskSpace string,
keepWorkspace bool,
logger command.Logger,
) (blockDeviceFile, tmpMountDir, blockDevice string, err error) {
handle := logger.Log("setup.fs.workspace", nil)
defer func() {
// add the error to the bottom of the step's log output,
// but only if this isnt from exec.Command, as those get added
// by our logging wrapper
if !errors.HasType(err, &exec.ExitError{}) {
fmt.Fprint(handle, err.Error())
}
if err != nil {
handle.Finalize(1)
} else {
handle.Finalize(0)
}
handle.Close()
}()
// Create a temp file to hold the block device on disk.
loopFile, err := MakeTempFile("workspace-loop-" + strconv.Itoa(jobID))
if err != nil {
return "", "", "", err
}
defer func() {
if err != nil && !keepWorkspace {
os.Remove(loopFile.Name())
}
}()
blockDeviceFile = loopFile.Name()
fmt.Fprintf(handle, "Created backing workspace file at %q\n", blockDeviceFile)
// Truncate the file to be of the size of the maximum permissible disk space.
diskSize, err := datasize.ParseString(diskSpace)
if err != nil {
return "", "", "", errors.Wrapf(err, "invalid disk size provided: %q", diskSpace)
}
if err := loopFile.Truncate(int64(diskSize.Bytes())); err != nil {
return "", "", "", errors.Wrapf(err, "failed to make backing file sparse with %d bytes", diskSize.Bytes())
}
fmt.Fprintf(handle, "Created sparse file of size %s from %q\n", diskSize.HumanReadable(), blockDeviceFile)
if err := loopFile.Close(); err != nil {
return "", "", "", errors.Wrap(err, "failed to close backing file")
}
// Create an ext4 file system in the device backing file.
out, err := commandLogger(ctx, handle, "mkfs.ext4", blockDeviceFile)
if err != nil {
return "", "", "", errors.Wrapf(err, "failed to create ext4 filesystem in backing file: %q", out)
}
fmt.Fprintf(handle, "Wrote ext4 filesystem to backing file %q\n", blockDeviceFile)
// Create a loop device pointing to our block device.
out, err = commandLogger(ctx, handle, "losetup", "--find", "--show", blockDeviceFile)
if err != nil {
return "", "", "", errors.Wrapf(err, "failed to create loop device: %q", out)
}
blockDevice = strings.TrimSpace(out)
defer func() {
// If something further down in this function failed we detach the loop device
// to not hoard them.
if err != nil {
err := detachLoopDevice(ctx, blockDevice, handle)
if err != nil {
fmt.Fprint(handle, "stderr: "+strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", "\nstderr: "))
}
}
}()
fmt.Fprintf(handle, "Created loop device at %q backed by %q\n", blockDevice, blockDeviceFile)
// Mount the loop device at a temporary directory so we can write the workspace contents to it.
tmpMountDir, err = mountLoopDevice(ctx, blockDevice, handle)
if err != nil {
// important to set at least blockDevice for the above defer
return blockDeviceFile, "", blockDevice, err
}
fmt.Fprintf(handle, "Created temporary workspace mount location at %q\n", tmpMountDir)
return blockDeviceFile, tmpMountDir, blockDevice, nil
}
// detachLoopDevice detaches a loop device by path (/dev/loopX).
func detachLoopDevice(ctx context.Context, blockDevice string, handle command.LogEntry) error {
out, err := commandLogger(ctx, handle, "losetup", "--detach", blockDevice)
if err != nil {
return errors.Wrapf(err, "failed to detach loop device: %s", out)
}
return nil
}
// mountLoopDevice takes a path to a loop device (/dev/loopX) and mounts it at a
// random temporary mount point. The mount point is returned.
func mountLoopDevice(ctx context.Context, blockDevice string, handle command.LogEntry) (string, error) {
tmpMountDir, err := MakeTempDirectory("workspace-mountpoints")
if err != nil {
return "", err
}
if out, err := commandLogger(ctx, handle, "mount", blockDevice, tmpMountDir); err != nil {
_ = os.RemoveAll(tmpMountDir)
return "", errors.Wrapf(err, "failed to mount loop device %q to %q: %q", blockDevice, tmpMountDir, out)
}
return tmpMountDir, nil
}

21
enterprise/cmd/executor/internal/worker/workspace/iface.go Normal file
View File

@ -0,0 +1,21 @@
package workspace
import "context"
type CloneOptions struct {
EndpointURL string
GitServicePath string
ExecutorToken string
}
type Workspace interface {
// Path represents the block device path when firecracker is enabled and the
// directory when firecracker is disabled where the workspace is configured.
Path() string
// ScriptFilenames holds the ordered set of script filenames to be invoked.
ScriptFilenames() []string
// Remove cleans up the workspace post execution. If keep workspace is true,
// the implementation will only clean up additional resources, while keeping
// the workspace contents on disk for debugging purposes.
Remove(ctx context.Context, keepWorkspace bool)
}

55
enterprise/cmd/executor/internal/worker/workspace/util.go Normal file
View File

@ -0,0 +1,55 @@
package workspace
import (
"context"
"fmt"
"os"
"os/exec"
"strings"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
)
// MakeTempFile defaults to makeTemporaryFile and can be replaced for testing
// with determinstic workspace/scripts directories.
var MakeTempFile = makeTemporaryFile
func makeTemporaryFile(prefix string) (*os.File, error) {
if tempdir := os.Getenv("TMPDIR"); tempdir != "" {
if err := os.MkdirAll(tempdir, os.ModePerm); err != nil {
return nil, err
}
return os.CreateTemp(tempdir, prefix+"-*")
}
return os.CreateTemp("", prefix+"-*")
}
// MakeTempDirectory defaults to makeTemporaryDirectory and can be replaced for testing
// with determinstic workspace/scripts directories.
var MakeTempDirectory = MakeTemporaryDirectory
func MakeTemporaryDirectory(prefix string) (string, error) {
if tempdir := os.Getenv("TMPDIR"); tempdir != "" {
if err := os.MkdirAll(tempdir, os.ModePerm); err != nil {
return "", err
}
return os.MkdirTemp(tempdir, prefix+"-*")
}
return os.MkdirTemp("", prefix+"-*")
}
// runs the given command with args and logs the invocation and output to the provided log entry handle.
func commandLogger(ctx context.Context, handle command.LogEntry, command string, args ...string) (string, error) {
fmt.Fprintf(handle, "$ %s %s\n", command, strings.Join(args, " "))
cmd := exec.CommandContext(ctx, command, args...)
out, err := cmd.CombinedOutput()
if len(out) == 0 {
fmt.Fprint(handle, "stderr: <no output>\n")
} else {
fmt.Fprintf(handle, "stderr: %s\n", strings.ReplaceAll(strings.TrimSpace(string(out)), "\n", "\nstderr: "))
}
return string(out), err
}

80
enterprise/cmd/executor/internal/worker/workspace_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/apiclient"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/executor/internal/command"
"github.com/sourcegraph/sourcegraph/enterprise/internal/executor"
"github.com/sourcegraph/sourcegraph/internal/observation"
)
@ -29,11 +30,15 @@ func TestPrepareWorkspace_Clone(t *testing.T) {
operations: command.NewOperations(&observation.TestContext),
}
dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "", "deadbeef", true, false, []string{})
workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{
RepositoryName: "torvalds/linux",
Commit: "deadbeef",
FetchTags: true,
}, nil)
if err != nil {
t.Fatalf("unexpected error preparing workspace: %s", err)
}
defer os.RemoveAll(dir)
defer os.RemoveAll(workspace.Path())
if value := len(runner.RunFunc.History()); value != 6 {
t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 6, value)
@ -45,12 +50,12 @@ func TestPrepareWorkspace_Clone(t *testing.T) {
}
expectedCommands := [][]string{
{"git", "-C", dir, "init"},
{"git", "-C", dir, "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"},
{"git", "-C", dir, "config", "--local", "gc.auto", "0"},
{"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--tags", "origin", "deadbeef"},
{"git", "-C", dir, "checkout", "--progress", "--force", "deadbeef"},
{"git", "-C", dir, "remote", "set-url", "origin", "torvalds/linux"},
{"git", "-C", workspace.Path(), "init"},
{"git", "-C", workspace.Path(), "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"},
{"git", "-C", workspace.Path(), "config", "--local", "gc.auto", "0"},
{"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--tags", "origin", "deadbeef"},
{"git", "-C", workspace.Path(), "checkout", "--progress", "--force", "deadbeef"},
{"git", "-C", workspace.Path(), "remote", "set-url", "origin", "torvalds/linux"},
}
if diff := cmp.Diff(expectedCommands, commands); diff != "" {
t.Errorf("unexpected commands (-want +got):\n%s", diff)
@ -73,13 +78,17 @@ func TestPrepareWorkspace_Clone_Subdirectory(t *testing.T) {
operations: command.NewOperations(&observation.TestContext),
}
dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "subdirectory", "deadbeef", false, false, []string{})
workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{
RepositoryName: "torvalds/linux",
RepositoryDirectory: "subdirectory",
Commit: "deadbeef",
}, nil)
if err != nil {
t.Fatalf("unexpected error preparing workspace: %s", err)
}
defer os.RemoveAll(dir)
defer os.RemoveAll(workspace.Path())
repoDir := filepath.Join(dir, "subdirectory")
repoDir := filepath.Join(workspace.Path(), "subdirectory")
if value := len(runner.RunFunc.History()); value != 6 {
t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 6, value)
@ -119,11 +128,15 @@ func TestPrepareWorkspace_ShallowClone(t *testing.T) {
operations: command.NewOperations(&observation.TestContext),
}
dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "", "deadbeef", false, true, []string{})
workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{
RepositoryName: "torvalds/linux",
Commit: "deadbeef",
ShallowClone: true,
}, nil)
if err != nil {
t.Fatalf("unexpected error preparing workspace: %s", err)
}
defer os.RemoveAll(dir)
defer os.RemoveAll(workspace.Path())
if value := len(runner.RunFunc.History()); value != 6 {
t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 6, value)
@ -135,12 +148,12 @@ func TestPrepareWorkspace_ShallowClone(t *testing.T) {
}
expectedCommands := [][]string{
{"git", "-C", dir, "init"},
{"git", "-C", dir, "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"},
{"git", "-C", dir, "config", "--local", "gc.auto", "0"},
{"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "origin", "deadbeef"},
{"git", "-C", dir, "checkout", "--progress", "--force", "deadbeef"},
{"git", "-C", dir, "remote", "set-url", "origin", "torvalds/linux"},
{"git", "-C", workspace.Path(), "init"},
{"git", "-C", workspace.Path(), "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"},
{"git", "-C", workspace.Path(), "config", "--local", "gc.auto", "0"},
{"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "origin", "deadbeef"},
{"git", "-C", workspace.Path(), "checkout", "--progress", "--force", "deadbeef"},
{"git", "-C", workspace.Path(), "remote", "set-url", "origin", "torvalds/linux"},
}
if diff := cmp.Diff(expectedCommands, commands); diff != "" {
t.Errorf("unexpected commands (-want +got):\n%s", diff)
@ -163,11 +176,16 @@ func TestPrepareWorkspace_SparseCheckout(t *testing.T) {
operations: command.NewOperations(&observation.TestContext),
}
dir, err := handler.prepareWorkspace(context.Background(), runner, "torvalds/linux", "", "deadbeef", false, true, []string{"kernel"})
workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{
RepositoryName: "torvalds/linux",
Commit: "deadbeef",
ShallowClone: true,
SparseCheckout: []string{"kernel"},
}, nil)
if err != nil {
t.Fatalf("unexpected error preparing workspace: %s", err)
}
defer os.RemoveAll(dir)
defer os.RemoveAll(workspace.Path())
if value := len(runner.RunFunc.History()); value != 8 {
t.Fatalf("unexpected number of calls to Run. want=%d have=%d", 8, value)
@ -179,14 +197,14 @@ func TestPrepareWorkspace_SparseCheckout(t *testing.T) {
}
expectedCommands := [][]string{
{"git", "-C", dir, "init"},
{"git", "-C", dir, "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"},
{"git", "-C", dir, "config", "--local", "gc.auto", "0"},
{"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "--filter=blob:none", "origin", "deadbeef"},
{"git", "-C", dir, "config", "--local", "core.sparseCheckout", "1"},
{"git", "-C", dir, "sparse-checkout", "set", "--no-cone", "--", "kernel"},
{"git", "-C", dir, "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "checkout", "--progress", "--force", "deadbeef"},
{"git", "-C", dir, "remote", "set-url", "origin", "torvalds/linux"},
{"git", "-C", workspace.Path(), "init"},
{"git", "-C", workspace.Path(), "remote", "add", "origin", "https://executor@test.io/internal/git/torvalds/linux"},
{"git", "-C", workspace.Path(), "config", "--local", "gc.auto", "0"},
{"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "fetch", "--progress", "--no-recurse-submodules", "--no-tags", "--depth=1", "--filter=blob:none", "origin", "deadbeef"},
{"git", "-C", workspace.Path(), "config", "--local", "core.sparseCheckout", "1"},
{"git", "-C", workspace.Path(), "sparse-checkout", "set", "--no-cone", "--", "kernel"},
{"git", "-C", workspace.Path(), "-c", "protocol.version=2", "-c", "http.extraHeader=Authorization: token-executor hunter2", "-c", "http.extraHeader=X-Sourcegraph-Actor-UID: internal", "checkout", "--progress", "--force", "deadbeef"},
{"git", "-C", workspace.Path(), "remote", "set-url", "origin", "torvalds/linux"},
}
if diff := cmp.Diff(expectedCommands, commands); diff != "" {
t.Errorf("unexpected commands (-want +got):\n%s", diff)
@ -201,11 +219,11 @@ func TestPrepareWorkspace_NoRepository(t *testing.T) {
operations: command.NewOperations(&observation.TestContext),
}
dir, err := handler.prepareWorkspace(context.Background(), runner, "", "", "", false, false, []string{})
workspace, err := handler.prepareWorkspace(context.Background(), runner, executor.Job{}, nil)
if err != nil {
t.Fatalf("unexpected error preparing workspace: %s", err)
}
defer os.RemoveAll(dir)
defer os.RemoveAll(workspace.Path())
if value := len(runner.RunFunc.History()); value != 0 {
t.Fatalf("unexpected call to Run")

1
go.mod
View File

@ -215,6 +215,7 @@ require (
require github.com/hmarr/codeowners v0.4.0
require (
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
github.com/hashicorp/hcl v1.0.0
github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.13
github.com/prometheus/prometheus v0.37.1

4
go.sum
View File

@ -386,6 +386,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
github.com/buildkite/go-buildkite/v3 v3.0.1 h1:5kX1fFDj3Co7cP6cqZKuW1VoCJz3u4cOx6wfdCeM4ZA=
github.com/buildkite/go-buildkite/v3 v3.0.1/go.mod h1:6pweknacVv7He5Lvbf54urp2P0W6/b4Nrcxn718PQrE=
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY=
github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw=
github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo=
github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oDpT4efm8tSYHXV5tHSdRvBet/b/QzxZ+XyyPehvm3A=
@ -1249,8 +1251,6 @@ github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7
github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
github.com/goware/urlx v0.3.1 h1:BbvKl8oiXtJAzOzMqAQ0GfIhf96fKeNEZfm9ocNSUBI=
github.com/goware/urlx v0.3.1/go.mod h1:h8uwbJy68o+tQXCGZNa9D73WN8n0r9OBae5bUnLcgjw=
github.com/grafana-tools/sdk v0.0.0-20220203092117-edae16afa87b h1:R9LID2XreyUOQfJ/NKLGuYOF4/Wz6ljmYFAhlOaHVQ4=
github.com/grafana-tools/sdk v0.0.0-20220203092117-edae16afa87b/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4=
github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc h1:PXZQA2WCxe85Tnn+WEvr8fDpfwibmEPgfgFEaC87G24=
github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4=
github.com/grafana/regexp v0.0.0-20220304100321-149c8afcd6cb h1:wwzNkyaQwcXCzQuKoWz3lwngetmcyg+EhW0fF5lz73M=

13
sg.config.yaml
View File

@ -485,6 +485,19 @@ commands:
EXECUTOR_QUEUE_NAME: codeintel
SRC_PROF_HTTP: ':6092'
# If you want to use this, either start it with `sg run batches-executor-firecracker` or
# modify the `commandsets.batches` in your local `sg.config.overwrite.yaml`
codeintel-executor-firecracker:
<<: *executor_template
cmd: |
env TMPDIR="$HOME/.sourcegraph/codeintel-executor-temp" \
sudo --preserve-env=TMPDIR,EXECUTOR_QUEUE_NAME,SRC_PROF_HTTP,EXECUTOR_FRONTEND_URL,EXECUTOR_FRONTEND_PASSWORD,EXECUTOR_USE_FIRECRACKER,EXECUTOR_IMAGE_ARCHIVE_PATH \
.bin/executor
env:
EXECUTOR_USE_FIRECRACKER: true
EXECUTOR_QUEUE_NAME: codeintel
SRC_PROF_HTTP: ':6093'
batches-executor:
<<: *executor_template
cmd: |