artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 20:51:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move ArchiveReader's git exec command to gitcli (#59933)

Browse Source 
This PR migrates the ArchiveReader function's use of the exec endpoint to be in line with the rest of #59738

Additionally, sub-repo permission checks are now done on the server instead of in the client code.

---------

Co-authored-by: Erik Seliger <erikseliger@me.com>
This commit is contained in:
Petri-Johan Last 2024-02-13 13:40:45 +02:00 committed by GitHub
parent cc0f1ab67f
commit 458ce56cf3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
31 changed files with 3474 additions and 1328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/frontend/internal/app/ui/raw.go
View File

@ -19,11 +19,11 @@ import (
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/cmd/frontend/globals"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/errcode"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
)
// Examples:
@ -195,7 +195,7 @@ func serveRaw(logger log.Logger, db database.DB, gitserverClient gitserver.Clien
// internet, so we use default compression levels on zips (instead of no
// compression).
f, err := gitserverClient.ArchiveReader(r.Context(), common.Repo.Name,
gitserver.ArchiveOptions{Format: format, Treeish: string(common.CommitID), Pathspecs: []gitdomain.Pathspec{gitdomain.PathspecLiteral(relativePath)}})
gitserver.ArchiveOptions{Format: format, Treeish: string(common.CommitID), Paths: []string{relativePath}})
if err != nil {
return err
}

34
cmd/frontend/internal/app/ui/raw_test.go
View File

@ -2,7 +2,6 @@ package ui
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
@ -13,9 +12,8 @@ import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/sourcegraph/log/logtest"
"github.com/stretchr/testify/assert"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
@ -53,16 +51,6 @@ func initHTTPTestGitServer(t *testing.T, httpStatusCode int, resp string) {
s.Close()
gitserver.ResetClientMocks()
})
gitserver.ClientMocks.Archive = func(ctx context.Context, repo api.RepoName, opt gitserver.ArchiveOptions) (reader io.ReadCloser, err error) {
if httpStatusCode != http.StatusOK {
err = errors.New("error")
} else {
stringReader := strings.NewReader(resp)
reader = io.NopCloser(stringReader)
}
return reader, err
}
}
func Test_serveRawWithHTTPRequestMethodHEAD(t *testing.T) {
@ -154,7 +142,10 @@ func Test_serveRawWithContentArchive(t *testing.T) {
w := httptest.NewRecorder()
db := dbmocks.NewMockDB()
err := serveRaw(logger, db, gitserver.NewTestClient(t))(w, req)
client := gitserver.NewMockClient()
client.ArchiveReaderFunc.SetDefaultReturn(io.NopCloser(strings.NewReader(mockGitServerResponse)), nil)
err := serveRaw(logger, db, client)(w, req)
if err != nil {
t.Fatalf("Failed to invoke serveRaw: %v", err)
}
@ -179,7 +170,7 @@ func Test_serveRawWithContentArchive(t *testing.T) {
}
}
body := string(w.Body.Bytes())
body := w.Body.String()
if body != mockGitServerResponse {
t.Errorf("Want %q in body, but got %q", mockGitServerResponse, body)
}
@ -194,7 +185,9 @@ func Test_serveRawWithContentArchive(t *testing.T) {
w := httptest.NewRecorder()
db := dbmocks.NewMockDB()
err := serveRaw(logger, db, gitserver.NewTestClient(t))(w, req)
client := gitserver.NewMockClient()
client.ArchiveReaderFunc.SetDefaultReturn(io.NopCloser(strings.NewReader(mockGitServerResponse)), nil)
err := serveRaw(logger, db, client)(w, req)
if err != nil {
t.Fatalf("Failed to invoke serveRaw: %v", err)
}
@ -219,12 +212,11 @@ func Test_serveRawWithContentArchive(t *testing.T) {
}
}
body := string(w.Body.Bytes())
body := w.Body.String()
if body != mockGitServerResponse {
t.Errorf("Want %q in body, but got %q", mockGitServerResponse, body)
}
})
}
func Test_serveRawWithContentTypePlain(t *testing.T) {
@ -316,7 +308,7 @@ func Test_serveRawWithContentTypePlain(t *testing.T) {
want := `a/
b/
c.go`
body := string(w.Body.Bytes())
body := w.Body.String()
if body != want {
t.Errorf("Want %q in body, but got %q", want, body)
}
@ -348,7 +340,7 @@ c.go`
want := "this is a test file"
body := string(w.Body.Bytes())
body := w.Body.String()
if body != want {
t.Errorf("Want %q in body, but got %q", want, body)
}
@ -381,7 +373,7 @@ c.go`
want := "this is a test file"
body := string(w.Body.Bytes())
body := w.Body.String()
if body != want {
t.Errorf("Want %q in body, but got %q", want, body)
}

3
cmd/gitserver/internal/git/gitcli/BUILD.bazel
View File

@ -4,6 +4,7 @@ load("//dev:go_defs.bzl", "go_test")
go_library(
name = "gitcli",
srcs = [
"archivereader.go",
"blame.go",
"clibackend.go",
"config.go",
@ -22,6 +23,7 @@ go_library(
"//cmd/gitserver/internal/executil",
"//cmd/gitserver/internal/git",
"//internal/api",
"//internal/collections",
"//internal/gitserver/gitdomain",
"//internal/lazyregexp",
"//internal/trace",
@ -38,6 +40,7 @@ go_library(
go_test(
name = "gitcli_test",
srcs = [
"archivereader_test.go",
"blame_test.go",
"config_test.go",
"exec_test.go",

116
cmd/gitserver/internal/git/gitcli/archivereader.go Normal file
View File

@ -0,0 +1,116 @@
package gitcli
import (
"bytes"
"context"
"io"
"os"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git"
"github.com/sourcegraph/sourcegraph/internal/collections"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func (g *gitCLIBackend) ArchiveReader(ctx context.Context, format git.ArchiveFormat, treeish string, paths []string) (io.ReadCloser, error) {
if err := g.verifyPaths(ctx, treeish, paths); err != nil {
return nil, err
}
archiveArgs := buildArchiveArgs(format, treeish, paths)
cmd, cancel, err := g.gitCommand(ctx, archiveArgs...)
if err != nil {
cancel()
return nil, err
}
r, err := g.runGitCommand(ctx, cmd)
if err != nil {
cancel()
return nil, err
}
return &closingFileReader{
ReadCloser: r,
onClose: func() { cancel() },
}, nil
}
func buildArchiveArgs(format git.ArchiveFormat, treeish string, paths []string) []string {
args := []string{"archive", "--worktree-attributes", "--format=" + string(format)}
if format == git.ArchiveFormatZip {
args = append(args, "-0")
}
args = append(args, treeish, "--")
for _, p := range paths {
args = append(args, pathspecLiteral(p))
}
return args
}
// pathspecLiteral constructs a pathspec that matches a path without interpreting "*" or "?" as special
// characters.
//
// See: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-literal
func pathspecLiteral(s string) string { return ":(literal)" + s }
func (g *gitCLIBackend) verifyPaths(ctx context.Context, treeish string, paths []string) error {
args := []string{"ls-tree", treeish, "--"}
args = append(args, paths...)
cmd, cancel, err := g.gitCommand(ctx, args...)
defer cancel()
if err != nil {
return err
}
out, err := g.runGitCommand(ctx, cmd)
if err != nil {
return err
}
defer out.Close()
stdout, err := io.ReadAll(out)
if err != nil {
// If exit code is 128 and `not a tree object` is part of stderr, most likely we
// are referencing a commit that does not exist.
// We want to return a gitdomain.RevisionNotFoundError in that case.
var e *CommandFailedError
if errors.As(err, &e) && e.ExitStatus == 128 && (bytes.Contains(e.Stderr, []byte("not a tree object")) || bytes.Contains(e.Stderr, []byte("Not a valid object name"))) {
return &gitdomain.RevisionNotFoundError{Repo: g.repoName, Spec: treeish}
}
return err
}
if len(paths) == 0 {
return nil
}
// Check if the resulting objects match the requested
// paths. If not, one or more of the requested
// file paths don't exist.
gotPaths := bytes.Split(bytes.TrimSpace(stdout), []byte("\n"))
fileSet := collections.NewSet[string]()
for _, p := range gotPaths {
if len(p) == 0 {
continue
}
pathSegments := bytes.Fields(p)
fileSet.Add(string(pathSegments[len(pathSegments)-1]))
}
pathsSet := collections.NewSet[string]()
for _, path := range paths {
pathsSet.Add(path)
}
diff := pathsSet.Difference(fileSet)
if len(diff) != 0 {
return &os.PathError{Op: "open", Path: diff.Values()[0], Err: os.ErrNotExist}
}
return nil
}

167
cmd/gitserver/internal/git/gitcli/archivereader_test.go Normal file
View File

@ -0,0 +1,167 @@
package gitcli
import (
"archive/tar"
"archive/zip"
"bytes"
"context"
"io"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func readFileContentsFromTar(t *testing.T, tr *tar.Reader, name string) string {
for {
h, err := tr.Next()
if err == io.EOF {
break
}
require.NoError(t, err)
if h.Name == name {
contents, err := io.ReadAll(tr)
require.NoError(t, err)
return string(contents)
}
}
t.Fatalf("File %q not found in tar archive", name)
return ""
}
func readFileContentsFromZip(t *testing.T, zr *zip.Reader, name string) string {
f, err := zr.Open(name)
if err != nil {
t.Fatalf("File %q not found in zip archive", name)
}
contents, err := io.ReadAll(f)
require.NoError(t, err)
return string(contents)
}
func TestBuildArchiveArgs(t *testing.T) {
t.Run("no paths", func(t *testing.T) {
args := buildArchiveArgs(git.ArchiveFormatTar, "HEAD", nil)
require.Equal(t, []string{"archive", "--worktree-attributes", "--format=tar", "HEAD", "--"}, args)
})
t.Run("with paths", func(t *testing.T) {
args := buildArchiveArgs(git.ArchiveFormatTar, "HEAD", []string{"file1", "file2"})
require.Equal(t, []string{"archive", "--worktree-attributes", "--format=tar", "HEAD", "--", ":(literal)file1", ":(literal)file2"}, args)
})
t.Run("zip adds -0", func(t *testing.T) {
args := buildArchiveArgs(git.ArchiveFormatZip, "HEAD", nil)
require.Equal(t, []string{"archive", "--worktree-attributes", "--format=zip", "-0", "HEAD", "--"}, args)
})
}
func TestGitCLIBackend_ArchiveReader(t *testing.T) {
ctx := context.Background()
backend := BackendWithRepoCommands(t,
"echo abcd > file1",
"mkdir dir1",
"echo efgh > dir1/file2",
"git add file1",
"git add dir1",
"git commit -m commit --author='Foo Author <foo@sourcegraph.com>'",
)
commitID, err := backend.RevParseHead(ctx)
require.NoError(t, err)
t.Run("read simple tar archive", func(t *testing.T) {
r, err := backend.ArchiveReader(ctx, "tar", string(commitID), nil)
require.NoError(t, err)
t.Cleanup(func() { r.Close() })
tr := tar.NewReader(r)
contents := readFileContentsFromTar(t, tr, "file1")
require.Equal(t, "abcd\n", contents)
})
t.Run("read simple zip archive", func(t *testing.T) {
r, err := backend.ArchiveReader(ctx, "zip", string(commitID), nil)
require.NoError(t, err)
t.Cleanup(func() { r.Close() })
contents, err := io.ReadAll(r)
require.NoError(t, err)
zr, err := zip.NewReader(bytes.NewReader([]byte(contents)), int64(len(contents)))
require.NoError(t, err)
fileContents := readFileContentsFromZip(t, zr, "file1")
require.Equal(t, "abcd\n", fileContents)
})
t.Run("read multiple files from tar archive using paths", func(t *testing.T) {
r, err := backend.ArchiveReader(ctx, "tar", string(commitID), []string{"file1", "dir1/file2"})
require.NoError(t, err)
t.Cleanup(func() { r.Close() })
tr := tar.NewReader(r)
contents := readFileContentsFromTar(t, tr, "dir1/file2")
require.Equal(t, "efgh\n", contents)
r, err = backend.ArchiveReader(ctx, "tar", string(commitID), []string{"file1", "dir1/file2"})
require.NoError(t, err)
t.Cleanup(func() { r.Close() })
tr = tar.NewReader(r)
contents = readFileContentsFromTar(t, tr, "file1")
require.Equal(t, "abcd\n", contents)
})
t.Run("read file in directory", func(t *testing.T) {
r, err := backend.ArchiveReader(ctx, "tar", string(commitID), nil)
require.NoError(t, err)
t.Cleanup(func() { r.Close() })
tr := tar.NewReader(r)
contents := readFileContentsFromTar(t, tr, "dir1/file2")
require.Equal(t, "efgh\n", contents)
})
t.Run("non existent commit", func(t *testing.T) {
_, err := backend.ArchiveReader(ctx, "tar", "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", nil)
require.Error(t, err)
require.True(t, errors.HasType(err, &gitdomain.RevisionNotFoundError{}))
})
t.Run("non existent ref", func(t *testing.T) {
_, err := backend.ArchiveReader(ctx, "tar", "head-2", nil)
require.Error(t, err)
require.True(t, errors.HasType(err, &gitdomain.RevisionNotFoundError{}))
})
t.Run("non existent file", func(t *testing.T) {
_, err := backend.ArchiveReader(ctx, "tar", string(commitID), []string{"no-file"})
require.Error(t, err)
require.True(t, os.IsNotExist(err))
})
t.Run("invalid path pattern", func(t *testing.T) {
_, err := backend.ArchiveReader(ctx, "tar", string(commitID), []string{"dir1/*"})
require.Error(t, err)
require.True(t, os.IsNotExist(err))
})
// Verify that if the context is canceled, the reader returns an error.
t.Run("context cancelation", func(t *testing.T) {
ctx, cancel := context.WithCancel(ctx)
t.Cleanup(cancel)
r, err := backend.ArchiveReader(ctx, git.ArchiveFormatTar, string(commitID), nil)
require.NoError(t, err)
cancel()
tr := tar.NewReader(r)
_, err = tr.Next()
require.Error(t, err)
require.True(t, errors.Is(err, context.Canceled), "unexpected error: %v", err)
require.NoError(t, r.Close())
})
}

18
cmd/gitserver/internal/git/iface.go
View File

@ -45,6 +45,13 @@ type GitBackend interface {
// the list of all files touched in this commit.
// If the commit doesn't exist, a RevisionNotFoundError is returned.
GetCommit(ctx context.Context, commit api.CommitID, includeModifiedFiles bool) (*GitCommitWithFiles, error)
// ArchiveReader returns a reader for an archive in the given format.
// Treeish is the tree or commit to archive, and paths is the list of
// paths to include in the archive. If empty, all paths are included.
//
// If the commit does not exist, a RevisionNotFoundError is returned.
// If any path does not exist, a os.PathError is returned.
ArchiveReader(ctx context.Context, format ArchiveFormat, treeish string, paths []string) (io.ReadCloser, error)
// Exec is a temporary helper to run arbitrary git commands from the exec endpoint.
// No new usages of it should be introduced and once the migration is done we will
@ -91,3 +98,14 @@ type GitCommitWithFiles struct {
*gitdomain.Commit
ModifiedFiles []string
}
// ArchiveFormat indicates the desired format of the archive as an enum.
type ArchiveFormat string
const (
// ArchiveFormatZip indicates a zip archive is desired.
ArchiveFormatZip ArchiveFormat = "zip"
// ArchiveFormatTar indicates a tar archive is desired.
ArchiveFormatTar ArchiveFormat = "tar"
)

130
cmd/gitserver/internal/git/mock.go
View File

@ -283,6 +283,9 @@ func (c BlameHunkReaderReadFuncCall) Results() []interface{} {
// github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git) used for
// unit testing.
type MockGitBackend struct {
// ArchiveReaderFunc is an instance of a mock function object
// controlling the behavior of the method ArchiveReader.
ArchiveReaderFunc *GitBackendArchiveReaderFunc
// BlameFunc is an instance of a mock function object controlling the
// behavior of the method Blame.
BlameFunc *GitBackendBlameFunc
@ -316,6 +319,11 @@ type MockGitBackend struct {
// methods return zero values for all results, unless overwritten.
func NewMockGitBackend() *MockGitBackend {
return &MockGitBackend{
ArchiveReaderFunc: &GitBackendArchiveReaderFunc{
defaultHook: func(context.Context, ArchiveFormat, string, []string) (r0 io.ReadCloser, r1 error) {
return
},
},
BlameFunc: &GitBackendBlameFunc{
defaultHook: func(context.Context, api.CommitID, string, BlameOptions) (r0 BlameHunkReader, r1 error) {
return
@ -368,6 +376,11 @@ func NewMockGitBackend() *MockGitBackend {
// All methods panic on invocation, unless overwritten.
func NewStrictMockGitBackend() *MockGitBackend {
return &MockGitBackend{
ArchiveReaderFunc: &GitBackendArchiveReaderFunc{
defaultHook: func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error) {
panic("unexpected invocation of MockGitBackend.ArchiveReader")
},
},
BlameFunc: &GitBackendBlameFunc{
defaultHook: func(context.Context, api.CommitID, string, BlameOptions) (BlameHunkReader, error) {
panic("unexpected invocation of MockGitBackend.Blame")
@ -420,6 +433,9 @@ func NewStrictMockGitBackend() *MockGitBackend {
// All methods delegate to the given implementation, unless overwritten.
func NewMockGitBackendFrom(i GitBackend) *MockGitBackend {
return &MockGitBackend{
ArchiveReaderFunc: &GitBackendArchiveReaderFunc{
defaultHook: i.ArchiveReader,
},
BlameFunc: &GitBackendBlameFunc{
defaultHook: i.Blame,
},
@ -450,6 +466,120 @@ func NewMockGitBackendFrom(i GitBackend) *MockGitBackend {
}
}
// GitBackendArchiveReaderFunc describes the behavior when the ArchiveReader
// method of the parent MockGitBackend instance is invoked.
type GitBackendArchiveReaderFunc struct {
defaultHook func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error)
hooks []func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error)
history []GitBackendArchiveReaderFuncCall
mutex sync.Mutex
}
// ArchiveReader delegates to the next hook function in the queue and stores
// the parameter and result values of this invocation.
func (m *MockGitBackend) ArchiveReader(v0 context.Context, v1 ArchiveFormat, v2 string, v3 []string) (io.ReadCloser, error) {
r0, r1 := m.ArchiveReaderFunc.nextHook()(v0, v1, v2, v3)
m.ArchiveReaderFunc.appendCall(GitBackendArchiveReaderFuncCall{v0, v1, v2, v3, r0, r1})
return r0, r1
}
// SetDefaultHook sets function that is called when the ArchiveReader method
// of the parent MockGitBackend instance is invoked and the hook queue is
// empty.
func (f *GitBackendArchiveReaderFunc) SetDefaultHook(hook func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error)) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// ArchiveReader method of the parent MockGitBackend instance invokes the
// hook at the front of the queue and discards it. After the queue is empty,
// the default hook function is invoked for any future action.
func (f *GitBackendArchiveReaderFunc) PushHook(hook func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error)) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *GitBackendArchiveReaderFunc) SetDefaultReturn(r0 io.ReadCloser, r1 error) {
f.SetDefaultHook(func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error) {
return r0, r1
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *GitBackendArchiveReaderFunc) PushReturn(r0 io.ReadCloser, r1 error) {
f.PushHook(func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error) {
return r0, r1
})
}
func (f *GitBackendArchiveReaderFunc) nextHook() func(context.Context, ArchiveFormat, string, []string) (io.ReadCloser, error) {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *GitBackendArchiveReaderFunc) appendCall(r0 GitBackendArchiveReaderFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of GitBackendArchiveReaderFuncCall objects
// describing the invocations of this function.
func (f *GitBackendArchiveReaderFunc) History() []GitBackendArchiveReaderFuncCall {
f.mutex.Lock()
history := make([]GitBackendArchiveReaderFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// GitBackendArchiveReaderFuncCall is an object that describes an invocation
// of method ArchiveReader on an instance of MockGitBackend.
type GitBackendArchiveReaderFuncCall struct {
// Arg0 is the value of the 1st argument passed to this method
// invocation.
Arg0 context.Context
// Arg1 is the value of the 2nd argument passed to this method
// invocation.
Arg1 ArchiveFormat
// Arg2 is the value of the 3rd argument passed to this method
// invocation.
Arg2 string
// Arg3 is the value of the 4th argument passed to this method
// invocation.
Arg3 []string
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 io.ReadCloser
// Result1 is the value of the 2nd result returned from this method
// invocation.
Result1 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c GitBackendArchiveReaderFuncCall) Args() []interface{} {
return []interface{}{c.Arg0, c.Arg1, c.Arg2, c.Arg3}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c GitBackendArchiveReaderFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1}
}
// GitBackendBlameFunc describes the behavior when the Blame method of the
// parent MockGitBackend instance is invoked.
type GitBackendBlameFunc struct {

4
cmd/gitserver/internal/integration_tests/BUILD.bazel
View File

@ -35,7 +35,6 @@ go_test(
name = "integration_tests_test",
timeout = "short",
srcs = [
"archivereader_test.go",
"clone_test.go",
"main_test.go",
"object_test.go",
@ -72,16 +71,13 @@ go_test(
"//internal/types",
"//internal/vcs",
"//internal/wrexec",
"//lib/errors",
"//schema",
"@com_github_derision_test_go_mockgen//testutil/assert",
"@com_github_derision_test_go_mockgen//testutil/require",
"@com_github_google_go_cmp//cmp",
"@com_github_sourcegraph_log//:log",
"@com_github_sourcegraph_log//logtest",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_x_time//rate",
],
)

323
cmd/gitserver/internal/integration_tests/archivereader_test.go
View File

@ -1,323 +0,0 @@
package inttests
import (
"archive/zip"
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/sourcegraph/log/logtest"
"golang.org/x/time/rate"
"google.golang.org/grpc"
server "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal"
common "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/common"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git/gitcli"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/vcssyncer"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc"
"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
"github.com/sourcegraph/sourcegraph/internal/wrexec"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func TestClient_ArchiveReader(t *testing.T) {
root := gitserver.CreateRepoDir(t)
type test struct {
name string
remote string
revision string
want map[string]string
clientErr error
readerError error
skipReader bool
}
tests := []test{
{
name: "simple",
remote: createSimpleGitRepo(t, root),
revision: "HEAD",
want: map[string]string{
"dir1/": "",
"dir1/file1": "infile1",
"file 2": "infile2",
},
skipReader: false,
},
{
name: "repo-with-dotgit-dir",
remote: createRepoWithDotGitDir(t, root),
revision: "HEAD",
want: map[string]string{
"file1": "hello\n",
".git/mydir/file2": "milton\n",
".git/mydir/": "",
".git/": "",
},
skipReader: false,
},
{
name: "not-found",
revision: "HEAD",
clientErr: errors.New("repository does not exist: not-found"),
skipReader: false,
},
{
name: "revision-not-found",
remote: createRepoWithDotGitDir(t, root),
revision: "revision-not-found",
clientErr: nil,
readerError: &gitdomain.RevisionNotFoundError{Repo: "revision-not-found", Spec: "revision-not-found"},
skipReader: true,
},
}
runArchiveReaderTestfunc := func(t *testing.T, mkClient func(t *testing.T, addrs []string) gitserver.Client, name api.RepoName, test test) {
t.Run(string(name), func(t *testing.T) {
// Setup: Prepare the test Gitserver server + register the gRPC server
s := server.NewServer(&server.ServerOpts{
Logger: logtest.Scoped(t),
ReposDir: filepath.Join(root, "repos"),
DB: newMockDB(),
GetBackendFunc: func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
return gitcli.NewBackend(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory(), dir, repoName)
},
GetRemoteURLFunc: func(_ context.Context, name api.RepoName) (string, error) {
if test.remote != "" {
return test.remote, nil
}
return "", errors.Errorf("no remote for %s", test.name)
},
GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) {
return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory()), nil
},
RecordingCommandFactory: wrexec.NewNoOpRecordingCommandFactory(),
Locker: server.NewRepositoryLocker(),
RPSLimiter: ratelimit.NewInstrumentedLimiter("GitserverTest", rate.NewLimiter(100, 10)),
})
grpcServer := defaults.NewServer(logtest.Scoped(t))
proto.RegisterGitserverServiceServer(grpcServer, server.NewGRPCServer(s))
handler := internalgrpc.MultiplexHandlers(grpcServer, s.Handler())
srv := httptest.NewServer(handler)
defer srv.Close()
u, _ := url.Parse(srv.URL)
addrs := []string{u.Host}
cli := mkClient(t, addrs)
ctx := context.Background()
if test.remote != "" {
if _, err := cli.RequestRepoUpdate(ctx, name, 0); err != nil {
t.Fatal(err)
}
}
rc, err := cli.ArchiveReader(ctx, name, gitserver.ArchiveOptions{Treeish: test.revision, Format: gitserver.ArchiveFormatZip})
if have, want := fmt.Sprint(err), fmt.Sprint(test.clientErr); have != want {
t.Errorf("archive: have err %v, want %v", have, want)
}
if rc == nil {
return
}
t.Cleanup(func() {
if err := rc.Close(); err != nil {
t.Fatal(err)
}
})
data, readErr := io.ReadAll(rc)
if readErr != nil {
if readErr.Error() != test.readerError.Error() {
t.Errorf("archive: have reader err %v, want %v", readErr.Error(), test.readerError.Error())
}
if test.skipReader {
return
}
t.Fatal(readErr)
}
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
t.Fatal(err)
}
got := map[string]string{}
for _, f := range zr.File {
r, err := f.Open()
if err != nil {
t.Errorf("failed to open %q because %s", f.Name, err)
continue
}
contents, err := io.ReadAll(r)
_ = r.Close()
if err != nil {
t.Errorf("Read(%q): %s", f.Name, err)
continue
}
got[f.Name] = string(contents)
}
if !cmp.Equal(test.want, got) {
t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(test.want, got))
}
})
}
for _, test := range tests {
repoName := api.RepoName(test.name)
called := false
mkClient := func(t *testing.T, addrs []string) gitserver.Client {
t.Helper()
source := gitserver.NewTestClientSource(t, addrs, func(o *gitserver.TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
base := proto.NewGitserverServiceClient(cc)
mockArchive := func(ctx context.Context, in *proto.ArchiveRequest, opts ...grpc.CallOption) (proto.GitserverService_ArchiveClient, error) {
called = true
return base.Archive(ctx, in, opts...)
}
mockRepoUpdate := func(ctx context.Context, in *proto.RepoUpdateRequest, opts ...grpc.CallOption) (*proto.RepoUpdateResponse, error) {
base := proto.NewGitserverServiceClient(cc)
return base.RepoUpdate(ctx, in, opts...)
}
cli := gitserver.NewMockGitserverServiceClient()
cli.ArchiveFunc.SetDefaultHook(mockArchive)
cli.RepoUpdateFunc.SetDefaultHook(mockRepoUpdate)
return cli
}
})
return gitserver.NewTestClient(t).WithClientSource(source)
}
runArchiveReaderTestfunc(t, mkClient, repoName, test)
if !called {
t.Error("archiveReader: GitserverServiceClient should have been called")
}
}
}
func createSimpleGitRepo(t *testing.T, root string) string {
t.Helper()
dir := filepath.Join(root, "remotes", "simple")
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
for _, cmd := range []string{
"git init",
"mkdir dir1",
"echo -n infile1 > dir1/file1",
"touch --date=2006-01-02T15:04:05Z dir1 dir1/file1 || touch -t 200601021704.05 dir1 dir1/file1",
"git add dir1/file1",
"GIT_COMMITTER_NAME=a GIT_COMMITTER_EMAIL=a@a.com GIT_AUTHOR_DATE=2006-01-02T15:04:05Z GIT_COMMITTER_DATE=2006-01-02T15:04:05Z git commit -m commit1 --author='a <a@a.com>' --date 2006-01-02T15:04:05Z",
"echo -n infile2 > 'file 2'",
"touch --date=2014-05-06T19:20:21Z 'file 2' || touch -t 201405062120.21 'file 2'",
"git add 'file 2'",
"GIT_COMMITTER_NAME=a GIT_COMMITTER_EMAIL=a@a.com GIT_AUTHOR_DATE=2006-01-02T15:04:05Z GIT_COMMITTER_DATE=2014-05-06T19:20:21Z git commit -m commit2 --author='a <a@a.com>' --date 2014-05-06T19:20:21Z",
"git branch test-ref HEAD~1",
"git branch test-nested-ref test-ref",
} {
c := exec.Command("bash", "-c", `GIT_CONFIG_GLOBAL="" GIT_CONFIG_SYSTEM="" `+cmd)
c.Dir = dir
out, err := c.CombinedOutput()
if err != nil {
t.Fatalf("Command %q failed. Output was:\n\n%s", cmd, out)
}
}
return dir
}
func createRepoWithDotGitDir(t *testing.T, root string) string {
t.Helper()
b64 := func(s string) string {
t.Helper()
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
t.Fatal(err)
}
return string(b)
}
dir := filepath.Join(root, "remotes", "repo-with-dot-git-dir")
// This repo was synthesized by hand to contain a file whose path is `.git/mydir/file2` (the Git
// CLI will not let you create a file with a `.git` path component).
//
// The synthesized bad commit is:
//
// commit aa600fc517ea6546f31ae8198beb1932f13b0e4c (HEAD -> master)
// Author: Quinn Slack <qslack@qslack.com>
// Date: Tue Jun 5 16:17:20 2018 -0700
//
// wip
//
// diff --git a/.git/mydir/file2 b/.git/mydir/file2
// new file mode 100644
// index 0000000..82b919c
// --- /dev/null
// +++ b/.git/mydir/file2
// @@ -0,0 +1 @@
// +milton
files := map[string]string{
"config": `
[core]
repositoryformatversion=0
filemode=true
`,
"HEAD": `ref: refs/heads/master`,
"refs/heads/master": `aa600fc517ea6546f31ae8198beb1932f13b0e4c`,
"objects/e7/9c5e8f964493290a409888d5413a737e8e5dd5": b64("eAFLyslPUrBgyMzLLMlMzOECACgtBOw="),
"objects/ce/013625030ba8dba906f756967f9e9ca394464a": b64("eAFLyslPUjBjyEjNycnnAgAdxQQU"),
"objects/82/b919c9c565d162c564286d9d6a2497931be47e": b64("eAFLyslPUjBnyM3MKcnP4wIAIw8ElA=="),
"objects/e5/231c1d547df839dce09809e43608fe6c537682": b64("eAErKUpNVTAzYTAxAAIFvfTMEgbb8lmsKdJ+zz7ukeMOulcqZqOllmloYGBmYqKQlpmTashwjtFMlZl7xe2VbN/DptXPm7N4ipsXACOoGDo="),
"objects/da/5ecc846359eaf23e8abe907b3125fdd7abdbc0": b64("eAErKUpNVTA2ZjA0MDAzMVFIy8xJNWJo2il58mjqxaSjKRq5c7NUpk+WflIHABZRD2I="),
"objects/d0/01d287018593691c36042e1c8089fde7415296": b64("eAErKUpNVTA2ZjA0MDAzMVFIy8xJNWQ4x2imysy94vZKtu9h0+rnzVk8xc0LAP2TDiQ="),
"objects/b4/009ecbf1eba01c5279f25840e2afc0d15f5005": b64("eAGdjdsJAjEQRf1OFdOAMpPN5gEitiBWEJIRBzcJu2b7N2IHfh24nMtJrRTpQA4PfWOGjEhZe4fk5zDZQGmyaDRT8ujDI7MzNOtgVdz7s21w26VWuC8xveC8vr+8/nBKrVxgyF4bJBfgiA5RjXUEO/9xVVKlS1zUB/JxNbA="),
"objects/3d/779a05641b4ee6f1bc1e0b52de75163c2a2669": b64("eAErKUpNVTA2YjAxAAKF3MqUzCKGW3FnWpIjX32y69o3odpQ9e/11bcPAAAipRGQ"),
"objects/aa/600fc517ea6546f31ae8198beb1932f13b0e4c": b64("eAGdjlkKAjEQBf3OKfoCSmfpLCDiFcQTZDodHHQWxwxe3xFv4FfBKx4UT8PQNzDa7doiAkLGataFXCg12lRYMEVM4qzHWMUz2eCjUXNeZGzQOdwkd1VLl1EzmZCqoehQTK6MRVMlRFJ5bbdpgcvajyNcH5nvcHy+vjz/cOBpOIEmE41D7xD2GBDVtm6BTf64qnc/qw9c4UKS"),
"objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391": b64("eAFLyslPUjBgAAAJsAHw"),
}
for name, data := range files {
name = filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(name), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(name, []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
return dir
}

33
cmd/gitserver/internal/integration_tests/test_utils.go
View File

@ -161,3 +161,36 @@ func GitCommand(dir, name string, args ...string) *exec.Cmd {
)
return c
}
func createSimpleGitRepo(t *testing.T, root string) string {
t.Helper()
dir := filepath.Join(root, "remotes", "simple")
if err := os.MkdirAll(dir, 0o700); err != nil {
t.Fatal(err)
}
for _, cmd := range []string{
"git init",
"mkdir dir1",
"echo -n infile1 > dir1/file1",
"touch --date=2006-01-02T15:04:05Z dir1 dir1/file1 || touch -t 200601021704.05 dir1 dir1/file1",
"git add dir1/file1",
"GIT_COMMITTER_NAME=a GIT_COMMITTER_EMAIL=a@a.com GIT_AUTHOR_DATE=2006-01-02T15:04:05Z GIT_COMMITTER_DATE=2006-01-02T15:04:05Z git commit -m commit1 --author='a <a@a.com>' --date 2006-01-02T15:04:05Z",
"echo -n infile2 > 'file 2'",
"touch --date=2014-05-06T19:20:21Z 'file 2' || touch -t 201405062120.21 'file 2'",
"git add 'file 2'",
"GIT_COMMITTER_NAME=a GIT_COMMITTER_EMAIL=a@a.com GIT_AUTHOR_DATE=2006-01-02T15:04:05Z GIT_COMMITTER_DATE=2014-05-06T19:20:21Z git commit -m commit2 --author='a <a@a.com>' --date 2014-05-06T19:20:21Z",
"git branch test-ref HEAD~1",
"git branch test-nested-ref test-ref",
} {
c := exec.Command("bash", "-c", `GIT_CONFIG_GLOBAL="" GIT_CONFIG_SYSTEM="" `+cmd)
c.Dir = dir
out, err := c.CombinedOutput()
if err != nil {
t.Fatalf("Command %q failed. Output was:\n\n%s", cmd, out)
}
}
return dir
}

122
cmd/gitserver/internal/server_grpc.go
View File

@ -2,6 +2,7 @@ package internal
import (
"context"
"fmt"
"io"
"os"
"strings"
@ -21,7 +22,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/gitserver/protocol"
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
@ -158,46 +158,100 @@ func (gs *grpcServer) Exec(req *proto.ExecRequest, ss proto.GitserverService_Exe
}
func (gs *grpcServer) Archive(req *proto.ArchiveRequest, ss proto.GitserverService_ArchiveServer) error {
// Log which which actor is accessing the repo.
accesslog.Record(ss.Context(), req.GetRepo(),
log.String("treeish", req.GetTreeish()),
log.String("format", req.GetFormat()),
log.Strings("path", req.GetPathspecs()),
)
if err := git.CheckSpecArgSafety(req.GetTreeish()); err != nil {
return status.Error(codes.InvalidArgument, err.Error())
}
if req.GetRepo() == "" || req.GetFormat() == "" {
return status.Error(codes.InvalidArgument, "empty repo or format")
}
ctx := ss.Context()
if req.GetRepo() == "" {
return status.New(codes.InvalidArgument, "repo must be specified").Err()
}
if req.GetTreeish() == "" {
return status.New(codes.InvalidArgument, "treeish must be specified").Err()
}
var format git.ArchiveFormat
switch req.GetFormat() {
case proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP:
format = git.ArchiveFormatZip
case proto.ArchiveFormat_ARCHIVE_FORMAT_TAR:
format = git.ArchiveFormatTar
default:
return status.Error(codes.InvalidArgument, fmt.Sprintf("unknown archive format %q", req.GetFormat()))
}
if err := git.CheckSpecArgSafety(req.GetTreeish()); err != nil {
return status.Error(codes.InvalidArgument, err.Error())
}
accesslog.Record(ctx, req.GetRepo(),
log.String("treeish", req.GetTreeish()),
log.String("format", string(format)),
log.Strings("path", req.GetPaths()),
)
repoName := api.RepoName(req.GetRepo())
repoDir := gitserverfs.RepoDirFromName(gs.reposDir, repoName)
if err := gs.maybeStartClone(ss.Context(), repoName); err != nil {
return err
}
execReq := &protocol.ExecRequest{
Repo: api.RepoName(req.GetRepo()),
Args: []string{
"archive",
"--worktree-attributes",
"--format=" + req.GetFormat(),
},
if !actor.FromContext(ctx).IsInternal() {
if enabled, err := gs.subRepoChecker.EnabledForRepo(ctx, repoName); err != nil {
return errors.Wrap(err, "sub-repo permissions check")
} else if enabled {
s := status.New(codes.Unimplemented, "archiveReader invoked for a repo with sub-repo permissions")
return s.Err()
}
}
if req.GetFormat() == string(gitserver.ArchiveFormatZip) {
execReq.Args = append(execReq.Args, "-0")
}
// This is a long time, but this never blocks a user request for this
// long. Even repos that are not that large can take a long time, for
// example a search over all repos in an organization may have several
// large repos. All of those repos will be competing for IO => we need
// a larger timeout.
ctx, cancel := context.WithTimeout(ctx, conf.GitLongCommandTimeout())
defer cancel()
execReq.Args = append(execReq.Args, req.GetTreeish(), "--")
execReq.Args = append(execReq.Args, req.GetPathspecs()...)
backend := gs.getBackendFunc(repoDir, repoName)
r, err := backend.ArchiveReader(ctx, format, req.GetTreeish(), req.GetPaths())
if err != nil {
if os.IsNotExist(err) {
var path string
var pathError *os.PathError
if errors.As(err, &pathError) {
path = pathError.Path
}
s, err := status.New(codes.NotFound, "file not found").WithDetails(&proto.FileNotFoundPayload{
Repo: string(repoName),
// TODO: I'm not sure this should be allowed, a treeish is not necessarily
// a commit.
Commit: string(req.GetTreeish()),
Path: path,
})
if err != nil {
return err
}
return s.Err()
}
var e *gitdomain.RevisionNotFoundError
if errors.As(err, &e) {
s, err := status.New(codes.NotFound, "revision not found").WithDetails(&proto.RevisionNotFoundPayload{
Repo: req.GetRepo(),
Spec: e.Spec,
})
if err != nil {
return err
}
return s.Err()
}
gs.svc.LogIfCorrupt(ctx, repoName, err)
// TODO: Better error checking.
return err
}
defer r.Close()
w := streamio.NewWriter(func(p []byte) error {
return ss.Send(&proto.ArchiveResponse{
@ -205,15 +259,8 @@ func (gs *grpcServer) Archive(req *proto.ArchiveRequest, ss proto.GitserverServi
})
})
// This is a long time, but this never blocks a user request for this
// long. Even repos that are not that large can take a long time, for
// example a search over all repos in an organization may have several
// large repos. All of those repos will be competing for IO => we need
// a larger timeout.
ctx, cancel := context.WithTimeout(ss.Context(), conf.GitLongCommandTimeout())
defer cancel()
return gs.doExec(ctx, execReq, w)
_, err = io.Copy(w, r)
return err
}
// doExec executes the given git command and streams the output to the given writer.
@ -257,7 +304,6 @@ func (gs *grpcServer) doExec(ctx context.Context, req *protocol.ExecRequest, w i
}
return nil
}
func (gs *grpcServer) GetObject(ctx context.Context, req *proto.GetObjectRequest) (*proto.GetObjectResponse, error) {
@ -342,7 +388,6 @@ func (gs *grpcServer) RepoClone(ctx context.Context, in *proto.RepoCloneRequest)
repo := protocol.NormalizeRepo(api.RepoName(in.GetRepo()))
if _, err := gs.svc.CloneRepo(ctx, repo, CloneOptions{Block: false}); err != nil {
return &proto.RepoCloneResponse{Error: err.Error()}, nil
}
@ -940,7 +985,6 @@ func (gs *grpcServer) Blame(req *proto.BlameRequest, ss proto.GitserverService_B
}
r, err := backend.Blame(ctx, api.CommitID(req.GetCommit()), req.GetPath(), opts)
if err != nil {
if os.IsNotExist(err) {
s, err := status.New(codes.NotFound, "file not found").WithDetails(&proto.FileNotFoundPayload{

138
cmd/gitserver/internal/server_grpc_test.go
View File

@ -428,6 +428,144 @@ func TestGRPCServer_ReadFile(t *testing.T) {
})
}
func TestGRPCServer_Archive(t *testing.T) {
mockSS := gitserver.NewMockGitserverService_ArchiveServer()
// Add an actor to the context.
a := actor.FromUser(1)
mockSS.ContextFunc.SetDefaultReturn(actor.WithActor(context.Background(), a))
t.Run("argument validation", func(t *testing.T) {
gs := &grpcServer{}
err := gs.Archive(&v1.ArchiveRequest{Repo: ""}, mockSS)
require.ErrorContains(t, err, "repo must be specified")
assertGRPCStatusCode(t, err, codes.InvalidArgument)
err = gs.Archive(&v1.ArchiveRequest{Repo: "therepo", Format: proto.ArchiveFormat_ARCHIVE_FORMAT_TAR}, mockSS)
require.ErrorContains(t, err, "treeish must be specified")
assertGRPCStatusCode(t, err, codes.InvalidArgument)
err = gs.Archive(&v1.ArchiveRequest{Repo: "therepo", Treeish: "HEAD"}, mockSS)
require.ErrorContains(t, err, "unknown archive format")
assertGRPCStatusCode(t, err, codes.InvalidArgument)
})
t.Run("checks for uncloned repo", func(t *testing.T) {
svc := NewMockService()
svc.MaybeStartCloneFunc.SetDefaultReturn(&protocol.NotFoundPayload{CloneInProgress: true, CloneProgress: "cloning"}, false)
gs := &grpcServer{svc: svc}
err := gs.Archive(&v1.ArchiveRequest{Repo: "therepo", Treeish: "HEAD", Format: proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP}, mockSS)
require.Error(t, err)
assertGRPCStatusCode(t, err, codes.NotFound)
assertHasGRPCErrorDetailOfType(t, err, &proto.RepoNotFoundPayload{})
require.Contains(t, err.Error(), "repo not found")
mockassert.Called(t, svc.MaybeStartCloneFunc)
})
t.Run("checks if sub-repo perms are enabled for repo", func(t *testing.T) {
srp := authz.NewMockSubRepoPermissionChecker()
svc := NewMockService()
// Repo is cloned, proceed!
svc.MaybeStartCloneFunc.SetDefaultReturn(nil, true)
gs := &grpcServer{
subRepoChecker: srp,
svc: svc,
getBackendFunc: func(common.GitDir, api.RepoName) git.GitBackend {
b := git.NewMockGitBackend()
b.ArchiveReaderFunc.SetDefaultReturn(io.NopCloser(bytes.NewReader([]byte("filecontent"))), nil)
return b
},
}
t.Run("subrepo perms are enabled but actor is internal", func(t *testing.T) {
srp.EnabledForRepoFunc.SetDefaultReturn(true, nil)
mockSS := gitserver.NewMockGitserverService_ArchiveServer()
// Add an internal actor to the context.
mockSS.ContextFunc.SetDefaultReturn(actor.WithInternalActor(context.Background()))
err := gs.Archive(&v1.ArchiveRequest{Repo: "therepo", Treeish: "HEAD", Format: proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP}, mockSS)
assert.NoError(t, err)
mockassert.NotCalled(t, srp.EnabledForRepoFunc)
})
t.Run("subrepo perms are not enabled", func(t *testing.T) {
srp.EnabledForRepoFunc.SetDefaultReturn(false, nil)
err := gs.Archive(&v1.ArchiveRequest{Repo: "therepo", Treeish: "HEAD", Format: proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP}, mockSS)
assert.NoError(t, err)
mockassert.Called(t, srp.EnabledForRepoFunc)
})
t.Run("subrepo perms are enabled, returns error", func(t *testing.T) {
srp.EnabledForRepoFunc.SetDefaultReturn(true, nil)
err := gs.Archive(&v1.ArchiveRequest{Repo: "therepo", Treeish: "HEAD", Format: proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP}, mockSS)
assert.Error(t, err)
assertGRPCStatusCode(t, err, codes.Unimplemented)
require.Contains(t, err.Error(), "archiveReader invoked for a repo with sub-repo permissions")
mockassert.Called(t, srp.EnabledForRepoFunc)
})
})
t.Run("e2e", func(t *testing.T) {
srp := authz.NewMockSubRepoPermissionChecker()
// Skip subrepo perms checks.
srp.EnabledForRepoFunc.SetDefaultReturn(false, nil)
svc := NewMockService()
// Repo is cloned, proceed!
svc.MaybeStartCloneFunc.SetDefaultReturn(nil, true)
b := git.NewMockGitBackend()
b.ArchiveReaderFunc.SetDefaultReturn(io.NopCloser(bytes.NewReader([]byte("filecontent"))), nil)
gs := &grpcServer{
subRepoChecker: srp,
svc: svc,
getBackendFunc: func(common.GitDir, api.RepoName) git.GitBackend {
return b
},
}
cli := spawnServer(t, gs)
r, err := cli.Archive(context.Background(), &v1.ArchiveRequest{
Repo: "therepo",
Treeish: "HEAD",
Format: proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP,
})
require.NoError(t, err)
for {
msg, err := r.Recv()
if err != nil {
if err == io.EOF {
break
}
require.NoError(t, err)
}
if diff := cmp.Diff(&proto.ArchiveResponse{
Data: []byte("filecontent"),
}, msg, cmpopts.IgnoreUnexported(proto.ArchiveResponse{})); diff != "" {
t.Fatalf("unexpected response (-want +got):\n%s", diff)
}
}
// Invalid file path.
b.ArchiveReaderFunc.SetDefaultReturn(nil, os.ErrNotExist)
cc, err := cli.Archive(context.Background(), &v1.ArchiveRequest{
Repo: "therepo",
Treeish: "HEAD",
Format: proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP,
})
require.NoError(t, err)
_, err = cc.Recv()
require.Error(t, err)
assertGRPCStatusCode(t, err, codes.NotFound)
assertHasGRPCErrorDetailOfType(t, err, &proto.FileNotFoundPayload{})
// TODO: Do we return this?
b.ArchiveReaderFunc.SetDefaultReturn(nil, &gitdomain.RevisionNotFoundError{})
cc, err = cli.Archive(context.Background(), &v1.ArchiveRequest{
Repo: "therepo",
Treeish: "HEAD",
Format: proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP,
})
require.NoError(t, err)
_, err = cc.Recv()
require.Error(t, err)
assertGRPCStatusCode(t, err, codes.NotFound)
assertHasGRPCErrorDetailOfType(t, err, &proto.RevisionNotFoundPayload{})
})
}
func TestGRPCServer_GetCommit(t *testing.T) {
// Add an actor to the context.
a := actor.FromUser(1)

2
cmd/searcher/internal/search/search_test.go
View File

@ -704,7 +704,7 @@ func fetchTimeoutForCI(t *testing.T) time.Duration {
if deadline, ok := t.Deadline(); ok {
return time.Until(deadline) / 2
}
return 500 * time.Millisecond
return 1000 * time.Millisecond
}
func toString(m []protocol.FileMatch) string {

4
cmd/searcher/internal/search/store.go
View File

@ -264,12 +264,12 @@ func (s *Store) fetch(ctx context.Context, repo api.RepoName, commit api.CommitI
if len(paths) == 0 {
r, err = s.FetchTar(ctx, repo, commit)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "fetching tar")
}
} else {
r, err = s.FetchTarPaths(ctx, repo, commit, paths)
if err != nil {
return nil, err
return nil, errors.Wrapf(err, "fetching tar paths: %v", paths)
}
}

1
cmd/searcher/shared/BUILD.bazel
View File

@ -16,7 +16,6 @@ go_library(
"//internal/debugserver",
"//internal/env",
"//internal/gitserver",
"//internal/gitserver/gitdomain",
"//internal/goroutine",
"//internal/grpc",
"//internal/grpc/defaults",

11
cmd/searcher/shared/shared.go
View File

@ -23,7 +23,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc"
"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
@ -138,17 +137,13 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi
})
},
FetchTarPaths: func(ctx context.Context, repo api.RepoName, commit api.CommitID, paths []string) (io.ReadCloser, error) {
pathspecs := make([]gitdomain.Pathspec, len(paths))
for i, p := range paths {
pathspecs[i] = gitdomain.PathspecLiteral(p)
}
// We pass in a nil sub-repo permissions checker and an internal actor here since
// searcher needs access to all data in the archive.
ctx = actor.WithInternalActor(ctx)
return git.ArchiveReader(ctx, repo, gitserver.ArchiveOptions{
Treeish: string(commit),
Format: gitserver.ArchiveFormatTar,
Pathspecs: pathspecs,
Treeish: string(commit),
Format: gitserver.ArchiveFormatTar,
Paths: paths,
})
},
FilterTar: search.NewFilter,

11
cmd/symbols/gitserver/client.go
View File

@ -65,15 +65,10 @@ func (c *gitserverClient) FetchTar(ctx context.Context, repo api.RepoName, commi
}})
defer endObservation(1, observation.Args{})
pathSpecs := []gitdomain.Pathspec{}
for _, path := range paths {
pathSpecs = append(pathSpecs, gitdomain.PathspecLiteral(path))
}
opts := gitserver.ArchiveOptions{
Treeish: string(commit),
Format: gitserver.ArchiveFormatTar,
Pathspecs: pathSpecs,
Treeish: string(commit),
Format: gitserver.ArchiveFormatTar,
Paths: paths,
}
// Note: the sub-repo perms checker is nil here because we do the sub-repo filtering at a higher level

6
internal/authz/subrepoperms/sub_repo_perms.go
View File

@ -351,6 +351,9 @@ func (s *SubRepoPermsClient) getCompiledRules(ctx context.Context, userID int32)
}
func (s *SubRepoPermsClient) Enabled() bool {
if s == nil {
return false
}
return s.enabled.Load()
}
@ -371,6 +374,9 @@ func (s *SubRepoPermsClient) EnabledForRepoID(ctx context.Context, id api.RepoID
}
func (s *SubRepoPermsClient) EnabledForRepo(ctx context.Context, repo api.RepoName) (bool, error) {
if !s.Enabled() {
return false, nil
}
return s.permissionsGetter.RepoSupported(ctx, repo)
}

11
internal/codeintel/autoindexing/internal/inference/service.go
View File

@ -17,7 +17,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/codeintel/autoindexing/internal/inference/luatypes"
"github.com/sourcegraph/sourcegraph/internal/codeintel/autoindexing/shared"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/luasandbox"
"github.com/sourcegraph/sourcegraph/internal/luasandbox/util"
"github.com/sourcegraph/sourcegraph/internal/observation"
@ -315,14 +314,10 @@ func (s *Service) resolveFileContents(
return nil, err
}
pathspecs := make([]gitdomain.Pathspec, 0, len(relevantPaths))
for _, p := range relevantPaths {
pathspecs = append(pathspecs, gitdomain.PathspecLiteral(p))
}
opts := gitserver.ArchiveOptions{
Treeish: invocationContext.commit,
Format: gitserver.ArchiveFormatTar,
Pathspecs: pathspecs,
Treeish: invocationContext.commit,
Format: gitserver.ArchiveFormatTar,
Paths: relevantPaths,
}
rc, err := invocationContext.gitService.Archive(ctx, invocationContext.repo, opts)
if err != nil {

7
internal/codeintel/autoindexing/internal/inference/service_test.go
View File

@ -4,7 +4,6 @@ import (
"context"
"io"
"sort"
"strings"
"testing"
"golang.org/x/time/rate"
@ -46,9 +45,9 @@ func testService(t *testing.T, repositoryContents map[string]string) *Service {
})
gitService.ArchiveFunc.SetDefaultHook(func(ctx context.Context, repoName api.RepoName, opts gitserver.ArchiveOptions) (io.ReadCloser, error) {
files := map[string]string{}
for _, spec := range opts.Pathspecs {
if contents, ok := repositoryContents[strings.TrimPrefix(string(spec), ":(literal)")]; ok {
files[string(spec)] = contents
for _, path := range opts.Paths {
if contents, ok := repositoryContents[path]; ok {
files[path] = contents
}
}

1
internal/gitserver/BUILD.bazel
View File

@ -79,7 +79,6 @@ go_test(
"//internal/gitserver/protocol",
"//internal/gitserver/v1:gitserver",
"//internal/grpc",
"//internal/types",
"//lib/errors",
"//schema",
"@com_github_google_go_cmp//cmp",

119
internal/gitserver/client.go
View File

@ -11,8 +11,6 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"go.opentelemetry.io/otel/attribute"
"google.golang.org/grpc/status"
@ -39,7 +37,6 @@ const git = "git"
var ClientMocks, emptyClientMocks struct {
GetObject func(repo api.RepoName, objectName string) (*gitdomain.GitObject, error)
Archive func(ctx context.Context, repo api.RepoName, opt ArchiveOptions) (_ io.ReadCloser, err error)
LocalGitserver bool
LocalGitCommandReposDir string
}
@ -260,6 +257,42 @@ type CommitLog struct {
ChangedFiles []string
}
// ArchiveOptions contains options for the Archive func.
type ArchiveOptions struct {
Treeish string // the tree or commit to produce an archive for
Format ArchiveFormat // format of the resulting archive (usually "tar" or "zip")
Paths []string // if nonempty, only include these paths.
}
func (a *ArchiveOptions) Attrs() []attribute.KeyValue {
pathAttrs := make([]string, len(a.Paths))
for i, path := range a.Paths {
pathAttrs[i] = string(path)
}
return []attribute.KeyValue{
attribute.String("treeish", a.Treeish),
attribute.String("format", string(a.Format)),
attribute.StringSlice("paths", pathAttrs),
}
}
func (o *ArchiveOptions) FromProto(x *proto.ArchiveRequest) {
*o = ArchiveOptions{
Treeish: x.GetTreeish(),
Format: ArchiveFormatFromProto(x.GetFormat()),
Paths: x.GetPaths(),
}
}
func (o *ArchiveOptions) ToProto(repo string) *proto.ArchiveRequest {
return &proto.ArchiveRequest{
Repo: repo,
Treeish: o.Treeish,
Format: o.Format.ToProto(),
Paths: o.Paths,
}
}
type Client interface {
// Scoped adds a usage scope to the client and returns a new client with that scope.
// Usage scopes should be descriptive and be lowercase plaintext, eg. batches.reconciler.
@ -555,80 +588,6 @@ func (c *clientImplementor) ClientForRepo(ctx context.Context, repo api.RepoName
return c.clientSource.ClientForRepo(ctx, repo)
}
// ArchiveOptions contains options for the Archive func.
type ArchiveOptions struct {
Treeish string // the tree or commit to produce an archive for
Format ArchiveFormat // format of the resulting archive (usually "tar" or "zip")
Pathspecs []gitdomain.Pathspec // if nonempty, only include these pathspecs.
}
func (a *ArchiveOptions) Attrs() []attribute.KeyValue {
specs := make([]string, len(a.Pathspecs))
for i, pathspec := range a.Pathspecs {
specs[i] = string(pathspec)
}
return []attribute.KeyValue{
attribute.String("treeish", a.Treeish),
attribute.String("format", string(a.Format)),
attribute.StringSlice("pathspecs", specs),
}
}
func (o *ArchiveOptions) FromProto(x *proto.ArchiveRequest) {
protoPathSpecs := x.GetPathspecs()
pathSpecs := make([]gitdomain.Pathspec, 0, len(protoPathSpecs))
for _, path := range protoPathSpecs {
pathSpecs = append(pathSpecs, gitdomain.Pathspec(path))
}
*o = ArchiveOptions{
Treeish: x.GetTreeish(),
Format: ArchiveFormat(x.GetFormat()),
Pathspecs: pathSpecs,
}
}
func (o *ArchiveOptions) ToProto(repo string) *proto.ArchiveRequest {
protoPathSpecs := make([]string, 0, len(o.Pathspecs))
for _, path := range o.Pathspecs {
protoPathSpecs = append(protoPathSpecs, string(path))
}
return &proto.ArchiveRequest{
Repo: repo,
Treeish: o.Treeish,
Format: string(o.Format),
Pathspecs: protoPathSpecs,
}
}
// archiveReader wraps the StdoutReader yielded by gitserver's
// RemoteGitCommand.StdoutReader with one that knows how to report a repository-not-found
// error more carefully.
type archiveReader struct {
base io.ReadCloser
repo api.RepoName
spec string
}
// Read checks the known output behavior of the StdoutReader.
func (a *archiveReader) Read(p []byte) (int, error) {
n, err := a.base.Read(p)
if err != nil {
// handle the special case where git archive failed because of an invalid spec
if isRevisionNotFound(err.Error()) {
return 0, &gitdomain.RevisionNotFoundError{Repo: a.repo, Spec: a.spec}
}
}
return n, err
}
func (a *archiveReader) Close() error {
return a.base.Close()
}
func (c *RemoteGitCommand) sendExec(ctx context.Context) (_ io.ReadCloser, err error) {
ctx, cancel := context.WithCancel(ctx)
ctx, _, endObservation := c.execOp.With(ctx, &err, observation.Args{
@ -652,7 +611,6 @@ func (c *RemoteGitCommand) sendExec(ctx context.Context) (_ io.ReadCloser, err e
// Check that ctx is not expired.
if err := ctx.Err(); err != nil {
deadlineExceededCounter.Inc()
return nil, err
}
@ -746,11 +704,6 @@ func (c *clientImplementor) Search(ctx context.Context, args *protocol.SearchReq
}
}
var deadlineExceededCounter = promauto.NewCounter(prometheus.CounterOpts{
Name: "src_gitserver_client_deadline_exceeded",
Help: "Times that Client.sendExec() returned context.DeadlineExceeded",
})
func (c *clientImplementor) gitCommand(repo api.RepoName, arg ...string) GitCommand {
if ClientMocks.LocalGitserver {
cmd := NewLocalGitCommand(repo, arg...)

2
internal/gitserver/client_test.go
View File

@ -24,7 +24,7 @@ import (
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
)
func TestClient_Archive_ProtoRoundTrip(t *testing.T) {
func TestClientArchiveOptions_ProtoRoundTrip(t *testing.T) {
var diff string
fn := func(original gitserver.ArchiveOptions) bool {

144
internal/gitserver/commands.go
View File

@ -2191,13 +2191,29 @@ const (
ArchiveFormatTar ArchiveFormat = "tar"
)
// ArchiveReader streams back the file contents of an archived git repo.
func (c *clientImplementor) ArchiveReader(
ctx context.Context,
repo api.RepoName,
options ArchiveOptions,
) (_ io.ReadCloser, err error) {
// TODO: this does not capture the lifetime of the request because we return a reader
func ArchiveFormatFromProto(pf proto.ArchiveFormat) ArchiveFormat {
switch pf {
case proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP:
return ArchiveFormatZip
case proto.ArchiveFormat_ARCHIVE_FORMAT_TAR:
return ArchiveFormatTar
default:
return ""
}
}
func (f ArchiveFormat) ToProto() proto.ArchiveFormat {
switch f {
case ArchiveFormatZip:
return proto.ArchiveFormat_ARCHIVE_FORMAT_ZIP
case ArchiveFormatTar:
return proto.ArchiveFormat_ARCHIVE_FORMAT_TAR
default:
return proto.ArchiveFormat_ARCHIVE_FORMAT_UNSPECIFIED
}
}
func (c *clientImplementor) ArchiveReader(ctx context.Context, repo api.RepoName, options ArchiveOptions) (_ io.ReadCloser, err error) {
ctx, _, endObservation := c.operations.archiveReader.With(ctx, &err, observation.Args{
MetricLabelValues: []string{c.scope},
Attrs: append(
@ -2205,103 +2221,87 @@ func (c *clientImplementor) ArchiveReader(
options.Attrs()...,
),
})
defer endObservation(1, observation.Args{})
if authz.SubRepoEnabled(c.subRepoPermsChecker) {
if enabled, err := authz.SubRepoEnabledForRepo(ctx, c.subRepoPermsChecker, repo); err != nil {
return nil, errors.Wrap(err, "sub-repo permissions check:")
} else if enabled {
return nil, errors.New("archiveReader invoked for a repo with sub-repo permissions")
}
}
if ClientMocks.Archive != nil {
return ClientMocks.Archive(ctx, repo, options)
}
// Check that ctx is not expired.
if err := ctx.Err(); err != nil {
deadlineExceededCounter.Inc()
return nil, err
}
client, err := c.clientSource.ClientForRepo(ctx, repo)
if err != nil {
endObservation(1, observation.Args{})
return nil, err
}
req := options.ToProto(string(repo)) // HACK: ArchiveOptions doesn't have a repository here, so we have to add it ourselves.
req := options.ToProto(string(repo))
ctx, cancel := context.WithCancel(ctx)
stream, err := client.Archive(ctx, req)
cli, err := client.Archive(ctx, req)
if err != nil {
cancel()
endObservation(1, observation.Args{})
return nil, err
}
// first message from the gRPC stream needs to be read to check for errors before continuing
// to read the rest of the stream. If the first message is an error, we cancel the stream
// and return the error.
//
// This is necessary to provide parity between the REST and gRPC implementations of
// ArchiveReader. Users of cli.ArchiveReader may assume error handling occurs immediately,
// as is the case with the HTTP implementation where errors are returned as soon as the
// function returns. gRPC is asynchronous, so we have to start consuming messages from
// the stream to see any errors from the server. Reading the first message ensures we
// handle any errors synchronously, similar to the HTTP implementation.
firstMessage, firstError := stream.Recv()
if firstError != nil {
// Hack: The ArchiveReader.Read() implementation handles surfacing the
// any "revision not found" errors returned from the invoked git binary.
//
// In order to maintainparity with the HTTP API, we return this error in the ArchiveReader.Read() method
// instead of returning it immediately.
// We return early only if this isn't a revision not found error.
err := firstError
var cse *CommandStatusError
if !errors.As(err, &cse) || !isRevisionNotFound(cse.Stderr) {
// We start by reading the first message to early-exit on potential errors,
// ie. revision not found errors or invalid git command.
firstMessage, firstErr := cli.Recv()
if firstErr != nil {
if s, ok := status.FromError(firstErr); ok {
if s.Code() == codes.NotFound {
for _, d := range s.Details() {
switch d.(type) {
case *proto.FileNotFoundPayload:
cancel()
err = firstErr
endObservation(1, observation.Args{})
// We don't have a specific path here, so we return ErrNotExist instead of PathError.
return nil, os.ErrNotExist
}
}
}
}
if errors.HasType(firstErr, &gitdomain.RevisionNotFoundError{}) {
cancel()
err = firstErr
endObservation(1, observation.Args{})
return nil, err
}
}
firstMessageRead := false
// Create a reader to read from the gRPC stream.
firstRespRead := false
r := streamio.NewReader(func() ([]byte, error) {
// Check if we've read the first message yet. If not, read it and return.
if !firstMessageRead {
firstMessageRead = true
if firstError != nil {
return nil, firstError
if !firstRespRead {
firstRespRead = true
if firstErr != nil {
return nil, firstErr
}
return firstMessage.GetData(), nil
}
// Receive the next message from the stream.
msg, err := stream.Recv()
m, err := cli.Recv()
if err != nil {
return nil, err
}
// Return the data from the received message.
return msg.GetData(), nil
return m.GetData(), nil
})
return &archiveReader{
base: &readCloseWrapper{Reader: r, closeFn: cancel},
repo: repo,
spec: options.Treeish,
Reader: r,
cancel: cancel,
onClose: func() {
endObservation(1, observation.Args{})
},
}, nil
}
type archiveReader struct {
io.Reader
cancel context.CancelFunc
onClose func()
}
func (br *archiveReader) Close() error {
br.cancel()
br.onClose()
return nil
}
func addNameOnly(opt CommitsOptions, checker authz.SubRepoPermissionChecker) CommitsOptions {
if authz.SubRepoEnabled(checker) {
// If sub-repo permissions enabled, must fetch files modified w/ commits to determine if user has access to view this commit

190
internal/gitserver/commands_test.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"io/fs"
"math/rand"
"os"
"path/filepath"
"reflect"
@ -30,10 +31,17 @@ import (
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
// Generate a random archive format.
func (f ArchiveFormat) Generate(rand *rand.Rand, _ int) reflect.Value {
choices := []ArchiveFormat{ArchiveFormatZip, ArchiveFormatTar}
index := rand.Intn(len(choices))
return reflect.ValueOf(choices[index])
}
func TestParseShortLog(t *testing.T) {
tests := []struct {
name string
@ -2020,81 +2028,6 @@ func CommitsEqual(a, b *gitdomain.Commit) bool {
return reflect.DeepEqual(a, b)
}
func TestArchiveReaderForRepoWithSubRepoPermissions(t *testing.T) {
repoName := MakeGitRepository(t,
"echo abcd > file1",
"git add file1",
"git commit -m commit1",
)
const commitID = "3d689662de70f9e252d4f6f1d75284e23587d670"
checker := authz.NewMockSubRepoPermissionChecker()
checker.EnabledFunc.SetDefaultHook(func() bool {
return true
})
checker.EnabledForRepoFunc.SetDefaultHook(func(ctx context.Context, name api.RepoName) (bool, error) {
// sub-repo permissions are enabled only for repo with repoID = 1
return name == repoName, nil
})
ClientMocks.Archive = func(ctx context.Context, repo api.RepoName, opt ArchiveOptions) (io.ReadCloser, error) {
stringReader := strings.NewReader("1337")
return io.NopCloser(stringReader), nil
}
defer ResetClientMocks()
repo := &types.Repo{Name: repoName, ID: 1}
opts := ArchiveOptions{
Format: ArchiveFormatZip,
Treeish: commitID,
Pathspecs: []gitdomain.Pathspec{"."},
}
client := NewTestClient(t).WithChecker(checker)
if _, err := client.ArchiveReader(context.Background(), repo.Name, opts); err == nil {
t.Error("Error should not be null because ArchiveReader is invoked for a repo with sub-repo permissions")
}
}
func TestArchiveReaderForRepoWithoutSubRepoPermissions(t *testing.T) {
repoName := MakeGitRepository(t,
"echo abcd > file1",
"git add file1",
"git commit -m commit1",
)
const commitID = "3d689662de70f9e252d4f6f1d75284e23587d670"
checker := authz.NewMockSubRepoPermissionChecker()
checker.EnabledFunc.SetDefaultHook(func() bool {
return true
})
checker.EnabledForRepoFunc.SetDefaultHook(func(ctx context.Context, name api.RepoName) (bool, error) {
// sub-repo permissions are not present for repo with repoID = 1
return name != repoName, nil
})
ClientMocks.Archive = func(ctx context.Context, repo api.RepoName, opt ArchiveOptions) (io.ReadCloser, error) {
stringReader := strings.NewReader("1337")
return io.NopCloser(stringReader), nil
}
defer ResetClientMocks()
repo := &types.Repo{Name: repoName, ID: 1}
opts := ArchiveOptions{
Format: ArchiveFormatZip,
Treeish: commitID,
Pathspecs: []gitdomain.Pathspec{"."},
}
client := NewClient("test")
readCloser, err := client.ArchiveReader(context.Background(), repo.Name, opts)
if err != nil {
t.Error("Error should not be thrown because ArchiveReader is invoked for a repo without sub-repo permissions")
}
err = readCloser.Close()
if err != nil {
t.Error("Error during closing a reader")
}
}
func TestRepository_ListBranches(t *testing.T) {
ClientMocks.LocalGitserver = true
t.Cleanup(func() {
@ -2565,3 +2498,108 @@ func TestErrorMessageTruncateOutput(t *testing.T) {
}
})
}
func TestClient_ArchiveReader(t *testing.T) {
t.Run("firstChunk memoization", func(t *testing.T) {
source := NewTestClientSource(t, []string{"gitserver"}, func(o *TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
c := NewMockGitserverServiceClient()
rfc := NewMockGitserverService_ArchiveClient()
rfc.RecvFunc.PushReturn(&proto.ArchiveResponse{Data: []byte("part1\n")}, nil)
rfc.RecvFunc.PushReturn(&proto.ArchiveResponse{Data: []byte("part2\n")}, nil)
rfc.RecvFunc.PushReturn(nil, io.EOF)
c.ArchiveFunc.SetDefaultReturn(rfc, nil)
return c
}
})
c := NewTestClient(t).WithClientSource(source)
r, err := c.ArchiveReader(context.Background(), "repo", ArchiveOptions{Treeish: "deadbeef", Format: ArchiveFormatTar, Paths: []string{"file"}})
require.NoError(t, err)
content, err := io.ReadAll(r)
require.NoError(t, err)
require.NoError(t, r.Close())
require.Equal(t, "part1\npart2\n", string(content))
})
t.Run("firstChunk error memoization", func(t *testing.T) {
source := NewTestClientSource(t, []string{"gitserver"}, func(o *TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
c := NewMockGitserverServiceClient()
rfc := NewMockGitserverService_ArchiveClient()
rfc.RecvFunc.PushReturn(nil, io.EOF)
c.ArchiveFunc.SetDefaultReturn(rfc, nil)
return c
}
})
c := NewTestClient(t).WithClientSource(source)
r, err := c.ArchiveReader(context.Background(), "repo", ArchiveOptions{Treeish: "deadbeef", Format: ArchiveFormatTar, Paths: []string{"file"}})
require.NoError(t, err)
content, err := io.ReadAll(r)
require.NoError(t, err)
require.NoError(t, r.Close())
require.Equal(t, "", string(content))
})
t.Run("file not found errors are returned early", func(t *testing.T) {
source := NewTestClientSource(t, []string{"gitserver"}, func(o *TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
c := NewMockGitserverServiceClient()
rfc := NewMockGitserverService_ArchiveClient()
s, err := status.New(codes.NotFound, "not found").WithDetails(&proto.FileNotFoundPayload{})
require.NoError(t, err)
rfc.RecvFunc.PushReturn(nil, s.Err())
c.ArchiveFunc.SetDefaultReturn(rfc, nil)
return c
}
})
c := NewTestClient(t).WithClientSource(source)
_, err := c.ArchiveReader(context.Background(), "repo", ArchiveOptions{Treeish: "deadbeef", Format: ArchiveFormatTar, Paths: []string{"file"}})
require.Error(t, err)
require.True(t, os.IsNotExist(err))
})
t.Run("revision not found errors are returned early", func(t *testing.T) {
source := NewTestClientSource(t, []string{"gitserver"}, func(o *TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
c := NewMockGitserverServiceClient()
rfc := NewMockGitserverService_ArchiveClient()
s, err := status.New(codes.NotFound, "revision not found").WithDetails(&proto.RevisionNotFoundPayload{})
require.NoError(t, err)
rfc.RecvFunc.PushReturn(nil, s.Err())
c.ArchiveFunc.SetDefaultReturn(rfc, nil)
return c
}
})
c := NewTestClient(t).WithClientSource(source)
_, err := c.ArchiveReader(context.Background(), "repo", ArchiveOptions{Treeish: "deadbeef", Format: ArchiveFormatTar, Paths: []string{"file"}})
require.Error(t, err)
require.True(t, errors.HasType(err, &gitdomain.RevisionNotFoundError{}))
})
t.Run("empty archive", func(t *testing.T) {
source := NewTestClientSource(t, []string{"gitserver"}, func(o *TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
c := NewMockGitserverServiceClient()
rfc := NewMockGitserverService_ArchiveClient()
rfc.RecvFunc.PushReturn(nil, io.EOF)
c.ArchiveFunc.SetDefaultReturn(rfc, nil)
return c
}
})
c := NewTestClient(t).WithClientSource(source)
r, err := c.ArchiveReader(context.Background(), "repo", ArchiveOptions{Treeish: "deadbeef", Format: ArchiveFormatTar, Paths: []string{"file"}})
require.NoError(t, err)
content, err := io.ReadAll(r)
require.NoError(t, err)
require.Empty(t, content)
require.NoError(t, r.Close())
})
}

6
internal/gitserver/gitdomain/common.go
View File

@ -458,9 +458,3 @@ func (gs RefGlobs) Match(ref string) bool {
// Pathspec is a git term for a pattern that matches paths using glob-like syntax.
// https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec
type Pathspec string
// PathspecLiteral constructs a pathspec that matches a path without interpreting "*" or "?" as special
// characters.
//
// See: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-literal
func PathspecLiteral(s string) Pathspec { return Pathspec(":(literal)" + s) }

1753
internal/gitserver/mock.go
View File

File diff suppressed because it is too large Load Diff

1381
internal/gitserver/v1/gitserver.pb.go generated
View File

File diff suppressed because it is too large Load Diff

33
internal/gitserver/v1/gitserver.proto
View File

@ -25,6 +25,19 @@ service GitserverService {
rpc Search(SearchRequest) returns (stream SearchResponse) {
option idempotency_level = NO_SIDE_EFFECTS;
}
// Archive creates an archive for the given treeish in the given format.
// If paths are specified, only those paths are included in the archive.
//
// If subrepo permissions are enabled for the repo, no archive will be created
// for non-internal actors and an unimplemented error will be returned. We can
// currently not filter parts of the archive, so this would be considered leaking
// information.
//
// If the given treeish does not exist, an error with a RevisionNotFoundPayload
// is returned.
//
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a RepoNotFoundPayload in the details.
rpc Archive(ArchiveRequest) returns (stream ArchiveResponse) {
option idempotency_level = NO_SIDE_EFFECTS;
}
@ -510,17 +523,23 @@ message CommitMatch {
repeated string modified_files = 9;
}
enum ArchiveFormat {
ARCHIVE_FORMAT_UNSPECIFIED = 0;
ARCHIVE_FORMAT_ZIP = 1;
ARCHIVE_FORMAT_TAR = 2;
}
// ArchiveRequest is set of parameters for the Archive RPC.
message ArchiveRequest {
// repo is the name of the repo to be archived
// repo is the name of the repo to be archived.
string repo = 1;
// treeish is the tree or commit to produce an archive for
// treeish is the tree or commit to produce an archive for.
string treeish = 2;
// format is the format of the resulting archive (usually "tar" or "zip")
string format = 3;
// pathspecs is the list of pathspecs to include in the archive. If empty, all
// pathspecs are included.
repeated string pathspecs = 4;
// format is the format of the resulting archive (either ZIP or TAR).
ArchiveFormat format = 3;
// paths is the list of paths to include in the archive. If empty, all
// paths are included.
repeated string paths = 4;
}
// ArchiveResponse is the response from the Archive RPC that returns a chunk of

26
internal/gitserver/v1/gitserver_grpc.pb.go generated
View File

@ -58,6 +58,19 @@ type GitserverServiceClient interface {
IsRepoCloneable(ctx context.Context, in *IsRepoCloneableRequest, opts ...grpc.CallOption) (*IsRepoCloneableResponse, error)
ListGitolite(ctx context.Context, in *ListGitoliteRequest, opts ...grpc.CallOption) (*ListGitoliteResponse, error)
Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (GitserverService_SearchClient, error)
// Archive creates an archive for the given treeish in the given format.
// If paths are specified, only those paths are included in the archive.
//
// If subrepo permissions are enabled for the repo, no archive will be created
// for non-internal actors and an unimplemented error will be returned. We can
// currently not filter parts of the archive, so this would be considered leaking
// information.
//
// If the given treeish does not exist, an error with a RevisionNotFoundPayload
// is returned.
//
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a RepoNotFoundPayload in the details.
Archive(ctx context.Context, in *ArchiveRequest, opts ...grpc.CallOption) (GitserverService_ArchiveClient, error)
// Deprecated: Do not use.
P4Exec(ctx context.Context, in *P4ExecRequest, opts ...grpc.CallOption) (GitserverService_P4ExecClient, error)
@ -535,6 +548,19 @@ type GitserverServiceServer interface {
IsRepoCloneable(context.Context, *IsRepoCloneableRequest) (*IsRepoCloneableResponse, error)
ListGitolite(context.Context, *ListGitoliteRequest) (*ListGitoliteResponse, error)
Search(*SearchRequest, GitserverService_SearchServer) error
// Archive creates an archive for the given treeish in the given format.
// If paths are specified, only those paths are included in the archive.
//
// If subrepo permissions are enabled for the repo, no archive will be created
// for non-internal actors and an unimplemented error will be returned. We can
// currently not filter parts of the archive, so this would be considered leaking
// information.
//
// If the given treeish does not exist, an error with a RevisionNotFoundPayload
// is returned.
//
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a RepoNotFoundPayload in the details.
Archive(*ArchiveRequest, GitserverService_ArchiveServer) error
// Deprecated: Do not use.
P4Exec(*P4ExecRequest, GitserverService_P4ExecServer) error

2
mockgen.temp.yaml
View File

@ -137,6 +137,8 @@
interfaces:
- GitserverServiceClient
- GitserverService_ExecServer
- GitserverService_ArchiveServer
- GitserverService_ArchiveClient
- GitserverService_BlameServer
- GitserverService_BlameClient
- GitserverService_ReadFileServer