artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 17:11:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

gitserver: Migrate Blame to rpc call (#59851)

Browse Source 
This PR migrates the first more sophisticated call to a gRPC equivalent.

I had to generate quite a bunch of mocks and convert some things to interfaces to properly test this, so the diff looks scary but it's mostly generated code.
Positive: Found a few bugs in the new gitcli backend by implementing these tests :)

## Test plan

Added a ton of tests, tried blame manually.
This commit is contained in:
Erik Seliger 2024-01-28 06:35:42 +01:00 committed by GitHub
parent 207f05b955
commit 3e4a40bcd5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
47 changed files with 6714 additions and 1756 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/frontend/graphqlbackend/hunk.go
View File

@ -5,12 +5,13 @@ import (
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
)
type hunkResolver struct {
db database.DB
repo *RepositoryResolver
hunk *gitserver.Hunk
hunk *gitdomain.Hunk
}
func (r *hunkResolver) Author() signatureResolver {

8
cmd/frontend/internal/httpapi/stream_blame.go
View File

@ -62,6 +62,10 @@ func handleStreamBlame(logger log.Logger, db database.DB, gitserverClient gitser
})
if err != nil {
tr.SetError(err)
if errcode.IsUnauthorized(err) {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@ -167,8 +171,8 @@ func handleStreamBlame(logger log.Logger, db database.DB, gitserverClient gitser
type BlameHunkResponse struct {
api.CommitID `json:"commitID"`
StartLine int `json:"startLine"` // 1-indexed start line number
EndLine int `json:"endLine"` // 1-indexed end line number
StartLine uint32 `json:"startLine"` // 1-indexed start line number
EndLine uint32 `json:"endLine"` // 1-indexed end line number
Author gitdomain.Signature `json:"author"`
Message string `json:"message"`
Filename string `json:"filename"`

37
cmd/frontend/internal/httpapi/stream_blame_test.go
View File

@ -2,6 +2,7 @@ package httpapi
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
@ -23,8 +24,34 @@ import (
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func setupMockGSClient(t *testing.T, wantRev api.CommitID, returnErr error, hunks []*gitserver.Hunk) gitserver.Client {
hunkReader := gitserver.NewMockHunkReader(hunks, returnErr)
type mockHunkReader struct {
hunks []*gitdomain.Hunk
err error
}
func newMockHunkReader(hunks []*gitdomain.Hunk, err error) gitserver.HunkReader {
return &mockHunkReader{
hunks: hunks,
err: err,
}
}
func (mh *mockHunkReader) Read() (*gitdomain.Hunk, error) {
if mh.err != nil {
return nil, mh.err
}
if len(mh.hunks) > 0 {
next := mh.hunks[0]
mh.hunks = mh.hunks[1:]
return next, nil
}
return nil, io.EOF
}
func (mh *mockHunkReader) Close() error { return nil }
func setupMockGSClient(t *testing.T, wantRev api.CommitID, returnErr error, hunks []*gitdomain.Hunk) gitserver.Client {
hunkReader := newMockHunkReader(hunks, returnErr)
gsClient := gitserver.NewMockClient()
gsClient.GetCommitFunc.SetDefaultHook(
func(_ context.Context,
@ -54,7 +81,7 @@ func setupMockGSClient(t *testing.T, wantRev api.CommitID, returnErr error, hunk
func TestStreamBlame(t *testing.T) {
logger, _ := logtest.Captured(t)
hunks := []*gitserver.Hunk{
hunks := []*gitdomain.Hunk{
{
StartLine: 1,
EndLine: 2,
@ -195,7 +222,7 @@ func TestStreamBlame(t *testing.T) {
"Repo": "github.com/bob/foo",
"path": "foo.c",
})
gsClient := setupMockGSClient(t, "efgh", nil, []*gitserver.Hunk{
gsClient := setupMockGSClient(t, "efgh", nil, []*gitdomain.Hunk{
{
StartLine: 1,
EndLine: 2,
@ -244,7 +271,7 @@ func TestStreamBlame(t *testing.T) {
"Repo": "foo",
"path": "foo.c",
})
gsClient := setupMockGSClient(t, "efgh", nil, []*gitserver.Hunk{
gsClient := setupMockGSClient(t, "efgh", nil, []*gitdomain.Hunk{
{
StartLine: 1,
EndLine: 2,

9
cmd/gitserver/internal/BUILD.bazel
View File

@ -39,6 +39,7 @@ go_library(
"//cmd/gitserver/internal/vcssyncer",
"//internal/actor",
"//internal/api",
"//internal/authz",
"//internal/conf",
"//internal/database",
"//internal/database/dbutil",
@ -71,6 +72,7 @@ go_library(
"//internal/wrexec",
"//lib/errors",
"//lib/gitservice",
"//lib/pointers",
"@com_github_mxk_go_flowrate//flowrate",
"@com_github_prometheus_client_golang//prometheus",
"@com_github_prometheus_client_golang//prometheus/promauto",
@ -92,8 +94,10 @@ go_test(
"cleanup_test.go",
"list_gitolite_test.go",
"main_test.go",
"mocks_test.go",
"p4exec_test.go",
"repo_info_test.go",
"server_grpc_test.go",
"server_test.go",
],
embed = [":internal"],
@ -114,12 +118,14 @@ go_test(
"//cmd/gitserver/internal/vcssyncer",
"//internal/actor",
"//internal/api",
"//internal/authz",
"//internal/conf",
"//internal/database",
"//internal/database/dbmocks",
"//internal/database/dbtest",
"//internal/extsvc/gitolite",
"//internal/gitserver",
"//internal/gitserver/gitdomain",
"//internal/gitserver/protocol",
"//internal/gitserver/v1:gitserver",
"//internal/grpc",
@ -127,12 +133,14 @@ go_test(
"//internal/limiter",
"//internal/observation",
"//internal/ratelimit",
"//internal/trace",
"//internal/types",
"//internal/vcs",
"//internal/wrexec",
"//lib/errors",
"//lib/pointers",
"//schema",
"@com_github_derision_test_go_mockgen//testutil/assert",
"@com_github_google_go_cmp//cmp",
"@com_github_google_go_cmp//cmp/cmpopts",
"@com_github_sourcegraph_log//:log",
@ -141,6 +149,7 @@ go_test(
"@com_github_stretchr_testify//require",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
"@org_golang_google_protobuf//types/known/timestamppb",
"@org_golang_x_sync//semaphore",
"@org_golang_x_time//rate",
],

2
cmd/gitserver/internal/batchlog.go
View File

@ -16,7 +16,7 @@ import (
// Note: The BatchLog endpoint has been deprecated. This file shall be removed after
// the 5.3 release has been cut.
func (s *Server) batchGitLogInstrumentedHandler(ctx context.Context, req *proto.BatchLogRequest) (resp *proto.BatchLogResponse, err error) {
func (s *Server) BatchGitLogInstrumentedHandler(ctx context.Context, req *proto.BatchLogRequest) (resp *proto.BatchLogResponse, err error) {
// Perform requests in each repository in the input batch. We perform these commands
// concurrently, but only allow for so many commands to be in-flight at a time so that
// we don't overwhelm a shard with either a large request or too many concurrent batch

2
cmd/gitserver/internal/cleanup.go
View File

@ -288,7 +288,7 @@ func cleanupRepos(
// Record the number of repos that should not belong on this instance and
// remove up to SRC_WRONG_SHARD_DELETE_LIMIT in a single Janitor run.
name := gitserverfs.RepoNameFromDir(reposDir, dir)
addr := addrForRepo(ctx, name, gitServerAddrs)
addr := gitServerAddrs.AddrForRepo(ctx, name)
if hostnameMatch(shardID, addr) {
return false, nil

8
cmd/gitserver/internal/clone.go
View File

@ -11,20 +11,20 @@ import (
"github.com/sourcegraph/sourcegraph/internal/gitserver/protocol"
)
// maybeStartClone checks if a given repository is cloned on disk. If not, it starts
// MaybeStartClone checks if a given repository is cloned on disk. If not, it starts
// cloning the repository in the background and returns a NotFound error, if no current
// clone operation is running for that repo yet. If it is already cloning, a NotFound
// error with CloneInProgress: true is returned.
// Note: If disableAutoGitUpdates is set in the site config, no operation is taken and
// a NotFound error is returned.
func (s *Server) maybeStartClone(ctx context.Context, logger log.Logger, repo api.RepoName) (notFound *protocol.NotFoundPayload, cloned bool) {
func (s *Server) MaybeStartClone(ctx context.Context, repo api.RepoName) (notFound *protocol.NotFoundPayload, cloned bool) {
dir := gitserverfs.RepoDirFromName(s.ReposDir, repo)
if repoCloned(dir) {
return nil, true
}
if conf.Get().DisableAutoGitUpdates {
logger.Debug("not cloning on demand as DisableAutoGitUpdates is set")
s.Logger.Debug("not cloning on demand as DisableAutoGitUpdates is set")
return &protocol.NotFoundPayload{}, false
}
@ -38,7 +38,7 @@ func (s *Server) maybeStartClone(ctx context.Context, logger log.Logger, repo ap
cloneProgress, err := s.CloneRepo(ctx, repo, CloneOptions{})
if err != nil {
logger.Debug("error starting repo clone", log.String("repo", string(repo)), log.Error(err))
s.Logger.Debug("error starting repo clone", log.String("repo", string(repo)), log.Error(err))
return &protocol.NotFoundPayload{CloneInProgress: false}, false
}

3
cmd/gitserver/internal/git/gitcli/BUILD.bazel
View File

@ -4,6 +4,7 @@ load("//dev:go_defs.bzl", "go_test")
go_library(
name = "gitcli",
srcs = [
"blame.go",
"clibackend.go",
"config.go",
"exec.go",
@ -35,6 +36,7 @@ go_library(
go_test(
name = "gitcli_test",
srcs = [
"blame_test.go",
"config_test.go",
"exec_test.go",
"mergebase_test.go",
@ -43,6 +45,7 @@ go_test(
embed = [":gitcli"],
deps = [
"//cmd/gitserver/internal/common",
"//cmd/gitserver/internal/git",
"//internal/api",
"//internal/gitserver",
"//internal/gitserver/gitdomain",

98
internal/gitserver/stream_hunks.go → cmd/gitserver/internal/git/gitcli/blame.go
View File

@ -1,42 +1,84 @@
package gitserver
package gitcli
import (
"bufio"
"context"
"fmt"
"io"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func (g *gitCLIBackend) Blame(ctx context.Context, path string, opt git.BlameOptions) (git.BlameHunkReader, error) {
if err := checkSpecArgSafety(string(opt.NewestCommit)); err != nil {
return nil, err
}
cmd, cancel, err := g.gitCommand(ctx, buildBlameArgs(path, opt)...)
if err != nil {
cancel()
return nil, err
}
r, err := g.runGitCommand(ctx, cmd)
if err != nil {
cancel()
return nil, err
}
return newBlameHunkReader(r, cancel), nil
}
func buildBlameArgs(path string, opt git.BlameOptions) []string {
args := []string{"blame", "--porcelain", "--incremental"}
if opt.IgnoreWhitespace {
args = append(args, "-w")
}
if opt.StartLine != 0 || opt.EndLine != 0 {
args = append(args, fmt.Sprintf("-L%d,%d", opt.StartLine, opt.EndLine))
}
if opt.NewestCommit != "" {
args = append(args, string(opt.NewestCommit))
}
args = append(args, "--", filepath.ToSlash(path))
return args
}
// blameHunkReader enables to read hunks from an io.Reader.
type blameHunkReader struct {
rc io.ReadCloser
sc *bufio.Scanner
rc io.ReadCloser
sc *bufio.Scanner
onClose func()
cur *Hunk
cur *gitdomain.Hunk
// commits stores previously seen commits, so new hunks
// whose annotations are abbreviated by git can still be
// filled by the correct data even if the hunk entry doesn't
// repeat them.
commits map[api.CommitID]*Hunk
commits map[api.CommitID]*gitdomain.Hunk
}
func newBlameHunkReader(rc io.ReadCloser) HunkReader {
func newBlameHunkReader(rc io.ReadCloser, onClose func()) git.BlameHunkReader {
return &blameHunkReader{
rc: rc,
sc: bufio.NewScanner(rc),
commits: make(map[api.CommitID]*Hunk),
commits: make(map[api.CommitID]*gitdomain.Hunk),
onClose: onClose,
}
}
// Read returns a slice of hunks, along with a done boolean indicating if there
// is more to read. After the last hunk has been returned, Read() will return
// an io.EOF error on success.
func (br *blameHunkReader) Read() (_ *Hunk, err error) {
func (br *blameHunkReader) Read() (_ *gitdomain.Hunk, err error) {
for {
// Do we have more to read?
if !br.sc.Scan() {
@ -101,12 +143,14 @@ func (br *blameHunkReader) Read() (_ *Hunk, err error) {
}
func (br *blameHunkReader) Close() error {
return br.rc.Close()
err := br.rc.Close()
br.onClose()
return err
}
// parseEntry turns a `67b7b725a7ff913da520b997d71c840230351e30 10 20 1` line from
// git blame into a hunk.
func parseEntry(rev string, content string) (*Hunk, error) {
func parseEntry(rev string, content string) (*gitdomain.Hunk, error) {
fields := strings.Split(content, " ")
if len(fields) != 3 {
return nil, errors.Errorf("Expected at least 4 parts to hunkHeader, but got: '%s %s'", rev, content)
@ -121,16 +165,16 @@ func parseEntry(rev string, content string) (*Hunk, error) {
return nil, err
}
return &Hunk{
return &gitdomain.Hunk{
CommitID: api.CommitID(rev),
StartLine: resultLine,
EndLine: resultLine + numLines,
StartLine: uint32(resultLine),
EndLine: uint32(resultLine + numLines),
}, nil
}
// parseExtra updates a hunk with data parsed from the other annotations such as `author ...`,
// `summary ...`.
func parseExtra(hunk *Hunk, annotation string, content string) (ok bool, err error) {
func parseExtra(hunk *gitdomain.Hunk, annotation string, content string) (ok bool, err error) {
ok = true
switch annotation {
case "author":
@ -172,29 +216,3 @@ func splitLine(line string) (annotation string, content string) {
}
return line, ""
}
type mockHunkReader struct {
hunks []*Hunk
err error
}
func NewMockHunkReader(hunks []*Hunk, err error) HunkReader {
return &mockHunkReader{
hunks: hunks,
err: err,
}
}
func (mh *mockHunkReader) Read() (*Hunk, error) {
if mh.err != nil {
return nil, mh.err
}
if len(mh.hunks) > 0 {
next := mh.hunks[0]
mh.hunks = mh.hunks[1:]
return next, nil
}
return nil, io.EOF
}
func (mh *mockHunkReader) Close() error { return nil }

451
cmd/gitserver/internal/git/gitcli/blame_test.go Normal file
View File

@ -0,0 +1,451 @@
package gitcli
import (
"context"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/log/logtest"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/common"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/wrexec"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func TestGitCLIBackend_Blame(t *testing.T) {
rcf := wrexec.NewNoOpRecordingCommandFactory()
reposDir := t.TempDir()
// Make a new bare repo on disk.
p := filepath.Join(reposDir, "repo")
require.NoError(t, os.MkdirAll(p, os.ModePerm))
dir := common.GitDir(filepath.Join(p, ".git"))
// Prepare repo state:
cmds := []string{
"echo 'hello\nworld\nfrom\nblame\n' > foo.txt",
"git add foo.txt",
"git commit -m foo --author='Foo Author <foo@sourcegraph.com>'",
// Add a second commit with a different author.
"echo 'hello\nworld\nfrom\nthe best blame\n' > foo.txt",
"git add foo.txt",
"git commit -m bar --author='Bar Author <bar@sourcegraph.com>'",
// Promote the repo to a bare repo.
"git config --bool core.bare true",
}
for _, cmd := range append([]string{"git init --initial-branch=master ."}, cmds...) {
out, err := gitserver.CreateGitCommand(p, "bash", "-c", cmd).CombinedOutput()
if err != nil {
t.Fatalf("Failed to run git command %v. Output was:\n\n%s", cmd, out)
}
}
backend := NewBackend(logtest.Scoped(t), rcf, dir, "repo")
ctx := context.Background()
t.Run("bad input", func(t *testing.T) {
// Bad commit triggers error.
_, err := backend.Blame(ctx, "foo.txt", git.BlameOptions{NewestCommit: "-very badarg"})
require.Error(t, err)
})
t.Run("stream hunks", func(t *testing.T) {
// Verify that the blame output is correct and that the hunk reader correctly
// terminates.
hr, err := backend.Blame(ctx, "foo.txt", git.BlameOptions{})
require.NoError(t, err)
h, err := hr.Read()
require.NoError(t, err)
require.Equal(t, &gitdomain.Hunk{
StartLine: 4,
EndLine: 5,
CommitID: "53e63d6dd6e61a58369bbc637b0ead2ee58d993c",
Author: gitdomain.Signature{
Name: "Bar Author",
Email: "bar@sourcegraph.com",
Date: h.Author.Date, // Hard to compare.
},
Message: "bar",
Filename: "foo.txt",
}, h)
h, err = hr.Read()
require.NoError(t, err)
require.Equal(t, &gitdomain.Hunk{
StartLine: 1,
EndLine: 4,
CommitID: "51f8be07ed2090b76e77b096c9d0737fc8ac70f4",
Author: gitdomain.Signature{
Name: "Foo Author",
Email: "foo@sourcegraph.com",
Date: h.Author.Date, // Hard to compare.
},
Message: "foo",
Filename: "foo.txt",
}, h)
h, err = hr.Read()
require.NoError(t, err)
require.Equal(t, &gitdomain.Hunk{
StartLine: 5,
EndLine: 6,
CommitID: "51f8be07ed2090b76e77b096c9d0737fc8ac70f4",
Author: gitdomain.Signature{
Name: "Foo Author",
Email: "foo@sourcegraph.com",
Date: h.Author.Date, // Hard to compare.
},
Message: "foo",
Filename: "foo.txt",
}, h)
_, err = hr.Read()
require.Equal(t, io.EOF, err)
require.NoError(t, hr.Close())
})
// Verify that if the context is canceled, the hunk reader returns an error.
t.Run("context cancelation", func(t *testing.T) {
ctx, cancel := context.WithCancel(ctx)
t.Cleanup(cancel)
hr, err := backend.Blame(ctx, "foo.txt", git.BlameOptions{})
require.NoError(t, err)
cancel()
_, err = hr.Read()
require.Error(t, err)
require.True(t, errors.Is(err, context.Canceled), "unexpected error: %v", err)
require.NoError(t, hr.Close())
})
}
func TestBuildBlameArgs(t *testing.T) {
path := "foo.txt"
t.Run("default options", func(t *testing.T) {
want := []string{"blame", "--porcelain", "--incremental", "--", "foo.txt"}
opt := git.BlameOptions{}
got := buildBlameArgs(path, opt)
if !equalSlice(got, want) {
t.Errorf("unexpected args:\ngot: %v\nwant: %v", got, want)
}
})
t.Run("with ignore whitespace", func(t *testing.T) {
want := []string{"blame", "--porcelain", "--incremental", "-w", "--", "foo.txt"}
opt := git.BlameOptions{IgnoreWhitespace: true}
got := buildBlameArgs(path, opt)
if !equalSlice(got, want) {
t.Errorf("unexpected args:\ngot: %v\nwant: %v", got, want)
}
})
t.Run("with line range", func(t *testing.T) {
want := []string{"blame", "--porcelain", "--incremental", "-L5,10", "--", "foo.txt"}
opt := git.BlameOptions{StartLine: 5, EndLine: 10}
got := buildBlameArgs(path, opt)
if !equalSlice(got, want) {
t.Errorf("unexpected args:\ngot: %v\nwant: %v", got, want)
}
})
t.Run("with commit", func(t *testing.T) {
want := []string{"blame", "--porcelain", "--incremental", "abc123", "--", "foo.txt"}
opt := git.BlameOptions{NewestCommit: api.CommitID("abc123")}
got := buildBlameArgs(path, opt)
if !equalSlice(got, want) {
t.Errorf("unexpected args:\ngot: %v\nwant: %v", got, want)
}
})
}
func equalSlice(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// testGitBlameOutputIncremental is produced by running
//
// git blame -w --porcelain release.sh
//
// `sourcegraph/src-cli`
var testGitBlameOutputIncremental = `8a75c6f8b4cbe2a2f3c8be0f2c50bc766499f498 15 15 1
author Adam Harvey
author-mail <adam@adamharvey.name>
author-time 1660860583
author-tz -0700
committer GitHub
committer-mail <noreply@github.com>
committer-time 1660860583
committer-tz +0000
summary release.sh: allow -rc.X suffixes (#829)
previous e6e03e850770dd0ba745f0fa4b23127e9d72ad30 release.sh
filename release.sh
fbb98e0b7ff0752798463d9f49d922858a4188f6 5 5 10
author Adam Harvey
author-mail <aharvey@sourcegraph.com>
author-time 1602630694
author-tz -0700
committer GitHub
committer-mail <noreply@github.com>
committer-time 1602630694
committer-tz -0700
summary release: add a prompt about DEVELOPMENT.md (#349)
previous 18f59760f4260518c29f0f07056245ed5d1d0f08 release.sh
filename release.sh
67b7b725a7ff913da520b997d71c840230351e30 10 20 1
author Thorsten Ball
author-mail <mrnugget@gmail.com>
author-time 1600334460
author-tz +0200
committer Thorsten Ball
committer-mail <mrnugget@gmail.com>
committer-time 1600334460
committer-tz +0200
summary Fix goreleaser GitHub action setup and release script
previous 6e931cc9745502184ce32d48b01f9a8706a4dfe8 release.sh
filename release.sh
67b7b725a7ff913da520b997d71c840230351e30 12 22 2
previous 6e931cc9745502184ce32d48b01f9a8706a4dfe8 release.sh
filename release.sh
3f61310114082d6179c23f75950b88d1842fe2de 1 1 4
author Thorsten Ball
author-mail <mrnugget@gmail.com>
author-time 1592827635
author-tz +0200
committer GitHub
committer-mail <noreply@github.com>
committer-time 1592827635
committer-tz +0200
summary Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)
previous ec809e79094cbcd05825446ee14c6d072466a0b7 release.sh
filename release.sh
3f61310114082d6179c23f75950b88d1842fe2de 6 16 4
previous ec809e79094cbcd05825446ee14c6d072466a0b7 release.sh
filename release.sh
3f61310114082d6179c23f75950b88d1842fe2de 10 21 1
previous ec809e79094cbcd05825446ee14c6d072466a0b7 release.sh
filename release.sh
`
// This test-data includes the boundary keyword, which is not present in the previous one.
var testGitBlameOutputIncremental2 = `bbca6551549492486ca1b0f8dee45553dd6aa6d7 16 16 1
author French Ben
author-mail <frenchben@docker.com>
author-time 1517407262
author-tz +0100
committer French Ben
committer-mail <frenchben@docker.com>
committer-time 1517407262
committer-tz +0100
summary Update error output to be clean
previous b7773ae218740a7be65057fc60b366a49b538a44 format.go
filename format.go
bbca6551549492486ca1b0f8dee45553dd6aa6d7 25 25 2
previous b7773ae218740a7be65057fc60b366a49b538a44 format.go
filename format.go
2c87fda17de1def6ea288141b8e7600b888e535b 15 15 1
author David Tolnay
author-mail <dtolnay@gmail.com>
author-time 1478451741
author-tz -0800
committer David Tolnay
committer-mail <dtolnay@gmail.com>
committer-time 1478451741
committer-tz -0800
summary Singular message for a single error
previous 8c5f0ad9360406a3807ce7de6bc73269a91a6e51 format.go
filename format.go
2c87fda17de1def6ea288141b8e7600b888e535b 17 17 2
previous 8c5f0ad9360406a3807ce7de6bc73269a91a6e51 format.go
filename format.go
31fee45604949934710ada68f0b307c4726fb4e8 1 1 14
author Mitchell Hashimoto
author-mail <mitchell.hashimoto@gmail.com>
author-time 1418673320
author-tz -0800
committer Mitchell Hashimoto
committer-mail <mitchell.hashimoto@gmail.com>
committer-time 1418673320
committer-tz -0800
summary Initial commit
boundary
filename format.go
31fee45604949934710ada68f0b307c4726fb4e8 15 19 6
filename format.go
31fee45604949934710ada68f0b307c4726fb4e8 23 27 1
filename format.go
`
var testGitBlameOutputHunks = []*gitdomain.Hunk{
{
StartLine: 1, EndLine: 5, StartByte: 0, EndByte: 41,
CommitID: "3f61310114082d6179c23f75950b88d1842fe2de",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: mustParseTime(time.RFC3339, "2020-06-22T12:07:15Z"),
},
Message: "Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)",
Filename: "release.sh",
},
{
StartLine: 5, EndLine: 15, StartByte: 41, EndByte: 249,
CommitID: "fbb98e0b7ff0752798463d9f49d922858a4188f6",
Author: gitdomain.Signature{
Name: "Adam Harvey",
Email: "aharvey@sourcegraph.com",
Date: mustParseTime(time.RFC3339, "2020-10-13T23:11:34Z"),
},
Message: "release: add a prompt about DEVELOPMENT.md (#349)",
Filename: "release.sh",
},
{
StartLine: 15, EndLine: 16, StartByte: 249, EndByte: 328,
CommitID: "8a75c6f8b4cbe2a2f3c8be0f2c50bc766499f498",
Author: gitdomain.Signature{
Name: "Adam Harvey",
Email: "adam@adamharvey.name",
Date: mustParseTime(time.RFC3339, "2022-08-18T22:09:43Z"),
},
Message: "release.sh: allow -rc.X suffixes (#829)",
Filename: "release.sh",
},
{
StartLine: 16, EndLine: 20, StartByte: 328, EndByte: 394,
CommitID: "3f61310114082d6179c23f75950b88d1842fe2de",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: mustParseTime(time.RFC3339, "2020-06-22T12:07:15Z"),
},
Message: "Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)",
Filename: "release.sh",
},
{
StartLine: 20, EndLine: 21, StartByte: 394, EndByte: 504,
CommitID: "67b7b725a7ff913da520b997d71c840230351e30",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: mustParseTime(time.RFC3339, "2020-09-17T09:21:00Z"),
},
Message: "Fix goreleaser GitHub action setup and release script",
Filename: "release.sh",
},
{
StartLine: 21, EndLine: 22, StartByte: 504, EndByte: 553,
CommitID: "3f61310114082d6179c23f75950b88d1842fe2de",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: mustParseTime(time.RFC3339, "2020-06-22T12:07:15Z"),
},
Message: "Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)",
Filename: "release.sh",
},
{
StartLine: 22, EndLine: 24, StartByte: 553, EndByte: 695,
CommitID: "67b7b725a7ff913da520b997d71c840230351e30",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: mustParseTime(time.RFC3339, "2020-09-17T09:21:00Z"),
},
Message: "Fix goreleaser GitHub action setup and release script",
Filename: "release.sh",
},
}
func TestBlameHunkReader(t *testing.T) {
t.Run("OK matching hunks", func(t *testing.T) {
rc := io.NopCloser(strings.NewReader(testGitBlameOutputIncremental))
reader := newBlameHunkReader(rc, func() {})
defer reader.Close()
hunks := []*gitdomain.Hunk{}
for {
hunk, err := reader.Read()
if errors.Is(err, io.EOF) {
break
} else if err != nil {
t.Fatalf("blameHunkReader.Read failed: %s", err)
}
hunks = append(hunks, hunk)
}
sortFn := func(x []*gitdomain.Hunk) func(i, j int) bool {
return func(i, j int) bool {
return x[i].Author.Date.After(x[j].Author.Date)
}
}
// We're not giving back bytes, as the output of --incremental only gives back annotations.
expectedHunks := make([]*gitdomain.Hunk, 0, len(testGitBlameOutputHunks))
for _, h := range testGitBlameOutputHunks {
dup := *h
dup.EndByte = 0
dup.StartByte = 0
expectedHunks = append(expectedHunks, &dup)
}
// Sort expected hunks by the most recent first, as --incremental does.
sort.SliceStable(expectedHunks, sortFn(expectedHunks))
if d := cmp.Diff(expectedHunks, hunks); d != "" {
t.Fatalf("unexpected hunks (-want, +got):\n%s", d)
}
})
t.Run("OK parsing hunks", func(t *testing.T) {
rc := io.NopCloser(strings.NewReader(testGitBlameOutputIncremental2))
reader := newBlameHunkReader(rc, func() {})
defer reader.Close()
for {
_, err := reader.Read()
if errors.Is(err, io.EOF) {
break
} else if err != nil {
t.Fatalf("blameHunkReader.Read failed: %s", err)
}
}
})
}
func mustParseTime(layout, value string) time.Time {
tm, err := time.Parse(layout, value)
if err != nil {
panic(err.Error())
}
return tm
}

21
cmd/gitserver/internal/git/gitcli/clibackend.go
View File

@ -41,7 +41,11 @@ type gitCLIBackend struct {
repoName api.RepoName
}
func commandFailedError(err error, cmd wrexec.Cmder, stderr []byte) error {
func commandFailedError(ctx context.Context, err error, cmd wrexec.Cmder, stderr []byte) error {
if ctx.Err() != nil {
return ctx.Err()
}
exitStatus := executil.UnsetExitStatus
if cmd.Unwrap().ProcessState != nil { // is nil if process failed to start
exitStatus = cmd.Unwrap().ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
@ -84,7 +88,7 @@ func (g *gitCLIBackend) gitCommand(ctx context.Context, args ...string) (wrexec.
return nil, cancel, ErrBadGitCommand
}
cmd := exec.Command("git", args...)
cmd := exec.CommandContext(ctx, "git", args...)
g.dir.Set(cmd)
return g.rcf.WrapWithRepoName(ctx, g.logger, g.repoName, cmd), cancel, nil
@ -92,6 +96,7 @@ func (g *gitCLIBackend) gitCommand(ctx context.Context, args ...string) (wrexec.
const maxStderrCapture = 1024
// make sure to pass the same context in here as was passed to gitCommand.
func (g *gitCLIBackend) runGitCommand(ctx context.Context, cmd wrexec.Cmder) (io.ReadCloser, error) {
// Set up a limited buffer to capture stderr for error reporting.
stderrBuf := bytes.NewBuffer(make([]byte, 0, maxStderrCapture))
@ -127,6 +132,7 @@ type cmdReader struct {
logger log.Logger
git git.GitBackend
repoName api.RepoName
closed bool
}
func (rc *cmdReader) Read(p []byte) (n int, err error) {
@ -134,12 +140,13 @@ func (rc *cmdReader) Read(p []byte) (n int, err error) {
writtenN, writeErr := rc.buf.Write(p[:n])
if err == io.EOF {
rc.ReadCloser.Close()
rc.closed = true
if err := rc.cmd.Wait(); err != nil {
if checkMaybeCorruptRepo(rc.ctx, rc.logger, rc.git, rc.repoName, rc.stderr.String()) {
return n, common.ErrRepoCorrupted{Reason: rc.stderr.String()}
}
return n, commandFailedError(err, rc.cmd, rc.stderr.Bytes())
return n, commandFailedError(rc.ctx, err, rc.cmd, rc.stderr.Bytes())
}
}
if err == nil && writeErr != nil {
@ -148,6 +155,14 @@ func (rc *cmdReader) Read(p []byte) (n int, err error) {
return n, err
}
func (rc *cmdReader) Close() error {
if rc.closed {
return nil
}
return rc.ReadCloser.Close()
}
// limitWriter is a io.Writer that writes to an W but discards after N bytes.
type limitWriter struct {
W io.Writer // underling writer

21
cmd/gitserver/internal/git/gitcli/exec.go
View File

@ -17,12 +17,29 @@ import (
func (g *gitCLIBackend) Exec(ctx context.Context, args ...string) (io.ReadCloser, error) {
cmd, cancel, err := g.gitCommand(ctx, args...)
defer cancel()
if err != nil {
cancel()
return nil, err
}
return g.runGitCommand(ctx, cmd)
r, err := g.runGitCommand(ctx, cmd)
if err != nil {
cancel()
return nil, err
}
return &cancelingCloser{ReadCloser: r, cancel: cancel}, nil
}
type cancelingCloser struct {
io.ReadCloser
cancel context.CancelFunc
}
func (c *cancelingCloser) Close() error {
err := c.ReadCloser.Close()
c.cancel()
return err
}
var (

21
cmd/gitserver/internal/git/iface.go
View File

@ -22,6 +22,10 @@ type GitBackend interface {
// Returns an empty string and no error if no common merge-base was found.
MergeBase(ctx context.Context, baseRevspec, headRevspec string) (api.CommitID, error)
// Blame returns a reader for the blame info of the given path.
// BlameHunkReader must always be closed.
Blame(ctx context.Context, path string, opt BlameOptions) (BlameHunkReader, error)
// Exec is a temporary helper to run arbitrary git commands from the exec endpoint.
// No new usages of it should be introduced and once the migration is done we will
// remove this method.
@ -39,3 +43,20 @@ type GitConfigBackend interface {
// no error is returned.
Unset(ctx context.Context, key string) error
}
// BlameOptions are options for git blame.
type BlameOptions struct {
NewestCommit api.CommitID
IgnoreWhitespace bool
// 1-indexed start line (or 0 for beginning of file)
StartLine int
// 1-indexed end line (or 0 for end of file)
EndLine int
}
// BlameHunkReader is a reader for git blame hunks.
type BlameHunkReader interface {
// Consume the next hunk. io.EOF is returned at the end of the stream.
Read() (*gitdomain.Hunk, error)
Close() error
}

389
cmd/gitserver/internal/git/mock.go
View File

@ -15,11 +15,277 @@ import (
gitdomain "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
)
// MockBlameHunkReader is a mock implementation of the BlameHunkReader
// interface (from the package
// github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git) used for
// unit testing.
type MockBlameHunkReader struct {
// CloseFunc is an instance of a mock function object controlling the
// behavior of the method Close.
CloseFunc *BlameHunkReaderCloseFunc
// ReadFunc is an instance of a mock function object controlling the
// behavior of the method Read.
ReadFunc *BlameHunkReaderReadFunc
}
// NewMockBlameHunkReader creates a new mock of the BlameHunkReader
// interface. All methods return zero values for all results, unless
// overwritten.
func NewMockBlameHunkReader() *MockBlameHunkReader {
return &MockBlameHunkReader{
CloseFunc: &BlameHunkReaderCloseFunc{
defaultHook: func() (r0 error) {
return
},
},
ReadFunc: &BlameHunkReaderReadFunc{
defaultHook: func() (r0 *gitdomain.Hunk, r1 error) {
return
},
},
}
}
// NewStrictMockBlameHunkReader creates a new mock of the BlameHunkReader
// interface. All methods panic on invocation, unless overwritten.
func NewStrictMockBlameHunkReader() *MockBlameHunkReader {
return &MockBlameHunkReader{
CloseFunc: &BlameHunkReaderCloseFunc{
defaultHook: func() error {
panic("unexpected invocation of MockBlameHunkReader.Close")
},
},
ReadFunc: &BlameHunkReaderReadFunc{
defaultHook: func() (*gitdomain.Hunk, error) {
panic("unexpected invocation of MockBlameHunkReader.Read")
},
},
}
}
// NewMockBlameHunkReaderFrom creates a new mock of the MockBlameHunkReader
// interface. All methods delegate to the given implementation, unless
// overwritten.
func NewMockBlameHunkReaderFrom(i BlameHunkReader) *MockBlameHunkReader {
return &MockBlameHunkReader{
CloseFunc: &BlameHunkReaderCloseFunc{
defaultHook: i.Close,
},
ReadFunc: &BlameHunkReaderReadFunc{
defaultHook: i.Read,
},
}
}
// BlameHunkReaderCloseFunc describes the behavior when the Close method of
// the parent MockBlameHunkReader instance is invoked.
type BlameHunkReaderCloseFunc struct {
defaultHook func() error
hooks []func() error
history []BlameHunkReaderCloseFuncCall
mutex sync.Mutex
}
// Close delegates to the next hook function in the queue and stores the
// parameter and result values of this invocation.
func (m *MockBlameHunkReader) Close() error {
r0 := m.CloseFunc.nextHook()()
m.CloseFunc.appendCall(BlameHunkReaderCloseFuncCall{r0})
return r0
}
// SetDefaultHook sets function that is called when the Close method of the
// parent MockBlameHunkReader instance is invoked and the hook queue is
// empty.
func (f *BlameHunkReaderCloseFunc) SetDefaultHook(hook func() error) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// Close method of the parent MockBlameHunkReader instance invokes the hook
// at the front of the queue and discards it. After the queue is empty, the
// default hook function is invoked for any future action.
func (f *BlameHunkReaderCloseFunc) PushHook(hook func() error) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *BlameHunkReaderCloseFunc) SetDefaultReturn(r0 error) {
f.SetDefaultHook(func() error {
return r0
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *BlameHunkReaderCloseFunc) PushReturn(r0 error) {
f.PushHook(func() error {
return r0
})
}
func (f *BlameHunkReaderCloseFunc) nextHook() func() error {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *BlameHunkReaderCloseFunc) appendCall(r0 BlameHunkReaderCloseFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of BlameHunkReaderCloseFuncCall objects
// describing the invocations of this function.
func (f *BlameHunkReaderCloseFunc) History() []BlameHunkReaderCloseFuncCall {
f.mutex.Lock()
history := make([]BlameHunkReaderCloseFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// BlameHunkReaderCloseFuncCall is an object that describes an invocation of
// method Close on an instance of MockBlameHunkReader.
type BlameHunkReaderCloseFuncCall struct {
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c BlameHunkReaderCloseFuncCall) Args() []interface{} {
return []interface{}{}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c BlameHunkReaderCloseFuncCall) Results() []interface{} {
return []interface{}{c.Result0}
}
// BlameHunkReaderReadFunc describes the behavior when the Read method of
// the parent MockBlameHunkReader instance is invoked.
type BlameHunkReaderReadFunc struct {
defaultHook func() (*gitdomain.Hunk, error)
hooks []func() (*gitdomain.Hunk, error)
history []BlameHunkReaderReadFuncCall
mutex sync.Mutex
}
// Read delegates to the next hook function in the queue and stores the
// parameter and result values of this invocation.
func (m *MockBlameHunkReader) Read() (*gitdomain.Hunk, error) {
r0, r1 := m.ReadFunc.nextHook()()
m.ReadFunc.appendCall(BlameHunkReaderReadFuncCall{r0, r1})
return r0, r1
}
// SetDefaultHook sets function that is called when the Read method of the
// parent MockBlameHunkReader instance is invoked and the hook queue is
// empty.
func (f *BlameHunkReaderReadFunc) SetDefaultHook(hook func() (*gitdomain.Hunk, error)) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// Read method of the parent MockBlameHunkReader instance invokes the hook
// at the front of the queue and discards it. After the queue is empty, the
// default hook function is invoked for any future action.
func (f *BlameHunkReaderReadFunc) PushHook(hook func() (*gitdomain.Hunk, error)) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *BlameHunkReaderReadFunc) SetDefaultReturn(r0 *gitdomain.Hunk, r1 error) {
f.SetDefaultHook(func() (*gitdomain.Hunk, error) {
return r0, r1
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *BlameHunkReaderReadFunc) PushReturn(r0 *gitdomain.Hunk, r1 error) {
f.PushHook(func() (*gitdomain.Hunk, error) {
return r0, r1
})
}
func (f *BlameHunkReaderReadFunc) nextHook() func() (*gitdomain.Hunk, error) {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *BlameHunkReaderReadFunc) appendCall(r0 BlameHunkReaderReadFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of BlameHunkReaderReadFuncCall objects
// describing the invocations of this function.
func (f *BlameHunkReaderReadFunc) History() []BlameHunkReaderReadFuncCall {
f.mutex.Lock()
history := make([]BlameHunkReaderReadFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// BlameHunkReaderReadFuncCall is an object that describes an invocation of
// method Read on an instance of MockBlameHunkReader.
type BlameHunkReaderReadFuncCall struct {
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 *gitdomain.Hunk
// Result1 is the value of the 2nd result returned from this method
// invocation.
Result1 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c BlameHunkReaderReadFuncCall) Args() []interface{} {
return []interface{}{}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c BlameHunkReaderReadFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1}
}
// MockGitBackend is a mock implementation of the GitBackend interface (from
// the package
// github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git) used for
// unit testing.
type MockGitBackend struct {
// BlameFunc is an instance of a mock function object controlling the
// behavior of the method Blame.
BlameFunc *GitBackendBlameFunc
// ConfigFunc is an instance of a mock function object controlling the
// behavior of the method Config.
ConfigFunc *GitBackendConfigFunc
@ -38,6 +304,11 @@ type MockGitBackend struct {
// methods return zero values for all results, unless overwritten.
func NewMockGitBackend() *MockGitBackend {
return &MockGitBackend{
BlameFunc: &GitBackendBlameFunc{
defaultHook: func(context.Context, string, BlameOptions) (r0 BlameHunkReader, r1 error) {
return
},
},
ConfigFunc: &GitBackendConfigFunc{
defaultHook: func() (r0 GitConfigBackend) {
return
@ -65,6 +336,11 @@ func NewMockGitBackend() *MockGitBackend {
// All methods panic on invocation, unless overwritten.
func NewStrictMockGitBackend() *MockGitBackend {
return &MockGitBackend{
BlameFunc: &GitBackendBlameFunc{
defaultHook: func(context.Context, string, BlameOptions) (BlameHunkReader, error) {
panic("unexpected invocation of MockGitBackend.Blame")
},
},
ConfigFunc: &GitBackendConfigFunc{
defaultHook: func() GitConfigBackend {
panic("unexpected invocation of MockGitBackend.Config")
@ -92,6 +368,9 @@ func NewStrictMockGitBackend() *MockGitBackend {
// All methods delegate to the given implementation, unless overwritten.
func NewMockGitBackendFrom(i GitBackend) *MockGitBackend {
return &MockGitBackend{
BlameFunc: &GitBackendBlameFunc{
defaultHook: i.Blame,
},
ConfigFunc: &GitBackendConfigFunc{
defaultHook: i.Config,
},
@ -107,6 +386,116 @@ func NewMockGitBackendFrom(i GitBackend) *MockGitBackend {
}
}
// GitBackendBlameFunc describes the behavior when the Blame method of the
// parent MockGitBackend instance is invoked.
type GitBackendBlameFunc struct {
defaultHook func(context.Context, string, BlameOptions) (BlameHunkReader, error)
hooks []func(context.Context, string, BlameOptions) (BlameHunkReader, error)
history []GitBackendBlameFuncCall
mutex sync.Mutex
}
// Blame delegates to the next hook function in the queue and stores the
// parameter and result values of this invocation.
func (m *MockGitBackend) Blame(v0 context.Context, v1 string, v2 BlameOptions) (BlameHunkReader, error) {
r0, r1 := m.BlameFunc.nextHook()(v0, v1, v2)
m.BlameFunc.appendCall(GitBackendBlameFuncCall{v0, v1, v2, r0, r1})
return r0, r1
}
// SetDefaultHook sets function that is called when the Blame method of the
// parent MockGitBackend instance is invoked and the hook queue is empty.
func (f *GitBackendBlameFunc) SetDefaultHook(hook func(context.Context, string, BlameOptions) (BlameHunkReader, error)) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// Blame method of the parent MockGitBackend instance invokes the hook at
// the front of the queue and discards it. After the queue is empty, the
// default hook function is invoked for any future action.
func (f *GitBackendBlameFunc) PushHook(hook func(context.Context, string, BlameOptions) (BlameHunkReader, error)) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *GitBackendBlameFunc) SetDefaultReturn(r0 BlameHunkReader, r1 error) {
f.SetDefaultHook(func(context.Context, string, BlameOptions) (BlameHunkReader, error) {
return r0, r1
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *GitBackendBlameFunc) PushReturn(r0 BlameHunkReader, r1 error) {
f.PushHook(func(context.Context, string, BlameOptions) (BlameHunkReader, error) {
return r0, r1
})
}
func (f *GitBackendBlameFunc) nextHook() func(context.Context, string, BlameOptions) (BlameHunkReader, error) {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *GitBackendBlameFunc) appendCall(r0 GitBackendBlameFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of GitBackendBlameFuncCall objects describing
// the invocations of this function.
func (f *GitBackendBlameFunc) History() []GitBackendBlameFuncCall {
f.mutex.Lock()
history := make([]GitBackendBlameFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// GitBackendBlameFuncCall is an object that describes an invocation of
// method Blame on an instance of MockGitBackend.
type GitBackendBlameFuncCall struct {
// Arg0 is the value of the 1st argument passed to this method
// invocation.
Arg0 context.Context
// Arg1 is the value of the 2nd argument passed to this method
// invocation.
Arg1 string
// Arg2 is the value of the 3rd argument passed to this method
// invocation.
Arg2 BlameOptions
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 BlameHunkReader
// Result1 is the value of the 2nd result returned from this method
// invocation.
Result1 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c GitBackendBlameFuncCall) Args() []interface{} {
return []interface{}{c.Arg0, c.Arg1, c.Arg2}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c GitBackendBlameFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1}
}
// GitBackendConfigFunc describes the behavior when the Config method of the
// parent MockGitBackend instance is invoked.
type GitBackendConfigFunc struct {

2
cmd/gitserver/internal/integration_tests/archivereader_test.go
View File

@ -119,7 +119,7 @@ func TestClient_ArchiveReader(t *testing.T) {
grpcServer := defaults.NewServer(logtest.Scoped(t))
proto.RegisterGitserverServiceServer(grpcServer, &server.GRPCServer{Server: s})
proto.RegisterGitserverServiceServer(grpcServer, server.NewGRPCServer(s))
handler := internalgrpc.MultiplexHandlers(grpcServer, s.Handler())
srv := httptest.NewServer(handler)
defer srv.Close()

4
cmd/gitserver/internal/integration_tests/clone_test.go
View File

@ -74,7 +74,7 @@ func TestClone(t *testing.T) {
}
grpcServer := defaults.NewServer(logtest.Scoped(t))
proto.RegisterGitserverServiceServer(grpcServer, &server.GRPCServer{Server: &s})
proto.RegisterGitserverServiceServer(grpcServer, server.NewGRPCServer(&s))
handler := internalgrpc.MultiplexHandlers(grpcServer, s.Handler())
srv := httptest.NewServer(handler)
@ -170,7 +170,7 @@ func TestClone_Fail(t *testing.T) {
}
grpcServer := defaults.NewServer(logtest.Scoped(t))
proto.RegisterGitserverServiceServer(grpcServer, &server.GRPCServer{Server: &s})
proto.RegisterGitserverServiceServer(grpcServer, server.NewGRPCServer(&s))
handler := internalgrpc.MultiplexHandlers(grpcServer, s.Handler())
srv := httptest.NewServer(handler)

2
cmd/gitserver/internal/integration_tests/resolverevisions_test.go
View File

@ -89,7 +89,7 @@ func TestClient_ResolveRevisions(t *testing.T) {
}
grpcServer := defaults.NewServer(logtest.Scoped(t))
proto.RegisterGitserverServiceServer(grpcServer, &server.GRPCServer{Server: &s})
proto.RegisterGitserverServiceServer(grpcServer, server.NewGRPCServer(&s))
handler := internalgrpc.MultiplexHandlers(grpcServer, s.Handler())
srv := httptest.NewServer(handler)

2
cmd/gitserver/internal/integration_tests/test_utils.go
View File

@ -97,7 +97,7 @@ func InitGitserver() {
}
grpcServer := defaults.NewServer(logger)
proto.RegisterGitserverServiceServer(grpcServer, &server.GRPCServer{Server: &s})
proto.RegisterGitserverServiceServer(grpcServer, server.NewGRPCServer(&s))
handler := internalgrpc.MultiplexHandlers(grpcServer, s.Handler())
srv := &http.Server{

1312
cmd/gitserver/internal/mocks_test.go generated Normal file
View File

File diff suppressed because it is too large Load Diff

15
cmd/gitserver/internal/p4exec.go
View File

@ -28,7 +28,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/trace"
)
func (gs *GRPCServer) P4Exec(req *proto.P4ExecRequest, ss proto.GitserverService_P4ExecServer) error {
func (gs *grpcServer) P4Exec(req *proto.P4ExecRequest, ss proto.GitserverService_P4ExecServer) error {
arguments := byteSlicesToStrings(req.GetArgs()) //nolint:staticcheck
if len(arguments) < 1 {
@ -50,7 +50,7 @@ func (gs *GRPCServer) P4Exec(req *proto.P4ExecRequest, ss proto.GitserverService
return status.Error(codes.InvalidArgument, fmt.Sprintf("subcommand %q is not allowed", subCommand))
}
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return status.Error(codes.Internal, err.Error())
}
@ -84,11 +84,11 @@ func (gs *GRPCServer) P4Exec(req *proto.P4ExecRequest, ss proto.GitserverService
var r p4ExecRequest
r.FromProto(req)
return gs.doP4Exec(ss.Context(), gs.Server.Logger, &r, "unknown-grpc-client", w)
return gs.doP4Exec(ss.Context(), &r, w)
}
func (gs *GRPCServer) doP4Exec(ctx context.Context, logger log.Logger, req *p4ExecRequest, userAgent string, w io.Writer) error {
execStatus := gs.Server.p4Exec(ctx, logger, req, userAgent, w)
func (gs *grpcServer) doP4Exec(ctx context.Context, req *p4ExecRequest, w io.Writer) error {
execStatus := gs.svc.P4Exec(ctx, gs.logger, req, w)
if execStatus.ExitStatus != 0 || execStatus.Err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
@ -105,7 +105,7 @@ func (gs *GRPCServer) doP4Exec(ctx context.Context, logger log.Logger, req *p4Ex
Stderr: execStatus.Stderr,
})
if err != nil {
gs.Server.Logger.Error("failed to marshal status", log.Error(err))
gs.logger.Error("failed to marshal status", log.Error(err))
return err
}
return s.Err()
@ -114,7 +114,7 @@ func (gs *GRPCServer) doP4Exec(ctx context.Context, logger log.Logger, req *p4Ex
return nil
}
func (s *Server) p4Exec(ctx context.Context, logger log.Logger, req *p4ExecRequest, userAgent string, w io.Writer) execStatus {
func (s *Server) P4Exec(ctx context.Context, logger log.Logger, req *p4ExecRequest, w io.Writer) execStatus {
start := time.Now()
var cmdStart time.Time // set once we have ensured commit
exitStatus := executil.UnsetExitStatus
@ -163,7 +163,6 @@ func (s *Server) p4Exec(ctx context.Context, logger log.Logger, req *p4ExecReque
ev.AddField("cmd", cmd)
ev.AddField("args", args)
ev.AddField("actor", act.UIDString())
ev.AddField("client", userAgent)
ev.AddField("duration_ms", duration.Milliseconds())
ev.AddField("stdout_size", stdoutN)
ev.AddField("stderr_size", stderrN)

2
cmd/gitserver/internal/p4exec_test.go
View File

@ -63,7 +63,7 @@ func TestServer_handleP4Exec(t *testing.T) {
}
server := defaults.NewServer(logger)
proto.RegisterGitserverServiceServer(server, &GRPCServer{Server: s})
proto.RegisterGitserverServiceServer(server, NewGRPCServer(s))
handler = grpc.MultiplexHandlers(server, s.Handler())
srv := httptest.NewServer(handler)

2
cmd/gitserver/internal/patch.go
View File

@ -35,7 +35,7 @@ import (
var patchID uint64
func (s *Server) createCommitFromPatch(ctx context.Context, req protocol.CreateCommitFromPatchRequest) (int, protocol.CreateCommitFromPatchResponse) {
func (s *Server) CreateCommitFromPatch(ctx context.Context, req protocol.CreateCommitFromPatchRequest) (int, protocol.CreateCommitFromPatchResponse) {
logger := s.Logger.Scoped("createCommitFromPatch").
With(
log.String("repo", string(req.Repo)),

2
cmd/gitserver/internal/repo_info_test.go
View File

@ -62,7 +62,7 @@ func testDeleteRepo(t *testing.T, deletedInDB bool) {
_ = s.Handler()
// This will perform an initial clone
s.repoUpdate(&protocol.RepoUpdateRequest{
s.RepoUpdate(&protocol.RepoUpdateRequest{
Repo: repoName,
})

2
cmd/gitserver/internal/search.go
View File

@ -21,7 +21,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/trace"
)
func (s *Server) searchWithObservability(ctx context.Context, tr trace.Trace, args *protocol.SearchRequest, onMatch func(*protocol.CommitMatch) error) (limitHit bool, err error) {
func (s *Server) SearchWithObservability(ctx context.Context, tr trace.Trace, args *protocol.SearchRequest, onMatch func(*protocol.CommitMatch) error) (limitHit bool, err error) {
searchStart := time.Now()
searchRunning.Inc()

24
cmd/gitserver/internal/server.go
View File

@ -43,7 +43,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/featureflag"
"github.com/sourcegraph/sourcegraph/internal/fileutil"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/gitserver/protocol"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
@ -269,10 +268,6 @@ func (s *Server) Handler() http.Handler {
return mux
}
func addrForRepo(ctx context.Context, repoName api.RepoName, gitServerAddrs gitserver.GitserverAddresses) string {
return gitServerAddrs.AddrForRepo(ctx, filepath.Base(os.Args[0]), repoName)
}
// NewClonePipeline creates a new pipeline that clones repos asynchronously. It
// creates a producer-consumer pipeline that handles clone requests asychronously.
func (s *Server) NewClonePipeline(logger log.Logger, cloneQueue *common.Queue[*cloneJob]) goroutine.BackgroundRoutine {
@ -444,7 +439,7 @@ func (s *Server) acquireCloneableLimiter(ctx context.Context) (context.Context,
return s.cloneableLimiter.Acquire(ctx)
}
func (s *Server) isRepoCloneable(ctx context.Context, repo api.RepoName) (protocol.IsRepoCloneableResponse, error) {
func (s *Server) IsRepoCloneable(ctx context.Context, repo api.RepoName) (protocol.IsRepoCloneableResponse, error) {
// We use an internal actor here as the repo may be private. It is safe since all
// we return is a bool indicating whether the repo is cloneable or not. Perhaps
// the only things that could leak here is whether a private repo exists although
@ -471,7 +466,7 @@ func (s *Server) isRepoCloneable(ctx context.Context, repo api.RepoName) (protoc
return resp, nil
}
func (s *Server) repoUpdate(req *protocol.RepoUpdateRequest) protocol.RepoUpdateResponse {
func (s *Server) RepoUpdate(req *protocol.RepoUpdateRequest) protocol.RepoUpdateResponse {
logger := s.Logger.Scoped("handleRepoUpdate")
var resp protocol.RepoUpdateResponse
req.Repo = protocol.NormalizeRepo(req.Repo)
@ -544,10 +539,10 @@ type execStatus struct {
}
// TODO: eseliger
// exec runs a git command. After the first write to w, it must not return an error.
// Exec runs a git command. After the first write to w, it must not return an error.
// TODO(@camdencheek): once gRPC is the only consumer of this, do everything with errors
// because gRPC can handle trailing errors on a stream.
func (s *Server) exec(ctx context.Context, logger log.Logger, req *protocol.ExecRequest, userAgent string, w io.Writer) (execStatus, error) {
func (s *Server) Exec(ctx context.Context, req *protocol.ExecRequest, w io.Writer) (execStatus, error) {
repoName := protocol.NormalizeRepo(req.Repo)
dir := gitserverfs.RepoDirFromName(s.ReposDir, repoName)
backend := s.GetBackendFunc(dir, repoName)
@ -579,7 +574,7 @@ func (s *Server) exec(ctx context.Context, logger log.Logger, req *protocol.Exec
attribute.String("args", args),
attribute.String("ensure_revision", req.EnsureRevision),
)
logger = logger.WithTrace(trace.Context(ctx))
logger := s.Logger.WithTrace(trace.Context(ctx))
execRunning.WithLabelValues(cmd).Inc()
defer func() {
@ -614,7 +609,6 @@ func (s *Server) exec(ctx context.Context, logger log.Logger, req *protocol.Exec
ev.AddField("actor", act.UIDString())
ev.AddField("ensure_revision", req.EnsureRevision)
ev.AddField("ensure_revision_status", ensureRevisionStatus)
ev.AddField("client", userAgent)
ev.AddField("duration_ms", duration.Milliseconds())
ev.AddField("exit_status", exitStatus)
ev.AddField("status", status)
@ -648,7 +642,7 @@ func (s *Server) exec(ctx context.Context, logger log.Logger, req *protocol.Exec
}()
}
if notFoundPayload, cloned := s.maybeStartClone(ctx, logger, repoName); !cloned {
if notFoundPayload, cloned := s.MaybeStartClone(ctx, repoName); !cloned {
if notFoundPayload.CloneInProgress {
status = "clone-in-progress"
} else {
@ -691,7 +685,7 @@ func (s *Server) exec(ctx context.Context, logger log.Logger, req *protocol.Exec
_, execErr = io.Copy(w, stdout)
if execErr != nil {
s.logIfCorrupt(ctx, repoName, execErr)
s.LogIfCorrupt(ctx, repoName, execErr)
commandFailedErr := &gitcli.CommandFailedError{}
if errors.As(execErr, &commandFailedErr) {
exitStatus = commandFailedErr.ExitStatus
@ -741,7 +735,7 @@ func (s *Server) setLastErrorNonFatal(ctx context.Context, name api.RepoName, er
}
}
func (s *Server) logIfCorrupt(ctx context.Context, repo api.RepoName, err error) {
func (s *Server) LogIfCorrupt(ctx context.Context, repo api.RepoName, err error) {
var corruptErr common.ErrRepoCorrupted
if errors.As(err, &corruptErr) {
if err := s.DB.GitserverRepos().LogCorruption(ctx, repo, corruptErr.Reason, s.Hostname); err != nil {
@ -1274,7 +1268,7 @@ func (s *Server) doRepoUpdate(ctx context.Context, repo api.RepoName, revspec st
// The repo update might have failed due to the repo being corrupt
var corruptErr common.ErrRepoCorrupted
if errors.As(err, &corruptErr) {
s.logIfCorrupt(ctx, repo, corruptErr)
s.LogIfCorrupt(ctx, repo, corruptErr)
}
}
s.setLastErrorNonFatal(s.ctx, repo, err)

249
cmd/gitserver/internal/server_grpc.go
View File

@ -15,8 +15,11 @@ import (
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git/gitcli"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/gitserverfs"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/perforce"
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/gitserver/protocol"
@ -24,16 +27,53 @@ import (
"github.com/sourcegraph/sourcegraph/internal/grpc/streamio"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/pointers"
)
type GRPCServer struct {
Server *Server
type service interface {
CreateCommitFromPatch(ctx context.Context, req protocol.CreateCommitFromPatchRequest) (int, protocol.CreateCommitFromPatchResponse)
LogIfCorrupt(context.Context, api.RepoName, error)
Exec(ctx context.Context, req *protocol.ExecRequest, w io.Writer) (execStatus, error)
MaybeStartClone(ctx context.Context, repo api.RepoName) (notFound *protocol.NotFoundPayload, cloned bool)
IsRepoCloneable(ctx context.Context, repo api.RepoName) (protocol.IsRepoCloneableResponse, error)
RepoUpdate(req *protocol.RepoUpdateRequest) protocol.RepoUpdateResponse
CloneRepo(ctx context.Context, repo api.RepoName, opts CloneOptions) (cloneProgress string, err error)
SearchWithObservability(ctx context.Context, tr trace.Trace, args *protocol.SearchRequest, onMatch func(*protocol.CommitMatch) error) (limitHit bool, err error)
BatchGitLogInstrumentedHandler(ctx context.Context, req *proto.BatchLogRequest) (resp *proto.BatchLogResponse, err error)
P4Exec(ctx context.Context, logger log.Logger, req *p4ExecRequest, w io.Writer) execStatus
}
func NewGRPCServer(server *Server) proto.GitserverServiceServer {
return &grpcServer{
logger: server.Logger,
reposDir: server.ReposDir,
db: server.DB,
hostname: server.Hostname,
subRepoChecker: authz.DefaultSubRepoPermsChecker,
locker: server.Locker,
getBackendFunc: server.GetBackendFunc,
svc: server,
}
}
type grpcServer struct {
logger log.Logger
reposDir string
db database.DB
hostname string
subRepoChecker authz.SubRepoPermissionChecker
locker RepositoryLocker
getBackendFunc Backender
svc service
proto.UnimplementedGitserverServiceServer
}
var _ proto.GitserverServiceServer = &GRPCServer{}
var _ proto.GitserverServiceServer = &grpcServer{}
func (gs *GRPCServer) BatchLog(ctx context.Context, req *proto.BatchLogRequest) (*proto.BatchLogResponse, error) {
func (gs *grpcServer) BatchLog(ctx context.Context, req *proto.BatchLogRequest) (*proto.BatchLogResponse, error) {
// Validate request parameters
if len(req.GetRepoCommits()) == 0 { //nolint:staticcheck
return &proto.BatchLogResponse{}, nil
@ -43,7 +83,7 @@ func (gs *GRPCServer) BatchLog(ctx context.Context, req *proto.BatchLogRequest)
}
// Handle unexpected error conditions
resp, err := gs.Server.batchGitLogInstrumentedHandler(ctx, req)
resp, err := gs.svc.BatchGitLogInstrumentedHandler(ctx, req)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -51,7 +91,7 @@ func (gs *GRPCServer) BatchLog(ctx context.Context, req *proto.BatchLogRequest)
return resp, nil
}
func (gs *GRPCServer) CreateCommitFromPatchBinary(s proto.GitserverService_CreateCommitFromPatchBinaryServer) error {
func (gs *grpcServer) CreateCommitFromPatchBinary(s proto.GitserverService_CreateCommitFromPatchBinaryServer) error {
var (
metadata *proto.CreateCommitFromPatchBinaryRequest_Metadata
patch []byte
@ -89,7 +129,7 @@ func (gs *GRPCServer) CreateCommitFromPatchBinary(s proto.GitserverService_Creat
var r protocol.CreateCommitFromPatchRequest
r.FromProto(metadata, patch)
_, resp := gs.Server.createCommitFromPatch(s.Context(), r)
_, resp := gs.svc.CreateCommitFromPatch(s.Context(), r)
res, err := resp.ToProto()
if err != nil {
return err.ToStatus().Err()
@ -98,11 +138,11 @@ func (gs *GRPCServer) CreateCommitFromPatchBinary(s proto.GitserverService_Creat
return s.SendAndClose(res)
}
func (gs *GRPCServer) DiskInfo(_ context.Context, _ *proto.DiskInfoRequest) (*proto.DiskInfoResponse, error) {
return getDiskInfo(gs.Server.ReposDir)
func (gs *grpcServer) DiskInfo(_ context.Context, _ *proto.DiskInfoRequest) (*proto.DiskInfoResponse, error) {
return getDiskInfo(gs.reposDir)
}
func (gs *GRPCServer) Exec(req *proto.ExecRequest, ss proto.GitserverService_ExecServer) error {
func (gs *grpcServer) Exec(req *proto.ExecRequest, ss proto.GitserverService_ExecServer) error {
internalReq := protocol.ExecRequest{
Repo: api.RepoName(req.GetRepo()),
Args: byteSlicesToStrings(req.GetArgs()),
@ -131,11 +171,10 @@ func (gs *GRPCServer) Exec(req *proto.ExecRequest, ss proto.GitserverService_Exe
log.Strings("args", args),
)
// TODO(mucles): set user agent from all grpc clients
return gs.doExec(ss.Context(), gs.Server.Logger, &internalReq, "unknown-grpc-client", w)
return gs.doExec(ss.Context(), &internalReq, w)
}
func (gs *GRPCServer) Archive(req *proto.ArchiveRequest, ss proto.GitserverService_ArchiveServer) error {
func (gs *grpcServer) Archive(req *proto.ArchiveRequest, ss proto.GitserverService_ArchiveServer) error {
// Log which which actor is accessing the repo.
accesslog.Record(ss.Context(), req.GetRepo(),
log.String("treeish", req.GetTreeish()),
@ -181,15 +220,14 @@ func (gs *GRPCServer) Archive(req *proto.ArchiveRequest, ss proto.GitserverServi
ctx, cancel := context.WithTimeout(ss.Context(), conf.GitLongCommandTimeout())
defer cancel()
// TODO(mucles): set user agent from all grpc clients
return gs.doExec(ctx, gs.Server.Logger, execReq, "unknown-grpc-client", w)
return gs.doExec(ctx, execReq, w)
}
// doExec executes the given git command and streams the output to the given writer.
//
// Note: This function wraps the underlying exec implementation and returns grpc specific error handling.
func (gs *GRPCServer) doExec(ctx context.Context, logger log.Logger, req *protocol.ExecRequest, userAgent string, w io.Writer) error {
execStatus, err := gs.Server.exec(ctx, logger, req, userAgent, w)
func (gs *grpcServer) doExec(ctx context.Context, req *protocol.ExecRequest, w io.Writer) error {
execStatus, err := gs.svc.Exec(ctx, req, w)
if err != nil {
if v := (&NotFoundError{}); errors.As(err, &v) {
s, err := status.New(codes.NotFound, "repo not found").WithDetails(&proto.NotFoundPayload{
@ -198,7 +236,7 @@ func (gs *GRPCServer) doExec(ctx context.Context, logger log.Logger, req *protoc
CloneProgress: v.Payload.CloneProgress,
})
if err != nil {
gs.Server.Logger.Error("failed to marshal status", log.Error(err))
gs.logger.Error("failed to marshal status", log.Error(err))
return err
}
return s.Err()
@ -230,7 +268,7 @@ func (gs *GRPCServer) doExec(ctx context.Context, logger log.Logger, req *protoc
Stderr: execStatus.Stderr,
})
if err != nil {
gs.Server.Logger.Error("failed to marshal status", log.Error(err))
gs.logger.Error("failed to marshal status", log.Error(err))
return err
}
return s.Err()
@ -240,19 +278,19 @@ func (gs *GRPCServer) doExec(ctx context.Context, logger log.Logger, req *protoc
}
func (gs *GRPCServer) GetObject(ctx context.Context, req *proto.GetObjectRequest) (*proto.GetObjectResponse, error) {
func (gs *grpcServer) GetObject(ctx context.Context, req *proto.GetObjectRequest) (*proto.GetObjectResponse, error) {
repoName := api.RepoName(req.GetRepo())
repoDir := gitserverfs.RepoDirFromName(gs.Server.ReposDir, repoName)
repoDir := gitserverfs.RepoDirFromName(gs.reposDir, repoName)
// Log which actor is accessing the repo.
accesslog.Record(ctx, string(repoName), log.String("objectname", req.GetObjectName()))
backend := gs.Server.GetBackendFunc(repoDir, repoName)
backend := gs.getBackendFunc(repoDir, repoName)
obj, err := backend.GetObject(ctx, req.GetObjectName())
if err != nil {
gs.Server.logIfCorrupt(ctx, repoName, err)
gs.Server.Logger.Error("getting object", log.Error(err))
gs.svc.LogIfCorrupt(ctx, repoName, err)
gs.logger.Error("getting object", log.Error(err))
return nil, err
}
@ -263,7 +301,7 @@ func (gs *GRPCServer) GetObject(ctx context.Context, req *proto.GetObjectRequest
return resp.ToProto(), nil
}
func (gs *GRPCServer) ListGitolite(ctx context.Context, req *proto.ListGitoliteRequest) (*proto.ListGitoliteResponse, error) {
func (gs *grpcServer) ListGitolite(ctx context.Context, req *proto.ListGitoliteRequest) (*proto.ListGitoliteResponse, error) {
host := req.GetGitoliteHost()
repos, err := defaultGitolite.listRepos(ctx, host)
if err != nil {
@ -281,7 +319,7 @@ func (gs *GRPCServer) ListGitolite(ctx context.Context, req *proto.ListGitoliteR
}, nil
}
func (gs *GRPCServer) Search(req *proto.SearchRequest, ss proto.GitserverService_SearchServer) error {
func (gs *grpcServer) Search(req *proto.SearchRequest, ss proto.GitserverService_SearchServer) error {
args, err := protocol.SearchRequestFromProto(req)
if err != nil {
return status.Error(codes.InvalidArgument, err.Error())
@ -296,7 +334,7 @@ func (gs *GRPCServer) Search(req *proto.SearchRequest, ss proto.GitserverService
tr, ctx := trace.New(ss.Context(), "search")
defer tr.End()
limitHit, err := gs.Server.searchWithObservability(ctx, tr, args, onMatch)
limitHit, err := gs.svc.SearchWithObservability(ctx, tr, args, onMatch)
if err != nil {
if notExistError := new(gitdomain.RepoNotExistError); errors.As(err, &notExistError) {
st, _ := status.New(codes.NotFound, err.Error()).WithDetails(&proto.NotFoundPayload{
@ -315,10 +353,10 @@ func (gs *GRPCServer) Search(req *proto.SearchRequest, ss proto.GitserverService
})
}
func (gs *GRPCServer) RepoClone(ctx context.Context, in *proto.RepoCloneRequest) (*proto.RepoCloneResponse, error) {
func (gs *grpcServer) RepoClone(ctx context.Context, in *proto.RepoCloneRequest) (*proto.RepoCloneResponse, error) {
repo := protocol.NormalizeRepo(api.RepoName(in.GetRepo()))
if _, err := gs.Server.CloneRepo(ctx, repo, CloneOptions{Block: false}); err != nil {
if _, err := gs.svc.CloneRepo(ctx, repo, CloneOptions{Block: false}); err != nil {
return &proto.RepoCloneResponse{Error: err.Error()}, nil
}
@ -326,7 +364,7 @@ func (gs *GRPCServer) RepoClone(ctx context.Context, in *proto.RepoCloneRequest)
return &proto.RepoCloneResponse{Error: ""}, nil
}
func (gs *GRPCServer) RepoCloneProgress(_ context.Context, req *proto.RepoCloneProgressRequest) (*proto.RepoCloneProgressResponse, error) {
func (gs *grpcServer) RepoCloneProgress(_ context.Context, req *proto.RepoCloneProgressRequest) (*proto.RepoCloneProgressResponse, error) {
repositories := req.GetRepos()
resp := protocol.RepoCloneProgressResponse{
@ -334,39 +372,39 @@ func (gs *GRPCServer) RepoCloneProgress(_ context.Context, req *proto.RepoCloneP
}
for _, repo := range repositories {
repoName := api.RepoName(repo)
result := repoCloneProgress(gs.Server.ReposDir, gs.Server.Locker, repoName)
result := repoCloneProgress(gs.reposDir, gs.locker, repoName)
resp.Results[repoName] = result
}
return resp.ToProto(), nil
}
func (gs *GRPCServer) RepoDelete(ctx context.Context, req *proto.RepoDeleteRequest) (*proto.RepoDeleteResponse, error) {
func (gs *grpcServer) RepoDelete(ctx context.Context, req *proto.RepoDeleteRequest) (*proto.RepoDeleteResponse, error) {
repo := req.GetRepo()
if err := deleteRepo(ctx, gs.Server.Logger, gs.Server.DB, gs.Server.Hostname, gs.Server.ReposDir, api.RepoName(repo)); err != nil {
gs.Server.Logger.Error("failed to delete repository", log.String("repo", repo), log.Error(err))
if err := deleteRepo(ctx, gs.logger, gs.db, gs.hostname, gs.reposDir, api.RepoName(repo)); err != nil {
gs.logger.Error("failed to delete repository", log.String("repo", repo), log.Error(err))
return &proto.RepoDeleteResponse{}, status.Errorf(codes.Internal, "failed to delete repository %s: %s", repo, err)
}
gs.Server.Logger.Info("deleted repository", log.String("repo", repo))
gs.logger.Info("deleted repository", log.String("repo", repo))
return &proto.RepoDeleteResponse{}, nil
}
func (gs *GRPCServer) RepoUpdate(_ context.Context, req *proto.RepoUpdateRequest) (*proto.RepoUpdateResponse, error) {
func (gs *grpcServer) RepoUpdate(_ context.Context, req *proto.RepoUpdateRequest) (*proto.RepoUpdateResponse, error) {
var in protocol.RepoUpdateRequest
in.FromProto(req)
grpcResp := gs.Server.repoUpdate(&in)
grpcResp := gs.svc.RepoUpdate(&in)
return grpcResp.ToProto(), nil
}
func (gs *GRPCServer) IsRepoCloneable(ctx context.Context, req *proto.IsRepoCloneableRequest) (*proto.IsRepoCloneableResponse, error) {
func (gs *grpcServer) IsRepoCloneable(ctx context.Context, req *proto.IsRepoCloneableRequest) (*proto.IsRepoCloneableResponse, error) {
repo := api.RepoName(req.GetRepo())
if req.Repo == "" {
return nil, status.Error(codes.InvalidArgument, "no Repo given")
}
resp, err := gs.Server.isRepoCloneable(ctx, repo)
resp, err := gs.svc.IsRepoCloneable(ctx, repo)
if err != nil {
return nil, err
}
@ -374,12 +412,12 @@ func (gs *GRPCServer) IsRepoCloneable(ctx context.Context, req *proto.IsRepoClon
return resp.ToProto(), nil
}
func (gs *GRPCServer) IsPerforcePathCloneable(ctx context.Context, req *proto.IsPerforcePathCloneableRequest) (*proto.IsPerforcePathCloneableResponse, error) {
func (gs *grpcServer) IsPerforcePathCloneable(ctx context.Context, req *proto.IsPerforcePathCloneableRequest) (*proto.IsPerforcePathCloneableResponse, error) {
if req.DepotPath == "" {
return nil, status.Error(codes.InvalidArgument, "no DepotPath given")
}
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -393,8 +431,8 @@ func (gs *GRPCServer) IsPerforcePathCloneable(ctx context.Context, req *proto.Is
return &proto.IsPerforcePathCloneableResponse{}, nil
}
func (gs *GRPCServer) CheckPerforceCredentials(ctx context.Context, req *proto.CheckPerforceCredentialsRequest) (*proto.CheckPerforceCredentialsResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
func (gs *grpcServer) CheckPerforceCredentials(ctx context.Context, req *proto.CheckPerforceCredentialsRequest) (*proto.CheckPerforceCredentialsResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -412,8 +450,8 @@ func (gs *GRPCServer) CheckPerforceCredentials(ctx context.Context, req *proto.C
return &proto.CheckPerforceCredentialsResponse{}, nil
}
func (gs *GRPCServer) PerforceUsers(ctx context.Context, req *proto.PerforceUsersRequest) (*proto.PerforceUsersResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
func (gs *grpcServer) PerforceUsers(ctx context.Context, req *proto.PerforceUsersRequest) (*proto.PerforceUsersResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -451,8 +489,8 @@ func (gs *GRPCServer) PerforceUsers(ctx context.Context, req *proto.PerforceUser
return resp, nil
}
func (gs *GRPCServer) PerforceProtectsForUser(ctx context.Context, req *proto.PerforceProtectsForUserRequest) (*proto.PerforceProtectsForUserResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
func (gs *grpcServer) PerforceProtectsForUser(ctx context.Context, req *proto.PerforceProtectsForUserRequest) (*proto.PerforceProtectsForUserResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -489,8 +527,8 @@ func (gs *GRPCServer) PerforceProtectsForUser(ctx context.Context, req *proto.Pe
}, nil
}
func (gs *GRPCServer) PerforceProtectsForDepot(ctx context.Context, req *proto.PerforceProtectsForDepotRequest) (*proto.PerforceProtectsForDepotResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
func (gs *grpcServer) PerforceProtectsForDepot(ctx context.Context, req *proto.PerforceProtectsForDepotRequest) (*proto.PerforceProtectsForDepotResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -527,8 +565,8 @@ func (gs *GRPCServer) PerforceProtectsForDepot(ctx context.Context, req *proto.P
}, nil
}
func (gs *GRPCServer) PerforceGroupMembers(ctx context.Context, req *proto.PerforceGroupMembersRequest) (*proto.PerforceGroupMembersResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
func (gs *grpcServer) PerforceGroupMembers(ctx context.Context, req *proto.PerforceGroupMembersRequest) (*proto.PerforceGroupMembersResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -560,8 +598,8 @@ func (gs *GRPCServer) PerforceGroupMembers(ctx context.Context, req *proto.Perfo
}, nil
}
func (gs *GRPCServer) IsPerforceSuperUser(ctx context.Context, req *proto.IsPerforceSuperUserRequest) (*proto.IsPerforceSuperUserResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
func (gs *grpcServer) IsPerforceSuperUser(ctx context.Context, req *proto.IsPerforceSuperUserRequest) (*proto.IsPerforceSuperUserResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -592,8 +630,8 @@ func (gs *GRPCServer) IsPerforceSuperUser(ctx context.Context, req *proto.IsPerf
return &proto.IsPerforceSuperUserResponse{}, nil
}
func (gs *GRPCServer) PerforceGetChangelist(ctx context.Context, req *proto.PerforceGetChangelistRequest) (*proto.PerforceGetChangelistResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.Server.ReposDir)
func (gs *grpcServer) PerforceGetChangelist(ctx context.Context, req *proto.PerforceGetChangelistRequest) (*proto.PerforceGetChangelistResponse, error) {
p4home, err := gitserverfs.MakeP4HomeDir(gs.reposDir)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
@ -633,7 +671,7 @@ func byteSlicesToStrings(in [][]byte) []string {
return res
}
func (gs *GRPCServer) MergeBase(ctx context.Context, req *proto.MergeBaseRequest) (*proto.MergeBaseResponse, error) {
func (gs *grpcServer) MergeBase(ctx context.Context, req *proto.MergeBaseRequest) (*proto.MergeBaseResponse, error) {
accesslog.Record(
ctx,
req.GetRepoName(),
@ -654,11 +692,11 @@ func (gs *GRPCServer) MergeBase(ctx context.Context, req *proto.MergeBaseRequest
}
repoName := api.RepoName(req.GetRepoName())
repoDir := gitserverfs.RepoDirFromName(gs.Server.ReposDir, repoName)
repoDir := gitserverfs.RepoDirFromName(gs.reposDir, repoName)
// Ensure that the repo is cloned and if not start a background clone, then
// return a well-known NotFound payload error.
if notFoundPayload, cloned := gs.Server.maybeStartClone(ctx, gs.Server.Logger, repoName); !cloned {
if notFoundPayload, cloned := gs.svc.MaybeStartClone(ctx, repoName); !cloned {
s, err := status.New(codes.NotFound, "repo not cloned").WithDetails(&proto.NotFoundPayload{
CloneInProgress: notFoundPayload.CloneInProgress,
CloneProgress: notFoundPayload.CloneProgress,
@ -671,13 +709,13 @@ func (gs *GRPCServer) MergeBase(ctx context.Context, req *proto.MergeBaseRequest
}
// TODO: This should be included in requests where we do ensure the revision exists.
// gs.Server.ensureRevision(ctx, repoName, "THE REVISION", repoDir)
// gs.server.ensureRevision(ctx, repoName, "THE REVISION", repoDir)
backend := gs.Server.GetBackendFunc(repoDir, repoName)
backend := gs.getBackendFunc(repoDir, repoName)
sha, err := backend.MergeBase(ctx, string(req.GetBase()), string(req.GetHead()))
if err != nil {
gs.Server.logIfCorrupt(ctx, repoName, err)
gs.svc.LogIfCorrupt(ctx, repoName, err)
// TODO: Better error checking.
return nil, err
}
@ -686,3 +724,92 @@ func (gs *GRPCServer) MergeBase(ctx context.Context, req *proto.MergeBaseRequest
MergeBaseCommitSha: string(sha),
}, nil
}
func (gs *grpcServer) Blame(req *proto.BlameRequest, ss proto.GitserverService_BlameServer) error {
ctx := ss.Context()
accesslog.Record(
ctx,
req.GetRepoName(),
log.String("path", req.GetPath()),
log.String("commit", req.GetCommit()),
)
if req.GetRepoName() == "" {
return status.New(codes.InvalidArgument, "repo must be specified").Err()
}
if len(req.GetPath()) == 0 {
return status.New(codes.InvalidArgument, "path must be specified").Err()
}
repoName := api.RepoName(req.GetRepoName())
repoDir := gitserverfs.RepoDirFromName(gs.reposDir, repoName)
// Ensure that the repo is cloned and if not start a background clone, then
// return a well-known NotFound payload error.
if notFoundPayload, cloned := gs.svc.MaybeStartClone(ctx, repoName); !cloned {
s, err := status.New(codes.NotFound, "repo not cloned").WithDetails(&proto.NotFoundPayload{
CloneInProgress: notFoundPayload.CloneInProgress,
CloneProgress: notFoundPayload.CloneProgress,
Repo: req.GetRepoName(),
})
if err != nil {
return err
}
return s.Err()
}
// First, verify that the actor has access to the given path.
hasAccess, err := authz.FilterActorPath(ctx, gs.subRepoChecker, actor.FromContext(ctx), repoName, req.GetPath())
if err != nil {
return err
}
if !hasAccess {
up := &proto.UnauthorizedPayload{
RepoName: req.GetRepoName(),
Path: pointers.Ptr(req.GetPath()),
}
if c := req.GetCommit(); c != "" {
up.Commit = &c
}
s, marshalErr := status.New(codes.PermissionDenied, "no access to path").WithDetails(up)
if marshalErr != nil {
gs.logger.Error("failed to marshal error", log.Error(marshalErr))
return err
}
return s.Err()
}
backend := gs.getBackendFunc(repoDir, repoName)
r, err := backend.Blame(ctx, req.GetPath(), git.BlameOptions{
NewestCommit: api.CommitID(req.GetCommit()),
IgnoreWhitespace: req.GetIgnoreWhitespace(),
StartLine: int(req.GetStartLine()),
EndLine: int(req.GetEndLine()),
})
if err != nil {
gs.svc.LogIfCorrupt(ctx, repoName, err)
// TODO: Better error checking.
return err
}
defer r.Close()
for {
h, err := r.Read()
if err != nil {
// Check if we're done yet.
if err == io.EOF {
return nil
}
gs.svc.LogIfCorrupt(ctx, repoName, err)
return err
}
if err := ss.Send(&proto.BlameResponse{
Hunk: h.ToProto(),
}); err != nil {
return err
}
}
}

176
cmd/gitserver/internal/server_grpc_test.go Normal file
View File

@ -0,0 +1,176 @@
package internal
import (
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"time"
mockassert "github.com/derision-test/go-mockgen/testutil/assert"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/sourcegraph/log/logtest"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/common"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git"
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/gitserver/protocol"
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
v1 "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc"
"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
)
func TestGRPCServer_Blame(t *testing.T) {
mockSS := gitserver.NewMockGitserverService_BlameServer()
// Add an actor to the context.
a := actor.FromUser(1)
mockSS.ContextFunc.SetDefaultReturn(actor.WithActor(context.Background(), a))
t.Run("argument validation", func(t *testing.T) {
gs := &grpcServer{}
err := gs.Blame(&v1.BlameRequest{RepoName: "", Path: "thepath"}, mockSS)
require.ErrorContains(t, err, "repo must be specified")
assertGRPCStatusCode(t, err, codes.InvalidArgument)
err = gs.Blame(&v1.BlameRequest{RepoName: "therepo", Path: ""}, mockSS)
require.ErrorContains(t, err, "path must be specified")
assertGRPCStatusCode(t, err, codes.InvalidArgument)
})
t.Run("checks for uncloned repo", func(t *testing.T) {
svc := NewMockService()
svc.MaybeStartCloneFunc.SetDefaultReturn(&protocol.NotFoundPayload{CloneInProgress: true, CloneProgress: "cloning"}, false)
gs := &grpcServer{svc: svc}
err := gs.Blame(&v1.BlameRequest{RepoName: "therepo", Path: "thepath"}, mockSS)
require.Error(t, err)
assertGRPCStatusCode(t, err, codes.NotFound)
require.Contains(t, err.Error(), "repo not cloned")
mockassert.Called(t, svc.MaybeStartCloneFunc)
})
t.Run("checks for subrepo perms access to given path", func(t *testing.T) {
srp := authz.NewMockSubRepoPermissionChecker()
svc := NewMockService()
// Repo is cloned, proceed!
svc.MaybeStartCloneFunc.SetDefaultReturn(nil, true)
gs := &grpcServer{
subRepoChecker: srp,
svc: svc,
getBackendFunc: func(common.GitDir, api.RepoName) git.GitBackend {
b := git.NewMockGitBackend()
hr := git.NewMockBlameHunkReader()
hr.ReadFunc.SetDefaultReturn(nil, io.EOF)
b.BlameFunc.SetDefaultReturn(hr, nil)
return b
},
}
t.Run("subrepo perms are not enabled", func(t *testing.T) {
srp.EnabledFunc.SetDefaultReturn(false)
err := gs.Blame(&v1.BlameRequest{RepoName: "therepo", Path: "thepath"}, mockSS)
assert.NoError(t, err)
mockassert.Called(t, srp.EnabledFunc)
})
t.Run("subrepo perms are enabled, permission granted", func(t *testing.T) {
srp.EnabledFunc.SetDefaultReturn(true)
srp.PermissionsFunc.SetDefaultReturn(authz.Read, nil)
err := gs.Blame(&v1.BlameRequest{RepoName: "therepo", Path: "thepath"}, mockSS)
assert.NoError(t, err)
mockassert.Called(t, srp.EnabledFunc)
mockassert.Called(t, srp.PermissionsFunc)
})
t.Run("subrepo perms are enabled, permission denied", func(t *testing.T) {
srp.EnabledFunc.SetDefaultReturn(true)
srp.PermissionsFunc.SetDefaultReturn(authz.None, nil)
err := gs.Blame(&v1.BlameRequest{RepoName: "therepo", Path: "thepath"}, mockSS)
require.Error(t, err)
mockassert.Called(t, srp.EnabledFunc)
mockassert.Called(t, srp.PermissionsFunc)
assertGRPCStatusCode(t, err, codes.PermissionDenied)
})
})
t.Run("e2e", func(t *testing.T) {
srp := authz.NewMockSubRepoPermissionChecker()
// Skip subrepo perms checks.
srp.EnabledFunc.SetDefaultReturn(false)
svc := NewMockService()
// Repo is cloned, proceed!
svc.MaybeStartCloneFunc.SetDefaultReturn(nil, true)
b := git.NewMockGitBackend()
hr := git.NewMockBlameHunkReader()
hr.ReadFunc.PushReturn(&gitdomain.Hunk{CommitID: "deadbeef"}, nil)
hr.ReadFunc.PushReturn(nil, io.EOF)
b.BlameFunc.SetDefaultReturn(hr, nil)
gs := &grpcServer{
subRepoChecker: srp,
svc: svc,
getBackendFunc: func(common.GitDir, api.RepoName) git.GitBackend {
return b
},
}
cli := spawnServer(t, gs)
r, err := cli.Blame(context.Background(), &v1.BlameRequest{
RepoName: "therepo",
Path: "thepath",
})
require.NoError(t, err)
for {
msg, err := r.Recv()
if err != nil {
if err == io.EOF {
return
}
require.NoError(t, err)
}
if diff := cmp.Diff(&proto.BlameResponse{
Hunk: &proto.BlameHunk{
Commit: "deadbeef",
Author: &v1.BlameAuthor{
Date: timestamppb.New(time.Time{}),
},
},
}, msg, cmpopts.IgnoreUnexported(proto.BlameResponse{}, proto.BlameHunk{}, proto.BlameAuthor{}, timestamppb.Timestamp{})); diff != "" {
t.Fatalf("unexpected response (-want +got):\n%s", diff)
}
}
})
}
func assertGRPCStatusCode(t *testing.T, err error, want codes.Code) {
t.Helper()
s, ok := status.FromError(err)
require.True(t, ok, "expected status.FromError to succeed")
require.Equal(t, want, s.Code())
}
func spawnServer(t *testing.T, server *grpcServer) proto.GitserverServiceClient {
t.Helper()
grpcServer := defaults.NewServer(logtest.Scoped(t))
proto.RegisterGitserverServiceServer(grpcServer, server)
handler := internalgrpc.MultiplexHandlers(grpcServer, http.NotFoundHandler())
srv := httptest.NewServer(handler)
t.Cleanup(func() {
srv.Close()
})
u, err := url.Parse(srv.URL)
require.NoError(t, err)
cc, err := defaults.Dial(u.Host, logtest.Scoped(t))
require.NoError(t, err)
return proto.NewGitserverServiceClient(cc)
}

16
cmd/gitserver/internal/server_test.go
View File

@ -192,7 +192,7 @@ func TestExecRequest(t *testing.T) {
// Initialize side-effects.
_ = s.Handler()
gs := &GRPCServer{Server: s}
gs := NewGRPCServer(s)
origRepoCloned := repoCloned
repoCloned = func(dir common.GitDir) bool {
@ -569,7 +569,7 @@ func TestHandleRepoUpdate(t *testing.T) {
s.GetRemoteURLFunc = func(ctx context.Context, name api.RepoName) (string, error) {
return "https://invalid.example.com/", nil
}
s.repoUpdate(&protocol.RepoUpdateRequest{
s.RepoUpdate(&protocol.RepoUpdateRequest{
Repo: repoName,
})
@ -600,7 +600,7 @@ func TestHandleRepoUpdate(t *testing.T) {
// This will perform an initial clone
s.GetRemoteURLFunc = oldRemoveURLFunc
s.repoUpdate(&protocol.RepoUpdateRequest{
s.RepoUpdate(&protocol.RepoUpdateRequest{
Repo: repoName,
})
@ -629,7 +629,7 @@ func TestHandleRepoUpdate(t *testing.T) {
t.Cleanup(func() { doBackgroundRepoUpdateMock = nil })
// This will trigger an update since the repo is already cloned
s.repoUpdate(&protocol.RepoUpdateRequest{
s.RepoUpdate(&protocol.RepoUpdateRequest{
Repo: repoName,
})
@ -654,7 +654,7 @@ func TestHandleRepoUpdate(t *testing.T) {
doBackgroundRepoUpdateMock = nil
// This will trigger an update since the repo is already cloned
s.repoUpdate(&protocol.RepoUpdateRequest{
s.RepoUpdate(&protocol.RepoUpdateRequest{
Repo: repoName,
})
@ -1045,7 +1045,7 @@ func TestHandleBatchLog(t *testing.T) {
// Initialize side-effects.
_ = server.Handler()
gs := &GRPCServer{Server: server}
gs := NewGRPCServer(server)
res, err := gs.BatchLog(context.Background(), test.Request)
@ -1101,7 +1101,7 @@ func TestLogIfCorrupt(t *testing.T) {
stdErr := "error: packfile .git/objects/pack/pack-e26c1fc0add58b7649a95f3e901e30f29395e174.pack does not match index"
s.logIfCorrupt(ctx, repoName, common.ErrRepoCorrupted{
s.LogIfCorrupt(ctx, repoName, common.ErrRepoCorrupted{
Reason: stdErr,
})
@ -1127,7 +1127,7 @@ func TestLogIfCorrupt(t *testing.T) {
db.Repos().Delete(ctx, dbRepo.ID)
})
s.logIfCorrupt(ctx, repoName, errors.New("Brought to you by Horsegraph"))
s.LogIfCorrupt(ctx, repoName, errors.New("Brought to you by Horsegraph"))
fromDB, err := s.DB.GitserverRepos().GetByName(ctx, repoName)
assert.NoError(t, err)

2
cmd/gitserver/internal/statesyncer.go
View File

@ -195,7 +195,7 @@ func syncRepoState(
repo.Name = api.UndeletedRepoName(repo.Name)
// Ensure we're only dealing with repos we are responsible for.
addr := addrForRepo(ctx, repo.Name, gitServerAddrs)
addr := gitServerAddrs.AddrForRepo(ctx, repo.Name)
if !hostnameMatch(shardID, addr) {
repoSyncStateCounter.WithLabelValues("other_shard").Inc()
continue

5
cmd/gitserver/shared/shared.go
View File

@ -242,6 +242,7 @@ func makeGRPCServer(logger log.Logger, s *server.Server) *grpc.Server {
proto.GitserverService_P4Exec_FullMethodName: logger.Scoped("p4exec.accesslog"),
proto.GitserverService_GetObject_FullMethodName: logger.Scoped("get-object.accesslog"),
proto.GitserverService_MergeBase_FullMethodName: logger.Scoped("merge-base.accesslog"),
proto.GitserverService_Blame_FullMethodName: logger.Scoped("blame.accesslog"),
} {
streamInterceptor := accesslog.StreamServerInterceptor(scopedLogger, configurationWatcher)
unaryInterceptor := accesslog.UnaryServerInterceptor(scopedLogger, configurationWatcher)
@ -253,9 +254,7 @@ func makeGRPCServer(logger log.Logger, s *server.Server) *grpc.Server {
}
grpcServer := defaults.NewServer(logger, additionalServerOptions...)
proto.RegisterGitserverServiceServer(grpcServer, &server.GRPCServer{
Server: s,
})
proto.RegisterGitserverServiceServer(grpcServer, server.NewGRPCServer(s))
return grpcServer
}

4
internal/gitserver/BUILD.bazel
View File

@ -13,7 +13,6 @@ go_library(
"observability.go",
"retry.go",
"stream_client.go",
"stream_hunks.go",
"test_utils.go",
],
importpath = "github.com/sourcegraph/sourcegraph/internal/gitserver",
@ -39,6 +38,7 @@ go_library(
"//internal/search/streaming/http",
"//internal/trace",
"//lib/errors",
"//lib/pointers",
"@com_github_go_git_go_git_v5//plumbing/format/config",
"@com_github_golang_groupcache//lru",
"@com_github_prometheus_client_golang//prometheus",
@ -91,5 +91,7 @@ go_test(
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
],
)

28
internal/gitserver/addrs.go
View File

@ -26,10 +26,10 @@ import (
)
var (
addrForRepoInvoked = promauto.NewCounterVec(prometheus.CounterOpts{
addrForRepoInvoked = promauto.NewCounter(prometheus.CounterOpts{
Name: "src_gitserver_addr_for_repo_invoked",
Help: "Number of times gitserver.AddrForRepo was invoked",
}, []string{"user_agent"})
})
)
// NewGitserverAddresses fetches the current set of gitserver addresses
@ -104,8 +104,8 @@ type testGitserverConns struct {
}
// AddrForRepo returns the gitserver address to use for the given repo name.
func (c *testGitserverConns) AddrForRepo(ctx context.Context, userAgent string, repo api.RepoName) string {
return c.conns.AddrForRepo(ctx, userAgent, repo)
func (c *testGitserverConns) AddrForRepo(ctx context.Context, repo api.RepoName) string {
return c.conns.AddrForRepo(ctx, repo)
}
// Addresses returns the current list of gitserver addresses.
@ -123,8 +123,8 @@ func (c *testGitserverConns) GetAddressWithClient(addr string) AddressWithClient
}
// ClientForRepo returns a client or host for the given repo name.
func (c *testGitserverConns) ClientForRepo(ctx context.Context, userAgent string, repo api.RepoName) (proto.GitserverServiceClient, error) {
conn, err := c.conns.ConnForRepo(ctx, userAgent, repo)
func (c *testGitserverConns) ClientForRepo(ctx context.Context, repo api.RepoName) (proto.GitserverServiceClient, error) {
conn, err := c.conns.ConnForRepo(ctx, repo)
if err != nil {
return nil, err
}
@ -163,8 +163,8 @@ type GitserverAddresses struct {
}
// AddrForRepo returns the gitserver address to use for the given repo name.
func (g *GitserverAddresses) AddrForRepo(ctx context.Context, userAgent string, repoName api.RepoName) string {
addrForRepoInvoked.WithLabelValues(userAgent).Inc()
func (g *GitserverAddresses) AddrForRepo(ctx context.Context, repoName api.RepoName) string {
addrForRepoInvoked.Inc()
// Normalizing the name in case the caller didn't.
name := string(protocol.NormalizeRepo(repoName))
@ -190,8 +190,8 @@ type GitserverConns struct {
grpcConns map[string]connAndErr
}
func (g *GitserverConns) ConnForRepo(ctx context.Context, userAgent string, repo api.RepoName) (*grpc.ClientConn, error) {
addr := g.AddrForRepo(ctx, userAgent, repo)
func (g *GitserverConns) ConnForRepo(ctx context.Context, repo api.RepoName) (*grpc.ClientConn, error) {
addr := g.AddrForRepo(ctx, repo)
ce, ok := g.grpcConns[addr]
if !ok {
return nil, errors.Newf("no gRPC connection found for address %q", addr)
@ -224,12 +224,12 @@ type atomicGitServerConns struct {
watchOnce sync.Once
}
func (a *atomicGitServerConns) AddrForRepo(ctx context.Context, userAgent string, repo api.RepoName) string {
return a.get().AddrForRepo(ctx, userAgent, repo)
func (a *atomicGitServerConns) AddrForRepo(ctx context.Context, repo api.RepoName) string {
return a.get().AddrForRepo(ctx, repo)
}
func (a *atomicGitServerConns) ClientForRepo(ctx context.Context, userAgent string, repo api.RepoName) (proto.GitserverServiceClient, error) {
conn, err := a.get().ConnForRepo(ctx, userAgent, repo)
func (a *atomicGitServerConns) ClientForRepo(ctx context.Context, repo api.RepoName) (proto.GitserverServiceClient, error) {
conn, err := a.get().ConnForRepo(ctx, repo)
if err != nil {
return nil, err
}

2
internal/gitserver/addrs_test.go
View File

@ -46,7 +46,7 @@ func TestAddrForRepo(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got := ga.AddrForRepo(ctx, "gitserver", tc.repo)
got := ga.AddrForRepo(ctx, tc.repo)
if got != tc.want {
t.Fatalf("Want %q, got %q", tc.want, got)
}

49
internal/gitserver/client.go
View File

@ -7,7 +7,6 @@ import (
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"time"
@ -61,9 +60,9 @@ var _ Client = &clientImplementor{}
// It allows for mocking out the client source in tests.
type ClientSource interface {
// ClientForRepo returns a Client for the given repo.
ClientForRepo(ctx context.Context, userAgent string, repo api.RepoName) (proto.GitserverServiceClient, error)
ClientForRepo(ctx context.Context, repo api.RepoName) (proto.GitserverServiceClient, error)
// AddrForRepo returns the address of the gitserver for the given repo.
AddrForRepo(ctx context.Context, userAgent string, repo api.RepoName) string
AddrForRepo(ctx context.Context, repo api.RepoName) string
// Address the current list of gitserver addresses.
Addresses() []AddressWithClient
// GetAddressWithClient returns the address and client for a gitserver instance.
@ -76,12 +75,8 @@ type ClientSource interface {
func NewClient(scope string) Client {
logger := sglog.Scoped("GitserverClient")
return &clientImplementor{
logger: logger,
scope: scope,
// Use the binary name for userAgent. This should effectively identify
// which service is making the request (excluding requests proxied via the
// frontend internal API)
userAgent: filepath.Base(os.Args[0]),
logger: logger,
scope: scope,
operations: getOperations(),
clientSource: conns,
subRepoPermsChecker: authz.DefaultSubRepoPermsChecker,
@ -93,12 +88,8 @@ func NewTestClient(t testing.TB) TestClient {
logger := logtest.Scoped(t)
return &clientImplementor{
logger: logger,
scope: fmt.Sprintf("gitserver.test.%s", t.Name()),
// Use the binary name for userAgent. This should effectively identify
// which service is making the request (excluding requests proxied via the
// frontend internal API)
userAgent: filepath.Base(os.Args[0]),
logger: logger,
scope: fmt.Sprintf("gitserver.test.%s", t.Name()),
operations: newOperations(observation.ContextWithLogger(logger, &observation.TestContext)),
clientSource: NewTestClientSource(t, nil),
subRepoPermsChecker: authz.DefaultSubRepoPermsChecker,
@ -194,10 +185,6 @@ func NewMockClientWithExecReader(checker authz.SubRepoPermissionChecker, execRea
// clientImplementor is a gitserver client.
type clientImplementor struct {
// userAgent is a string identifying who the client is. It will be logged in
// the telemetry in gitserver.
userAgent string
// the current scope of the client.
scope string
@ -219,7 +206,6 @@ func (c *clientImplementor) Scoped(scope string) Client {
return &clientImplementor{
logger: c.logger,
scope: appendScope(c.scope, scope),
userAgent: c.userAgent,
operations: c.operations,
clientSource: c.clientSource,
}
@ -233,10 +219,27 @@ func appendScope(existing, new string) string {
}
type HunkReader interface {
Read() (*Hunk, error)
Read() (*gitdomain.Hunk, error)
Close() error
}
// BlameOptions configures a blame.
type BlameOptions struct {
NewestCommit api.CommitID `json:",omitempty" url:",omitempty"`
IgnoreWhitespace bool `json:",omitempty" url:",omitempty"`
StartLine int `json:",omitempty" url:",omitempty"` // 1-indexed start line (or 0 for beginning of file)
EndLine int `json:",omitempty" url:",omitempty"` // 1-indexed end line (or 0 for end of file)
}
func (o *BlameOptions) Attrs() []attribute.KeyValue {
return []attribute.KeyValue{
attribute.String("newestCommit", string(o.NewestCommit)),
attribute.Int("startLine", o.StartLine),
attribute.Int("endLine", o.EndLine),
attribute.Bool("ignoreWhitespace", o.IgnoreWhitespace),
}
}
type CommitLog struct {
AuthorEmail string
AuthorName string
@ -530,11 +533,11 @@ func (c *clientImplementor) getDiskInfo(ctx context.Context, addr AddressWithCli
}
func (c *clientImplementor) AddrForRepo(ctx context.Context, repo api.RepoName) string {
return c.clientSource.AddrForRepo(ctx, c.userAgent, repo)
return c.clientSource.AddrForRepo(ctx, repo)
}
func (c *clientImplementor) ClientForRepo(ctx context.Context, repo api.RepoName) (proto.GitserverServiceClient, error) {
return c.clientSource.ClientForRepo(ctx, c.userAgent, repo)
return c.clientSource.ClientForRepo(ctx, repo)
}
// ArchiveOptions contains options for the Archive func.

143
internal/gitserver/commands.go
View File

@ -23,6 +23,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"go.opentelemetry.io/otel/attribute"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/sourcegraph/go-diff/diff"
@ -40,6 +42,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/pointers"
)
type DiffOptions struct {
@ -787,35 +790,6 @@ func (c *clientImplementor) LogReverseEach(ctx context.Context, repo string, com
return errors.Wrap(gitdomain.ParseLogReverseEach(stdout, onLogEntry), "ParseLogReverseEach")
}
// BlameOptions configures a blame.
type BlameOptions struct {
NewestCommit api.CommitID `json:",omitempty" url:",omitempty"`
IgnoreWhitespace bool `json:",omitempty" url:",omitempty"`
StartLine int `json:",omitempty" url:",omitempty"` // 1-indexed start line (or 0 for beginning of file)
EndLine int `json:",omitempty" url:",omitempty"` // 1-indexed end line (or 0 for end of file)
}
func (o *BlameOptions) Attrs() []attribute.KeyValue {
return []attribute.KeyValue{
attribute.String("newestCommit", string(o.NewestCommit)),
attribute.Int("startLine", o.StartLine),
attribute.Int("endLine", o.EndLine),
attribute.Bool("ignoreWhitespace", o.IgnoreWhitespace),
}
}
// A Hunk is a contiguous portion of a file associated with a commit.
type Hunk struct {
StartLine int // 1-indexed start line number
EndLine int // 1-indexed end line number
StartByte int // 0-indexed start byte position (inclusive)
EndByte int // 0-indexed end byte position (exclusive)
api.CommitID
Author gitdomain.Signature
Message string
Filename string
}
// StreamBlameFile returns Git blame information about a file.
func (c *clientImplementor) StreamBlameFile(ctx context.Context, repo api.RepoName, path string, opt *BlameOptions) (_ HunkReader, err error) {
ctx, _, endObservation := c.operations.streamBlameFile.With(ctx, &err, observation.Args{
@ -825,9 +799,58 @@ func (c *clientImplementor) StreamBlameFile(ctx context.Context, repo api.RepoNa
attribute.String("path", path),
}, opt.Attrs()...),
})
defer endObservation(1, observation.Args{})
return streamBlameFileCmd(ctx, c.subRepoPermsChecker, repo, path, opt, c.gitserverGitCommandFunc(repo))
client, err := c.clientSource.ClientForRepo(ctx, repo)
if err != nil {
endObservation(1, observation.Args{})
return nil, err
}
req := &proto.BlameRequest{
RepoName: string(repo),
Path: path,
IgnoreWhitespace: opt.IgnoreWhitespace,
}
if opt.NewestCommit != "" {
req.Commit = pointers.Ptr(string(opt.NewestCommit))
}
if opt.StartLine != 0 {
req.StartLine = pointers.Ptr(uint32(opt.StartLine))
}
if opt.EndLine != 0 {
req.EndLine = pointers.Ptr(uint32(opt.EndLine))
}
ctx, cancel := context.WithCancel(ctx)
cli, err := client.Blame(ctx, req)
if err != nil {
cancel()
endObservation(1, observation.Args{})
return nil, err
}
// We start by reading the first hunk to early-exit on potential errors,
// ie. permission denied errors or invalid git command.
firstHunkResp, err := cli.Recv()
if err != nil {
cancel()
endObservation(1, observation.Args{})
if s, ok := status.FromError(err); ok {
if s.Code() == codes.PermissionDenied {
return nil, errUnauthorizedStreamBlame{Repo: repo}
}
}
return nil, err
}
return &grpcBlameHunkReader{
firstHunk: firstHunkResp.GetHunk(),
c: cli,
cancel: cancel,
endObservation: func() { endObservation(1, observation.Args{}) },
}, nil
}
type errUnauthorizedStreamBlame struct {
@ -842,48 +865,32 @@ func (e errUnauthorizedStreamBlame) Error() string {
return fmt.Sprintf("not authorized (name=%s)", e.Repo)
}
func streamBlameFileCmd(ctx context.Context, checker authz.SubRepoPermissionChecker, repo api.RepoName, path string, opt *BlameOptions, command gitCommandFunc) (HunkReader, error) {
a := actor.FromContext(ctx)
hasAccess, err := authz.FilterActorPath(ctx, checker, a, repo, path)
type grpcBlameHunkReader struct {
firstHunk *proto.BlameHunk
firstHunkRead bool
c proto.GitserverService_BlameClient
cancel context.CancelFunc
endObservation func()
}
func (r *grpcBlameHunkReader) Read() (_ *gitdomain.Hunk, err error) {
if !r.firstHunkRead {
r.firstHunkRead = true
return gitdomain.HunkFromBlameProto(r.firstHunk), nil
}
p, err := r.c.Recv()
if err != nil {
return nil, err
}
if !hasAccess {
return nil, errUnauthorizedStreamBlame{Repo: repo}
}
if opt == nil {
opt = &BlameOptions{}
}
if err := checkSpecArgSafety(string(opt.NewestCommit)); err != nil {
return nil, err
}
args := []string{"blame", "--porcelain", "--incremental"}
if opt.IgnoreWhitespace {
args = append(args, "-w")
}
if opt.StartLine != 0 || opt.EndLine != 0 {
args = append(args, fmt.Sprintf("-L%d,%d", opt.StartLine, opt.EndLine))
}
args = append(args, string(opt.NewestCommit), "--", filepath.ToSlash(path))
rc, err := command(args).StdoutReader(ctx)
if err != nil {
return nil, errors.WithMessage(err, fmt.Sprintf("git command %v failed", args))
}
return newBlameHunkReader(rc), nil
return gitdomain.HunkFromBlameProto(p.GetHunk()), nil
}
func (c *clientImplementor) gitserverGitCommandFunc(repo api.RepoName) gitCommandFunc {
return func(args []string) GitCommand {
return c.gitCommand(repo, args...)
}
func (r *grpcBlameHunkReader) Close() error {
r.cancel()
r.endObservation()
return nil
}
// gitCommandFunc is a func that creates a new executable Git command.
type gitCommandFunc func(args []string) GitCommand
// ResolveRevisionOptions configure how we resolve revisions.
// The zero value should contain appropriate default values.
type ResolveRevisionOptions struct {
@ -1238,7 +1245,7 @@ func (c *clientImplementor) MergeBase(ctx context.Context, repo api.RepoName, ba
})
defer endObservation(1, observation.Args{})
client, err := c.clientSource.ClientForRepo(ctx, c.userAgent, repo)
client, err := c.clientSource.ClientForRepo(ctx, repo)
if err != nil {
return "", err
}
@ -2282,7 +2289,7 @@ func (c *clientImplementor) ArchiveReader(
return nil, err
}
client, err := c.clientSource.ClientForRepo(ctx, c.userAgent, repo)
client, err := c.clientSource.ClientForRepo(ctx, repo)
if err != nil {
return nil, err
}

318
internal/gitserver/commands_test.go
View File

@ -16,6 +16,9 @@ import (
"time"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/google/go-cmp/cmp"
godiff "github.com/sourcegraph/go-diff/diff"
@ -26,6 +29,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/errcode"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -2426,278 +2430,58 @@ func usePermissionsForFilePermissionsFunc(m *authz.MockSubRepoPermissionChecker)
})
}
// testGitBlameOutputIncremental is produced by running
//
// git blame -w --porcelain release.sh
//
// `sourcegraph/src-cli`
var testGitBlameOutputIncremental = `8a75c6f8b4cbe2a2f3c8be0f2c50bc766499f498 15 15 1
author Adam Harvey
author-mail <adam@adamharvey.name>
author-time 1660860583
author-tz -0700
committer GitHub
committer-mail <noreply@github.com>
committer-time 1660860583
committer-tz +0000
summary release.sh: allow -rc.X suffixes (#829)
previous e6e03e850770dd0ba745f0fa4b23127e9d72ad30 release.sh
filename release.sh
fbb98e0b7ff0752798463d9f49d922858a4188f6 5 5 10
author Adam Harvey
author-mail <aharvey@sourcegraph.com>
author-time 1602630694
author-tz -0700
committer GitHub
committer-mail <noreply@github.com>
committer-time 1602630694
committer-tz -0700
summary release: add a prompt about DEVELOPMENT.md (#349)
previous 18f59760f4260518c29f0f07056245ed5d1d0f08 release.sh
filename release.sh
67b7b725a7ff913da520b997d71c840230351e30 10 20 1
author Thorsten Ball
author-mail <mrnugget@gmail.com>
author-time 1600334460
author-tz +0200
committer Thorsten Ball
committer-mail <mrnugget@gmail.com>
committer-time 1600334460
committer-tz +0200
summary Fix goreleaser GitHub action setup and release script
previous 6e931cc9745502184ce32d48b01f9a8706a4dfe8 release.sh
filename release.sh
67b7b725a7ff913da520b997d71c840230351e30 12 22 2
previous 6e931cc9745502184ce32d48b01f9a8706a4dfe8 release.sh
filename release.sh
3f61310114082d6179c23f75950b88d1842fe2de 1 1 4
author Thorsten Ball
author-mail <mrnugget@gmail.com>
author-time 1592827635
author-tz +0200
committer GitHub
committer-mail <noreply@github.com>
committer-time 1592827635
committer-tz +0200
summary Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)
previous ec809e79094cbcd05825446ee14c6d072466a0b7 release.sh
filename release.sh
3f61310114082d6179c23f75950b88d1842fe2de 6 16 4
previous ec809e79094cbcd05825446ee14c6d072466a0b7 release.sh
filename release.sh
3f61310114082d6179c23f75950b88d1842fe2de 10 21 1
previous ec809e79094cbcd05825446ee14c6d072466a0b7 release.sh
filename release.sh
`
// This test-data includes the boundary keyword, which is not present in the previous one.
var testGitBlameOutputIncremental2 = `bbca6551549492486ca1b0f8dee45553dd6aa6d7 16 16 1
author French Ben
author-mail <frenchben@docker.com>
author-time 1517407262
author-tz +0100
committer French Ben
committer-mail <frenchben@docker.com>
committer-time 1517407262
committer-tz +0100
summary Update error output to be clean
previous b7773ae218740a7be65057fc60b366a49b538a44 format.go
filename format.go
bbca6551549492486ca1b0f8dee45553dd6aa6d7 25 25 2
previous b7773ae218740a7be65057fc60b366a49b538a44 format.go
filename format.go
2c87fda17de1def6ea288141b8e7600b888e535b 15 15 1
author David Tolnay
author-mail <dtolnay@gmail.com>
author-time 1478451741
author-tz -0800
committer David Tolnay
committer-mail <dtolnay@gmail.com>
committer-time 1478451741
committer-tz -0800
summary Singular message for a single error
previous 8c5f0ad9360406a3807ce7de6bc73269a91a6e51 format.go
filename format.go
2c87fda17de1def6ea288141b8e7600b888e535b 17 17 2
previous 8c5f0ad9360406a3807ce7de6bc73269a91a6e51 format.go
filename format.go
31fee45604949934710ada68f0b307c4726fb4e8 1 1 14
author Mitchell Hashimoto
author-mail <mitchell.hashimoto@gmail.com>
author-time 1418673320
author-tz -0800
committer Mitchell Hashimoto
committer-mail <mitchell.hashimoto@gmail.com>
committer-time 1418673320
committer-tz -0800
summary Initial commit
boundary
filename format.go
31fee45604949934710ada68f0b307c4726fb4e8 15 19 6
filename format.go
31fee45604949934710ada68f0b307c4726fb4e8 23 27 1
filename format.go
`
var testGitBlameOutputHunks = []*Hunk{
{
StartLine: 1, EndLine: 5, StartByte: 0, EndByte: 41,
CommitID: "3f61310114082d6179c23f75950b88d1842fe2de",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: MustParseTime(time.RFC3339, "2020-06-22T12:07:15Z"),
},
Message: "Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)",
Filename: "release.sh",
},
{
StartLine: 5, EndLine: 15, StartByte: 41, EndByte: 249,
CommitID: "fbb98e0b7ff0752798463d9f49d922858a4188f6",
Author: gitdomain.Signature{
Name: "Adam Harvey",
Email: "aharvey@sourcegraph.com",
Date: MustParseTime(time.RFC3339, "2020-10-13T23:11:34Z"),
},
Message: "release: add a prompt about DEVELOPMENT.md (#349)",
Filename: "release.sh",
},
{
StartLine: 15, EndLine: 16, StartByte: 249, EndByte: 328,
CommitID: "8a75c6f8b4cbe2a2f3c8be0f2c50bc766499f498",
Author: gitdomain.Signature{
Name: "Adam Harvey",
Email: "adam@adamharvey.name",
Date: MustParseTime(time.RFC3339, "2022-08-18T22:09:43Z"),
},
Message: "release.sh: allow -rc.X suffixes (#829)",
Filename: "release.sh",
},
{
StartLine: 16, EndLine: 20, StartByte: 328, EndByte: 394,
CommitID: "3f61310114082d6179c23f75950b88d1842fe2de",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: MustParseTime(time.RFC3339, "2020-06-22T12:07:15Z"),
},
Message: "Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)",
Filename: "release.sh",
},
{
StartLine: 20, EndLine: 21, StartByte: 394, EndByte: 504,
CommitID: "67b7b725a7ff913da520b997d71c840230351e30",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: MustParseTime(time.RFC3339, "2020-09-17T09:21:00Z"),
},
Message: "Fix goreleaser GitHub action setup and release script",
Filename: "release.sh",
},
{
StartLine: 21, EndLine: 22, StartByte: 504, EndByte: 553,
CommitID: "3f61310114082d6179c23f75950b88d1842fe2de",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: MustParseTime(time.RFC3339, "2020-06-22T12:07:15Z"),
},
Message: "Check that $VERSION is in MAJOR.MINOR.PATCH format in release.sh (#227)",
Filename: "release.sh",
},
{
StartLine: 22, EndLine: 24, StartByte: 553, EndByte: 695,
CommitID: "67b7b725a7ff913da520b997d71c840230351e30",
Author: gitdomain.Signature{
Name: "Thorsten Ball",
Email: "mrnugget@gmail.com",
Date: MustParseTime(time.RFC3339, "2020-09-17T09:21:00Z"),
},
Message: "Fix goreleaser GitHub action setup and release script",
Filename: "release.sh",
},
}
func TestStreamBlameFile(t *testing.T) {
t.Run("NOK unauthorized", func(t *testing.T) {
ctx := actor.WithActor(context.Background(), &actor.Actor{
UID: 1,
func TestClient_StreamBlameFile(t *testing.T) {
t.Run("firstChunk memoization", func(t *testing.T) {
source := NewTestClientSource(t, []string{"gitserver"}, func(o *TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
c := NewMockGitserverServiceClient()
bc := NewMockGitserverService_BlameClient()
bc.RecvFunc.PushReturn(&proto.BlameResponse{Hunk: &proto.BlameHunk{Commit: "deadbeef"}}, nil)
bc.RecvFunc.PushReturn(&proto.BlameResponse{Hunk: &proto.BlameHunk{Commit: "deadbeef2"}}, nil)
bc.RecvFunc.PushReturn(nil, io.EOF)
c.BlameFunc.SetDefaultReturn(bc, nil)
return c
}
})
checker := authz.NewMockSubRepoPermissionChecker()
checker.EnabledFunc.SetDefaultHook(func() bool {
return true
})
// User doesn't have access to this file
checker.PermissionsFunc.SetDefaultHook(func(ctx context.Context, i int32, content authz.RepoContent) (authz.Perms, error) {
return authz.None, nil
})
hr, err := streamBlameFileCmd(ctx, checker, "foobar", "README.md", nil, func(_ []string) GitCommand { return nil })
if hr != nil {
t.Fatalf("expected nil HunkReader")
}
if err == nil {
t.Fatalf("expected an error to be returned")
}
if !errcode.IsUnauthorized(err) {
t.Fatalf("expected err to be an authorization error, got %v", err)
}
c := NewTestClient(t).WithClientSource(source)
hr, err := c.StreamBlameFile(context.Background(), "repo", "file", &BlameOptions{})
require.NoError(t, err)
// This chunk comes from the memoized first message.
h, err := hr.Read()
require.NoError(t, err)
require.Equal(t, h.CommitID, api.CommitID("deadbeef"))
// This chunk is returned from Recv inside the hunk reader.
h, err = hr.Read()
require.NoError(t, err)
require.Equal(t, h.CommitID, api.CommitID("deadbeef2"))
// Done.
_, err = hr.Read()
require.Error(t, err)
require.Equal(t, io.EOF, err)
require.NoError(t, hr.Close())
})
}
func TestBlameHunkReader(t *testing.T) {
t.Run("OK matching hunks", func(t *testing.T) {
rc := io.NopCloser(strings.NewReader(testGitBlameOutputIncremental))
reader := newBlameHunkReader(rc)
defer reader.Close()
hunks := []*Hunk{}
for {
hunk, err := reader.Read()
if errors.Is(err, io.EOF) {
break
} else if err != nil {
t.Fatalf("blameHunkReader.Read failed: %s", err)
t.Run("permission errors are returned early", func(t *testing.T) {
source := NewTestClientSource(t, []string{"gitserver"}, func(o *TestClientSourceOptions) {
o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient {
c := NewMockGitserverServiceClient()
bc := NewMockGitserverService_BlameClient()
bc.RecvFunc.PushReturn(nil, status.New(codes.PermissionDenied, "bad actor").Err())
c.BlameFunc.SetDefaultReturn(bc, nil)
return c
}
hunks = append(hunks, hunk)
}
})
sortFn := func(x []*Hunk) func(i, j int) bool {
return func(i, j int) bool {
return x[i].Author.Date.After(x[j].Author.Date)
}
}
c := NewTestClient(t).WithClientSource(source)
// We're not giving back bytes, as the output of --incremental only gives back annotations.
expectedHunks := make([]*Hunk, 0, len(testGitBlameOutputHunks))
for _, h := range testGitBlameOutputHunks {
dup := *h
dup.EndByte = 0
dup.StartByte = 0
expectedHunks = append(expectedHunks, &dup)
}
// Sort expected hunks by the most recent first, as --incremental does.
sort.SliceStable(expectedHunks, sortFn(expectedHunks))
if d := cmp.Diff(expectedHunks, hunks); d != "" {
t.Fatalf("unexpected hunks (-want, +got):\n%s", d)
}
})
t.Run("OK parsing hunks", func(t *testing.T) {
rc := io.NopCloser(strings.NewReader(testGitBlameOutputIncremental2))
reader := newBlameHunkReader(rc)
defer reader.Close()
for {
_, err := reader.Read()
if errors.Is(err, io.EOF) {
break
} else if err != nil {
t.Fatalf("blameHunkReader.Read failed: %s", err)
}
}
_, err := c.StreamBlameFile(context.Background(), "repo", "file", &BlameOptions{})
require.Error(t, err)
require.True(t, errcode.IsUnauthorized(err))
})
}

1
internal/gitserver/gitdomain/BUILD.bazel
View File

@ -17,6 +17,7 @@ go_library(
"//internal/lazyregexp",
"//lib/errors",
"@com_github_gobwas_glob//:glob",
"@org_golang_google_protobuf//types/known/timestamppb",
],
)

55
internal/gitserver/gitdomain/common.go
View File

@ -8,6 +8,7 @@ import (
"time"
"github.com/gobwas/glob"
"google.golang.org/protobuf/types/known/timestamppb"
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
@ -181,6 +182,60 @@ func (m Message) Body() string {
return strings.TrimSpace(message[i:])
}
// A Hunk is a contiguous portion of a file associated with a commit.
type Hunk struct {
StartLine uint32 // 1-indexed start line number
EndLine uint32 // 1-indexed end line number
StartByte uint32 // 0-indexed start byte position (inclusive)
EndByte uint32 // 0-indexed end byte position (exclusive)
CommitID api.CommitID
Author Signature
Message string
Filename string
}
func HunkFromBlameProto(h *proto.BlameHunk) *Hunk {
if h == nil {
return nil
}
return &Hunk{
StartLine: h.GetStartLine(),
EndLine: h.GetEndLine(),
StartByte: h.GetStartByte(),
EndByte: h.GetEndByte(),
CommitID: api.CommitID(h.GetCommit()),
Message: h.GetMessage(),
Filename: h.GetFilename(),
Author: Signature{
Name: h.GetAuthor().GetName(),
Email: h.GetAuthor().GetEmail(),
Date: h.GetAuthor().GetDate().AsTime(),
},
}
}
func (h *Hunk) ToProto() *proto.BlameHunk {
if h == nil {
return nil
}
return &proto.BlameHunk{
StartLine: uint32(h.StartLine),
EndLine: uint32(h.EndLine),
StartByte: uint32(h.StartByte),
EndByte: uint32(h.EndByte),
Commit: string(h.CommitID),
Message: h.Message,
Filename: h.Filename,
Author: &proto.BlameAuthor{
Name: h.Author.Name,
Email: h.Author.Email,
Date: timestamppb.New(h.Author.Date),
},
}
}
// Signature represents a commit signature
type Signature struct {
Name string `json:"Name,omitempty"`

17
internal/gitserver/gitdomain/common_test.go
View File

@ -109,3 +109,20 @@ func TestIsAbsoluteRevision(t *testing.T) {
}
}
}
// TODO: Fails because a time.Time is embedded in the Hunk type.
// func TestRoundTripBlameHunk(t *testing.T) {
// diff := ""
// err := quick.Check(func(original *Hunk) bool {
// converted := HunkFromBlameProto(original.ToProto())
// if diff = cmp.Diff(original, converted); diff != "" {
// return false
// }
// return true
// }, nil)
// if err != nil {
// t.Fatalf("unexpected diff (-want +got):\n%s", diff)
// }
// }

4
internal/gitserver/grpc_test.go
View File

@ -29,8 +29,8 @@ func TestClientSource_AddrMatchesTarget(t *testing.T) {
ctx := context.Background()
for _, repo := range []api.RepoName{"a", "b", "c", "d"} {
addr := source.AddrForRepo(ctx, "test", repo)
conn, err := conns.ConnForRepo(ctx, "test", repo)
addr := source.AddrForRepo(ctx, repo)
conn, err := conns.ConnForRepo(ctx, repo)
if err != nil {
t.Fatal(err)
}

1879
internal/gitserver/mock.go
View File

File diff suppressed because it is too large Load Diff

5
internal/gitserver/retry.go
View File

@ -146,4 +146,9 @@ func (r *automaticRetryClient) MergeBase(ctx context.Context, in *proto.MergeBas
return r.base.MergeBase(ctx, in, opts...)
}
func (r *automaticRetryClient) Blame(ctx context.Context, in *proto.BlameRequest, opts ...grpc.CallOption) (proto.GitserverService_BlameClient, error) {
opts = append(defaults.RetryPolicy, opts...)
return r.base.Blame(ctx, in, opts...)
}
var _ proto.GitserverServiceClient = &automaticRetryClient{}

2929
internal/gitserver/v1/gitserver.pb.go generated
View File

File diff suppressed because it is too large Load Diff

55
internal/gitserver/v1/gitserver.proto
View File

@ -77,6 +77,52 @@ service GitserverService {
rpc MergeBase(MergeBaseRequest) returns (MergeBaseResponse) {
option idempotency_level = NO_SIDE_EFFECTS;
}
// Blame runs a blame operation on the specified file. It returns a stream of
// hunks as they are found. The --incremental flag is used on the git CLI level
// to achieve this behavior.
// The endpoint will verify that the user is allowed to blame the given file
// if subrepo permissions are enabled for the repo. If access is denied, an error
// with a UnauthorizedPayload in the details is returned.
//
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a NotFoundPayload in the details.
rpc Blame(BlameRequest) returns (stream BlameResponse) {
option idempotency_level = NO_SIDE_EFFECTS;
}
}
message BlameRequest {
// repo_name is the name of the repo to run the blame operation in.
// Note: We use field ID 2 here to reserve 1 for a future repo int32 field.
string repo_name = 2;
// commit is the commit to start the blame operation in. If not given, the latest
// HEAD is used.
optional string commit = 3;
string path = 4;
bool ignore_whitespace = 5;
optional uint32 start_line = 6;
optional uint32 end_line = 7;
}
message BlameResponse {
BlameHunk hunk = 1;
}
message BlameHunk {
uint32 start_line = 1;
uint32 end_line = 2;
uint32 start_byte = 3;
uint32 end_byte = 4;
string commit = 5;
BlameAuthor author = 6;
string message = 7;
string filename = 8;
}
message BlameAuthor {
string name = 1;
string email = 2;
google.protobuf.Timestamp date = 3;
}
// DiskInfoRequest is a empty request for the DiskInfo RPC.
@ -241,6 +287,15 @@ message ExecStatusPayload {
string stderr = 2;
}
// UnauthorizedPayload is the payload returned when an actor cannot access
// a commit or file due to subrepo permissions.
message UnauthorizedPayload {
// Note: We use field ID 2 here to reserve 1 for a future repo int32 field.
string repo_name = 2;
optional string path = 3;
optional string commit = 4;
}
message SearchRequest {
// repo is the name of the repo to be searched
string repo = 1;

82
internal/gitserver/v1/gitserver_grpc.pb.go generated
View File

@ -42,6 +42,7 @@ const (
GitserverService_IsPerforceSuperUser_FullMethodName = "/gitserver.v1.GitserverService/IsPerforceSuperUser"
GitserverService_PerforceGetChangelist_FullMethodName = "/gitserver.v1.GitserverService/PerforceGetChangelist"
GitserverService_MergeBase_FullMethodName = "/gitserver.v1.GitserverService/MergeBase"
GitserverService_Blame_FullMethodName = "/gitserver.v1.GitserverService/Blame"
)
// GitserverServiceClient is the client API for GitserverService service.
@ -78,6 +79,16 @@ type GitserverServiceClient interface {
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a NotFoundPayload in the details.
MergeBase(ctx context.Context, in *MergeBaseRequest, opts ...grpc.CallOption) (*MergeBaseResponse, error)
// Blame runs a blame operation on the specified file. It returns a stream of
// hunks as they are found. The --incremental flag is used on the git CLI level
// to achieve this behavior.
// The endpoint will verify that the user is allowed to blame the given file
// if subrepo permissions are enabled for the repo. If access is denied, an error
// with a UnauthorizedPayload in the details is returned.
//
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a NotFoundPayload in the details.
Blame(ctx context.Context, in *BlameRequest, opts ...grpc.CallOption) (GitserverService_BlameClient, error)
}
type gitserverServiceClient struct {
@ -414,6 +425,38 @@ func (c *gitserverServiceClient) MergeBase(ctx context.Context, in *MergeBaseReq
return out, nil
}
func (c *gitserverServiceClient) Blame(ctx context.Context, in *BlameRequest, opts ...grpc.CallOption) (GitserverService_BlameClient, error) {
stream, err := c.cc.NewStream(ctx, &GitserverService_ServiceDesc.Streams[5], GitserverService_Blame_FullMethodName, opts...)
if err != nil {
return nil, err
}
x := &gitserverServiceBlameClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type GitserverService_BlameClient interface {
Recv() (*BlameResponse, error)
grpc.ClientStream
}
type gitserverServiceBlameClient struct {
grpc.ClientStream
}
func (x *gitserverServiceBlameClient) Recv() (*BlameResponse, error) {
m := new(BlameResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
// GitserverServiceServer is the server API for GitserverService service.
// All implementations must embed UnimplementedGitserverServiceServer
// for forward compatibility
@ -448,6 +491,16 @@ type GitserverServiceServer interface {
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a NotFoundPayload in the details.
MergeBase(context.Context, *MergeBaseRequest) (*MergeBaseResponse, error)
// Blame runs a blame operation on the specified file. It returns a stream of
// hunks as they are found. The --incremental flag is used on the git CLI level
// to achieve this behavior.
// The endpoint will verify that the user is allowed to blame the given file
// if subrepo permissions are enabled for the repo. If access is denied, an error
// with a UnauthorizedPayload in the details is returned.
//
// If the given repo is not cloned, it will be enqueued for cloning and a NotFound
// error will be returned, with a NotFoundPayload in the details.
Blame(*BlameRequest, GitserverService_BlameServer) error
mustEmbedUnimplementedGitserverServiceServer()
}
@ -524,6 +577,9 @@ func (UnimplementedGitserverServiceServer) PerforceGetChangelist(context.Context
func (UnimplementedGitserverServiceServer) MergeBase(context.Context, *MergeBaseRequest) (*MergeBaseResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method MergeBase not implemented")
}
func (UnimplementedGitserverServiceServer) Blame(*BlameRequest, GitserverService_BlameServer) error {
return status.Errorf(codes.Unimplemented, "method Blame not implemented")
}
func (UnimplementedGitserverServiceServer) mustEmbedUnimplementedGitserverServiceServer() {}
// UnsafeGitserverServiceServer may be embedded to opt out of forward compatibility for this service.
@ -971,6 +1027,27 @@ func _GitserverService_MergeBase_Handler(srv interface{}, ctx context.Context, d
return interceptor(ctx, in, info, handler)
}
func _GitserverService_Blame_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(BlameRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(GitserverServiceServer).Blame(m, &gitserverServiceBlameServer{stream})
}
type GitserverService_BlameServer interface {
Send(*BlameResponse) error
grpc.ServerStream
}
type gitserverServiceBlameServer struct {
grpc.ServerStream
}
func (x *gitserverServiceBlameServer) Send(m *BlameResponse) error {
return x.ServerStream.SendMsg(m)
}
// GitserverService_ServiceDesc is the grpc.ServiceDesc for GitserverService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -1077,6 +1154,11 @@ var GitserverService_ServiceDesc = grpc.ServiceDesc{
Handler: _GitserverService_P4Exec_Handler,
ServerStreams: true,
},
{
StreamName: "Blame",
Handler: _GitserverService_Blame_Handler,
ServerStreams: true,
},
},
Metadata: "gitserver.proto",
}

3
mockgen.temp.yaml
View File

@ -137,8 +137,11 @@
interfaces:
- GitserverServiceClient
- GitserverService_ExecServer
- GitserverService_BlameServer
- GitserverService_BlameClient
- filename: cmd/gitserver/internal/git/mock.go
path: github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git
interfaces:
- GitBackend
- GitConfigBackend
- BlameHunkReader

6
mockgen.test.yaml
View File

@ -415,3 +415,9 @@
interfaces:
- RepositoryLocker
- RepositoryLock
- filename: cmd/gitserver/internal/mocks_test.go
package: internal
sources:
- path: github.com/sourcegraph/sourcegraph/cmd/gitserver/internal
interfaces:
- service