artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 17:31:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: gitserver.NewClient migration to add db everywhere (#54867)

Browse Source 
Follow up and stacked on #54850.

**Highlight of this change**

Searcher now initiates a connection to the DB. There's a [separate
commit](a90cd139f8)
just for this change to make it easier to review.

Deleted a bunch of code in search/decorate.go and related tests because it
was not being used.

Part of #54741. 

Co-authored-by: Erik Seliger <erikseliger@me.com>
This commit is contained in:
Indradhanush Gupta 2023-07-13 22:07:33 +05:30 committed by GitHub
parent 6daa5fc2c8
commit dc0dfd1178
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
58 changed files with 308 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/frontend/graphqlbackend/repositories.go
View File

@ -218,7 +218,7 @@ func (s *repositoriesConnectionStore) ComputeNodes(ctx context.Context, args *da
opt := s.opt
opt.PaginationArgs = args
client := gitserver.NewClientDeprecatedNeedsDB()
client := gitserver.NewClient(s.db)
repos, err := backend.NewRepos(s.logger, s.db, client).List(ctx, opt)
if err != nil {
return nil, err
@ -316,7 +316,7 @@ func (r *repositoryConnectionResolver) Nodes(ctx context.Context) ([]*Repository
return nil, err
}
resolvers := make([]*RepositoryResolver, 0, len(repos))
client := gitserver.NewClientDeprecatedNeedsDB()
client := gitserver.NewClient(r.db)
for i, repo := range repos {
if r.opt.LimitOffset != nil && i == r.opt.Limit {
break

2
cmd/frontend/graphqlbackend/repository_contributors.go
View File

@ -92,7 +92,7 @@ func (s *repositoryContributorConnectionStore) ComputeNodes(ctx context.Context,
func (s *repositoryContributorConnectionStore) compute(ctx context.Context) ([]*gitdomain.ContributorCount, error) {
s.once.Do(func() {
client := gitserver.NewClientDeprecatedNeedsDB()
client := gitserver.NewClient(s.db)
var opt gitserver.ContributorOptions
if s.args.RevisionRange != nil {
opt.Range = *s.args.RevisionRange

4
cmd/frontend/graphqlbackend/repository_git_refs.go
View File

@ -37,7 +37,7 @@ func (r *RepositoryResolver) GitRefs(ctx context.Context, args *refsArgs) (*gitR
var branches []*gitdomain.Branch
if args.Type == nil || *args.Type == gitRefTypeBranch {
var err error
branches, err = gitserver.NewClientDeprecatedNeedsDB().ListBranches(ctx, r.RepoName(), gitserver.BranchesOptions{
branches, err = gitserver.NewClient(r.db).ListBranches(ctx, r.RepoName(), gitserver.BranchesOptions{
// We intentionally do not ask for commits here since it requires
// a separate git call per branch. We only need the git commits to
// sort by author/commit date and there are few enough branches to
@ -104,7 +104,7 @@ func (r *RepositoryResolver) GitRefs(ctx context.Context, args *refsArgs) (*gitR
var tags []*gitdomain.Tag
if args.Type == nil || *args.Type == gitRefTypeTag {
var err error
tags, err = gitserver.NewClientDeprecatedNeedsDB().ListTags(ctx, r.RepoName())
tags, err = gitserver.NewClient(r.db).ListTags(ctx, r.RepoName())
if err != nil {
return nil, err
}

2
cmd/frontend/graphqlbackend/repository_stats.go
View File

@ -38,7 +38,7 @@ func (r *repositoryStatsResolver) GitDirBytes(ctx context.Context) (BigInt, erro
func (r *repositoryStatsResolver) computeGitDirBytes(ctx context.Context) (int64, error) {
r.gitDirBytesOnce.Do(func() {
stats, err := gitserver.NewClientDeprecatedNeedsDB().ReposStats(ctx)
stats, err := gitserver.NewClient(r.db).ReposStats(ctx)
if err != nil {
r.gitDirBytesErr = err
return

6
cmd/frontend/graphqlbackend/search_results.go
View File

@ -79,7 +79,7 @@ func (c *SearchResultsResolver) repositoryResolvers(ctx context.Context, ids []a
return nil, nil
}
gsClient := gitserver.NewClientDeprecatedNeedsDB()
gsClient := gitserver.NewClient(c.db)
resolvers := make([]*RepositoryResolver, 0, len(ids))
err := c.db.Repos().StreamMinimalRepos(ctx, database.ReposListOptions{
IDs: ids,
@ -134,7 +134,7 @@ func matchesToResolvers(db database.DB, matches []result.Match) []SearchResultRe
Rev string
}
repoResolvers := make(map[repoKey]*RepositoryResolver, 10)
gsClient := gitserver.NewClientDeprecatedNeedsDB()
gsClient := gitserver.NewClient(db)
getRepoResolver := func(repoName types.MinimalRepo, rev string) *RepositoryResolver {
if existing, ok := repoResolvers[repoKey{repoName, rev}]; ok {
return existing
@ -244,7 +244,7 @@ func (sr *SearchResultsResolver) blameFileMatch(ctx context.Context, fm *result.
return time.Time{}, nil
}
hm := fm.ChunkMatches[0]
hunks, err := gitserver.NewClientDeprecatedNeedsDB().BlameFile(ctx, authz.DefaultSubRepoPermsChecker, fm.Repo.Name, fm.Path, &gitserver.BlameOptions{
hunks, err := gitserver.NewClient(sr.db).BlameFile(ctx, authz.DefaultSubRepoPermsChecker, fm.Repo.Name, fm.Path, &gitserver.BlameOptions{
NewestCommit: fm.CommitID,
StartLine: hm.Ranges[0].Start.Line,
EndLine: hm.Ranges[0].Start.Line,

2
cmd/frontend/graphqlbackend/site_config_change_test.go
View File

@ -16,7 +16,7 @@ func TestSiteConfigurationDiff(t *testing.T) {
stubs := setupSiteConfigStubs(t)
ctx := actor.WithActor(context.Background(), &actor.Actor{UID: stubs.users[0].ID})
schemaResolver, err := newSchemaResolver(stubs.db, gitserver.NewClientDeprecatedNeedsDB()).Site().Configuration(ctx, &SiteConfigurationArgs{})
schemaResolver, err := newSchemaResolver(stubs.db, gitserver.NewClient(stubs.db)).Site().Configuration(ctx, &SiteConfigurationArgs{})
if err != nil {
t.Fatalf("failed to create schemaResolver: %v", err)
}

2
cmd/frontend/graphqlbackend/site_test.go
View File

@ -162,7 +162,7 @@ func TestSiteConfigurationHistory(t *testing.T) {
stubs := setupSiteConfigStubs(t)
ctx := actor.WithActor(context.Background(), &actor.Actor{UID: stubs.users[0].ID})
schemaResolver, err := newSchemaResolver(stubs.db, gitserver.NewClientDeprecatedNeedsDB()).Site().Configuration(ctx, &SiteConfigurationArgs{})
schemaResolver, err := newSchemaResolver(stubs.db, gitserver.NewClient(stubs.db)).Site().Configuration(ctx, &SiteConfigurationArgs{})
if err != nil {
t.Fatalf("failed to create schemaResolver: %v", err)
}

2
cmd/frontend/graphqlbackend/user_collaborators.go
View File

@ -29,7 +29,7 @@ func (r *UserResolver) InvitableCollaborators(ctx context.Context) ([]*invitable
// We'll search for collaborators in 25 of the user's most-starred repositories.
const maxReposToScan = 25
db := r.db
gsClient := gitserver.NewClientDeprecatedNeedsDB()
gsClient := gitserver.NewClient(db)
pickedRepos, err := backend.NewRepos(r.logger, db, gsClient).List(ctx, database.ReposListOptions{
// SECURITY: This must be the authenticated user's ID.
UserID: a.UID,

2
cmd/frontend/internal/cli/serve_cmd.go
View File

@ -219,7 +219,7 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
schema, err := graphqlbackend.NewSchema(
db,
gitserver.NewClientDeprecatedNeedsDB(),
gitserver.NewClient(db),
[]graphqlbackend.OptionalResolver{enterpriseServices.OptionalResolver},
)
if err != nil {

2
cmd/frontend/internal/handlerutil/repo.go
View File

@ -37,7 +37,7 @@ func GetRepo(ctx context.Context, logger log.Logger, db database.DB, vars map[st
// getRepoRev resolves the repository and commit specified in the route vars.
func getRepoRev(ctx context.Context, logger log.Logger, db database.DB, vars map[string]string, repoID api.RepoID) (api.RepoID, api.CommitID, error) {
repoRev := routevar.ToRepoRev(vars)
gsClient := gitserver.NewClientDeprecatedNeedsDB()
gsClient := gitserver.NewClient(db)
repo, err := backend.NewRepos(logger, db, gsClient).Get(ctx, repoID)
if err != nil {
return repoID, "", err

4
cmd/frontend/internal/httpapi/httpapi.go
View File

@ -173,7 +173,7 @@ func NewHandler(
// Return the minimum src-cli version that's compatible with this instance
m.Get(apirouter.SrcCli).Handler(trace.Route(newSrcCliVersionHandler(logger)))
gsClient := gitserver.NewClientDeprecatedNeedsDB()
gsClient := gitserver.NewClient(db)
m.Get(apirouter.GitBlameStream).Handler(trace.Route(handleStreamBlame(logger, db, gsClient)))
// Set up the src-cli version cache handler (this will effectively be a
@ -219,7 +219,7 @@ func RegisterInternalServices(
m.Get(apirouter.ExternalServiceConfigs).Handler(trace.Route(handler(serveExternalServiceConfigs(db))))
// zoekt-indexserver endpoints
gsClient := gitserver.NewClientDeprecatedNeedsDB()
gsClient := gitserver.NewClient(db)
indexer := &searchIndexerServer{
db: db,
logger: logger.Scoped("searchIndexerServer", "zoekt-indexserver endpoints"),

11
cmd/frontend/internal/search/BUILD.bazel
View File

@ -4,7 +4,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "search",
srcs = [
"decorate.go",
"event_writer.go",
"metadata.go",
"search.go",
@ -14,11 +13,8 @@ go_library(
deps = [
"//cmd/frontend/internal/search/logs",
"//internal/api",
"//internal/authz",
"//internal/conf",
"//internal/database",
"//internal/gitserver",
"//internal/highlight",
"//internal/honey",
"//internal/honey/search",
"//internal/lazyregexp",
@ -33,7 +29,6 @@ go_library(
"//internal/types",
"//lib/errors",
"//lib/pointers",
"@com_github_inconshreveable_log15//:log15",
"@com_github_prometheus_client_golang//prometheus",
"@com_github_prometheus_client_golang//prometheus/promauto",
"@com_github_sourcegraph_log//:log",
@ -44,10 +39,7 @@ go_library(
go_test(
name = "search_test",
timeout = "short",
srcs = [
"decorate_test.go",
"search_test.go",
],
srcs = ["search_test.go"],
embed = [":search"],
deps = [
"//internal/api",
@ -62,7 +54,6 @@ go_test(
"//internal/settings",
"//internal/types",
"//schema",
"@com_github_google_go_cmp//cmp",
"@com_github_sourcegraph_log//logtest",
"@com_github_stretchr_testify//require",
"@org_golang_x_sync//errgroup",

176
cmd/frontend/internal/search/decorate.go
View File

@ -1,176 +0,0 @@
package search
import (
"context"
"fmt"
"sort"
"strings"
"github.com/inconshreveable/log15"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/highlight"
"github.com/sourcegraph/sourcegraph/internal/search/result"
stream "github.com/sourcegraph/sourcegraph/internal/search/streaming/http"
)
// segmentToRangs converts line match ranges into absolute ranges.
func segmentToRanges(lineNumber int, segments [][2]int32) []stream.Range {
ranges := make([]stream.Range, 0, len(segments))
for _, segment := range segments {
ranges = append(ranges, stream.Range{
Start: stream.Location{
Offset: -1,
Line: lineNumber,
Column: int(segment[0]),
},
End: stream.Location{
Offset: -1,
Line: lineNumber,
Column: int(segment[0]) + int(segment[1]),
},
})
}
return ranges
}
// group is a list of contiguous line matches by line number.
type group []*result.LineMatch
// toMatch converts a group of line matches to absolute match ranges in the file. These ranges
// specify matched content to emphasize specially (e.g., with overlay-highlights) within the file.
func toMatchRanges(group group) []stream.Range {
matches := make([]stream.Range, 0, len(group))
for _, line := range group {
matches = append(matches, segmentToRanges(int(line.LineNumber), line.OffsetAndLengths)...)
}
return matches
}
// groupLineMatches converts a flat slice of line matches to groups of
// contiguous line matches based on line numbers.
func groupLineMatches(lineMatches []*result.LineMatch) []group {
var groups []group
var previousLine *result.LineMatch
var currentGroup group
for _, line := range lineMatches {
if previousLine == nil {
previousLine = line
}
if len(currentGroup) == 0 {
currentGroup = append(currentGroup, line)
// Invariant: previousLine is set to first line match.
continue
}
if line.LineNumber-1 == previousLine.LineNumber {
currentGroup = append(currentGroup, line)
previousLine = line
continue
}
groups = append(groups, currentGroup)
currentGroup = group{line}
previousLine = line
}
if len(currentGroup) > 0 {
groups = append(groups, currentGroup)
}
sort.Slice(groups, func(i, j int) bool {
// groups may be out of order, sort them. Invariant:
// indexing is safe because if groups is non-nil, then there
// exists at least one group with one element.
return groups[i][0].LineNumber < groups[j][0].LineNumber
})
return groups
}
// DecorateFileHTML returns decorated HTML rendering of file content. If
// successful and within bounds of timeout and line size, it returns HTML marked
// up with highlight classes. In other cases, it returns plaintext HTML.
func DecorateFileHTML(ctx context.Context, repo api.RepoName, commit api.CommitID, path string) (*highlight.HighlightedCode, error) {
content, err := fetchContent(ctx, repo, commit, path)
if err != nil {
return nil, err
}
highlightResponse, aborted, err := highlight.Code(ctx, highlight.Params{
Content: content,
Filepath: path,
DisableTimeout: false, // use default 3 second timeout
HighlightLongLines: false, // use default 2000 character line count
Metadata: highlight.Metadata{ // for logging
RepoName: string(repo),
Revision: string(commit),
},
})
if err != nil {
return nil, err
}
// TODO: Can I remove this?
if aborted {
// code decoration aborted, returns plaintext HTML.
return highlightResponse, nil
}
return highlightResponse, nil
}
// DecorateFileHunksHTML returns decorated file hunks given a file match.
// TODO: I can't find any references to this function...?
func DecorateFileHunksHTML(ctx context.Context, fm *result.FileMatch) []stream.DecoratedHunk {
fmt.Println("==> DecorateFileHunksHTML")
response, err := DecorateFileHTML(ctx, fm.Repo.Name, fm.CommitID, fm.Path)
if err != nil {
log15.Warn("stream result decoration could not highlight file", "error", err)
return nil
}
lines, err := response.SplitHighlightedLines(true)
if err != nil {
log15.Warn("stream result decoration could not split highlighted file", "error", err)
return nil
}
// a closure over lines that allows to splice line ranges.
spliceRows := func(lineStart, lineEnd int) []string {
if lineStart < 0 {
lineStart = 0
}
if lineEnd > len(lines) {
lineEnd = len(lines)
}
if lineStart > lineEnd {
lineStart = 0
lineEnd = 0
}
tableRows := make([]string, 0, lineEnd-lineStart)
for _, line := range lines[lineStart:lineEnd] {
tableRows = append(tableRows, string(line))
}
return tableRows
}
groups := groupLineMatches(fm.ChunkMatches.AsLineMatches())
hunks := make([]stream.DecoratedHunk, 0, len(groups))
for _, group := range groups {
rows := spliceRows(int(group[0].LineNumber), int(group[0].LineNumber)+len(group))
hunks = append(hunks, stream.DecoratedHunk{
Content: stream.DecoratedContent{HTML: strings.Join(rows, "\n")},
LineStart: int(group[0].LineNumber),
LineCount: len(group),
Matches: toMatchRanges(group),
})
}
return hunks
}
func fetchContent(ctx context.Context, repo api.RepoName, commit api.CommitID, path string) (content []byte, err error) {
content, err = gitserver.NewClientDeprecatedNeedsDB().ReadFile(ctx, authz.DefaultSubRepoPermsChecker, repo, commit, path)
if err != nil {
return nil, err
}
return content, nil
}

117
cmd/frontend/internal/search/decorate_test.go
View File

@ -1,117 +0,0 @@
package search
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/sourcegraph/sourcegraph/internal/search/result"
stream "github.com/sourcegraph/sourcegraph/internal/search/streaming/http"
)
func TestGroupLineMatches(t *testing.T) {
data := []*result.LineMatch{
{LineNumber: 1},
{LineNumber: 2},
{LineNumber: 3},
{LineNumber: 5},
{LineNumber: 6},
{LineNumber: 8},
}
want := []group{
{
{LineNumber: 1},
{LineNumber: 2},
{LineNumber: 3},
},
{
{LineNumber: 5},
{LineNumber: 6},
},
{
{LineNumber: 8},
},
}
got := groupLineMatches(data)
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("line match group partition wrong (-want +got):\n%s", diff)
}
}
func TestToHunk(t *testing.T) {
data := []group{
{
{LineNumber: 1, OffsetAndLengths: [][2]int32{{0, 1}}},
{LineNumber: 2, OffsetAndLengths: [][2]int32{{2, 3}, {4, 5}}},
{LineNumber: 3, OffsetAndLengths: [][2]int32{{6, 7}, {8, 9}}},
},
{
{LineNumber: 5, OffsetAndLengths: [][2]int32{{0, 1}}},
{LineNumber: 6, OffsetAndLengths: [][2]int32{{2, 3}, {4, 5}}},
},
{
{LineNumber: 8, OffsetAndLengths: [][2]int32{{6, 7}, {8, 9}}},
},
}
want := [][]stream.Range{
{
{
Start: stream.Location{Offset: -1, Line: 1, Column: 0},
End: stream.Location{Offset: -1, Line: 1, Column: 1},
},
{
Start: stream.Location{Offset: -1, Line: 2, Column: 2},
End: stream.Location{Offset: -1, Line: 2, Column: 5},
},
{
Start: stream.Location{Offset: -1, Line: 2, Column: 4},
End: stream.Location{Offset: -1, Line: 2, Column: 9},
},
{
Start: stream.Location{Offset: -1, Line: 3, Column: 6},
End: stream.Location{Offset: -1, Line: 3, Column: 13},
},
{
Start: stream.Location{Offset: -1, Line: 3, Column: 8},
End: stream.Location{Offset: -1, Line: 3, Column: 17},
},
},
{
{
Start: stream.Location{Offset: -1, Line: 5, Column: 0},
End: stream.Location{Offset: -1, Line: 5, Column: 1},
},
{
Start: stream.Location{Offset: -1, Line: 6, Column: 2},
End: stream.Location{Offset: -1, Line: 6, Column: 5},
},
{
Start: stream.Location{Offset: -1, Line: 6, Column: 4},
End: stream.Location{Offset: -1, Line: 6, Column: 9},
},
},
{
{
Start: stream.Location{Offset: -1, Line: 8, Column: 6},
End: stream.Location{Offset: -1, Line: 8, Column: 13},
},
{
Start: stream.Location{Offset: -1, Line: 8, Column: 8},
End: stream.Location{Offset: -1, Line: 8, Column: 17},
},
},
}
var got [][]stream.Range
for _, group := range data {
got = append(got, toMatchRanges(group))
}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("line match group partition wrong (-want +got):\n%s", diff)
}
}

1
cmd/searcher/internal/search/BUILD.bazel
View File

@ -132,6 +132,7 @@ go_test(
"//cmd/searcher/protocol",
"//internal/api",
"//internal/comby",
"//internal/database",
"//internal/errcode",
"//internal/gitserver",
"//internal/observation",

9
cmd/searcher/internal/search/search_regex_test.go
View File

@ -18,6 +18,8 @@ import (
"github.com/sourcegraph/sourcegraph/cmd/searcher/protocol"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/observation"
)
@ -424,9 +426,10 @@ func TestPathMatches(t *testing.T) {
// githubStore fetches from github and caches across test runs.
var githubStore = &Store{
FetchTar: fetchTarFromGithub,
Path: "/tmp/search_test/store",
ObservationCtx: &observation.TestContext,
GitserverClient: gitserver.NewClient(database.NewMockDB()),
FetchTar: fetchTarFromGithub,
Path: "/tmp/search_test/store",
ObservationCtx: &observation.TestContext,
}
func fetchTarFromGithub(ctx context.Context, repo api.RepoName, commit api.CommitID) (io.ReadCloser, error) {

2
cmd/searcher/internal/search/search_test.go
View File

@ -21,6 +21,7 @@ import (
"github.com/grafana/regexp"
"github.com/sourcegraph/log/logtest"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/cmd/searcher/internal/search"
@ -567,6 +568,7 @@ func newStore(t *testing.T, files map[string]struct {
}
return &search.Store{
GitserverClient: gitserver.NewClient(database.NewMockDB()),
FetchTar: func(ctx context.Context, repo api.RepoName, commit api.CommitID) (io.ReadCloser, error) {
r, w := io.Pipe()
go func() {

7
cmd/searcher/internal/search/store.go
View File

@ -55,6 +55,9 @@ const maxFileSize = 2 << 20 // 2MB; match https://sourcegraph.com/search?q=repo:
// (tar). We want to be able to support random concurrent access for reading,
// so we store as a zip.
type Store struct {
// GitserverClient is the client to interact with gitserver.
GitserverClient gitserver.Client
// FetchTar returns an io.ReadCloser to a tar archive of repo at commit.
// If the error implements "BadRequest() bool", it will be used to
// determine if the error is a bad request (eg invalid repo).
@ -280,7 +283,7 @@ func (s *Store) fetch(ctx context.Context, repo api.RepoName, commit api.CommitI
filter.CommitIgnore = func(hdr *tar.Header) bool { return false } // default: don't filter
if s.FilterTar != nil {
filter.CommitIgnore, err = s.FilterTar(ctx, gitserver.NewClientDeprecatedNeedsDB(), repo, commit)
filter.CommitIgnore, err = s.FilterTar(ctx, s.GitserverClient, repo, commit)
if err != nil {
return nil, errors.Errorf("error while calling FilterTar: %w", err)
}
@ -438,7 +441,7 @@ func (s *Store) watchAndEvict() {
func (s *Store) watchConfig() {
for {
// Allow roughly 10 fetches per gitserver
limit := 10 * len(gitserver.NewClientDeprecatedNeedsDB().Addrs())
limit := 10 * len(s.GitserverClient.Addrs())
if limit == 0 {
limit = 15
}

7
cmd/searcher/internal/search/store_test.go
View File

@ -18,7 +18,9 @@ import (
"github.com/sourcegraph/log/logtest"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/errcode"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
@ -299,8 +301,9 @@ func tarArchive(dir string) (*tar.Reader, error) {
func tmpStore(t *testing.T) *Store {
d := t.TempDir()
return &Store{
Path: d,
Log: logtest.Scoped(t),
GitserverClient: gitserver.NewClient(database.NewMockDB()),
Path: d,
Log: logtest.Scoped(t),
ObservationCtx: observation.TestContextTB(t),
}

3
cmd/searcher/shared/BUILD.bazel
View File

@ -14,6 +14,9 @@ go_library(
"//internal/actor",
"//internal/api",
"//internal/conf",
"//internal/conf/conftypes",
"//internal/database",
"//internal/database/connections/live",
"//internal/debugserver",
"//internal/env",
"//internal/gitserver",

24
cmd/searcher/shared/shared.go
View File

@ -4,6 +4,7 @@ package shared
import (
"context"
"database/sql"
"io"
"net"
"net/http"
@ -22,6 +23,9 @@ import (
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
@ -101,10 +105,25 @@ func setupTmpDir() error {
return nil
}
func initDB(observationCtx *observation.Context) (*sql.DB, error) {
dsn := conf.GetServiceConnectionValueAndRestartOnChange(func(serviceConnections conftypes.ServiceConnections) string {
return serviceConnections.PostgresDSN
})
db, err := connections.EnsureNewFrontendDB(observationCtx, dsn, "searcher")
return db, errors.Wrap(err, "searcher: failed to connect to frontend database")
}
func Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc) error {
logger := observationCtx.Logger
// Ready immediately
rawDB, err := initDB(observationCtx)
if err != nil {
return err
}
db := database.NewDB(observationCtx.Logger, rawDB)
// Ready as soon as the database connection has been established.
ready()
var cacheSizeBytes int64
@ -126,10 +145,11 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi
// Explicitly don't scope Store logger under the parent logger
storeObservationCtx := observation.NewContext(log.Scoped("Store", "searcher archives store"))
git := gitserver.NewClientDeprecatedNeedsDB()
git := gitserver.NewClient(db)
sService := &search.Service{
Store: &search.Store{
GitserverClient: git,
FetchTar: func(ctx context.Context, repo api.RepoName, commit api.CommitID) (io.ReadCloser, error) {
// We pass in a nil sub-repo permissions checker and an internal actor here since
// searcher needs access to all data in the archive.

1
cmd/symbols/gitserver/BUILD.bazel
View File

@ -12,6 +12,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//internal/api",
"//internal/database",
"//internal/gitserver",
"//internal/gitserver/gitdomain",
"//internal/metrics",

5
cmd/symbols/gitserver/client.go
View File

@ -8,6 +8,7 @@ import (
"go.opentelemetry.io/otel/attribute"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/observation"
@ -47,9 +48,9 @@ type gitserverClient struct {
operations *operations
}
func NewClient(observationCtx *observation.Context) GitserverClient {
func NewClient(observationCtx *observation.Context, db database.DB) GitserverClient {
return &gitserverClient{
innerClient: gitserver.NewClientDeprecatedNeedsDB(),
innerClient: gitserver.NewClient(db),
operations: newOperations(observationCtx),
}
}

2
cmd/symbols/shared/main.go
View File

@ -76,7 +76,7 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
db := database.NewDB(logger, sqlDB)
// Run setup
gitserverClient := gitserver.NewClient(observationCtx)
gitserverClient := gitserver.NewClient(observationCtx, db)
repositoryFetcher := fetcher.NewRepositoryFetcher(observationCtx, gitserverClient, RepositoryFetcherConfig.MaxTotalPathsLength, int64(RepositoryFetcherConfig.MaxFileSizeKb)*1000)
searchFunc, handleStatus, newRoutines, ctagsBinary, err := setup(observationCtx, db, gitserverClient, repositoryFetcher)
if err != nil {

2
enterprise/cmd/frontend/internal/app/init.go
View File

@ -23,7 +23,7 @@ func Init(
enterpriseServices *enterprise.Services,
) error {
if deploy.IsApp() {
gitserverClient := gitserver.NewClientDeprecatedNeedsDB()
gitserverClient := gitserver.NewClient(db)
enterpriseServices.OptionalResolver.AppResolver = resolvers.NewAppResolver(observationCtx.Logger, db, gitserverClient)
}
return nil

2
enterprise/cmd/frontend/internal/batches/init.go
View File

@ -44,7 +44,7 @@ func Init(
bstore := store.New(db, observationCtx, keyring.Default().BatchChangesCredentialKey)
// Register enterprise services.
gitserverClient := gitserver.NewClientDeprecatedNeedsDB()
gitserverClient := gitserver.NewClient(db)
logger := sglog.Scoped("Batches", "batch changes webhooks")
enterpriseServices.BatchChangesResolver = resolvers.New(db, bstore, gitserverClient, logger)
enterpriseServices.BatchesGitHubWebhook = webhooks.NewGitHubWebhook(bstore, gitserverClient, logger)

2
enterprise/cmd/frontend/internal/embeddings/init.go
View File

@ -23,7 +23,7 @@ func Init(
enterpriseServices *enterprise.Services,
) error {
repoEmbeddingsStore := repo.NewRepoEmbeddingJobsStore(db)
gitserverClient := gitserver.NewClientDeprecatedNeedsDB()
gitserverClient := gitserver.NewClient(db)
embeddingsClient := embeddings.NewDefaultClient()
enterpriseServices.EmbeddingsResolver = resolvers.NewResolver(
db,

2
enterprise/cmd/frontend/internal/executorqueue/queuehandler.go
View File

@ -50,7 +50,7 @@ func newExecutorQueuesHandler(
multiHandler := handler.NewMultiHandler(executorStore, jobTokenStore, metricsStore, codeIntelQueueHandler, batchesQueueHandler)
gitserverClient := gitserver.NewClientDeprecatedNeedsDB()
gitserverClient := gitserver.NewClient(db)
// Auth middleware
executorAuth := executorAuthMiddleware(logger, accessToken)

2
enterprise/cmd/frontend/internal/own/init.go
View File

@ -22,7 +22,7 @@ func Init(
_ conftypes.UnifiedWatchable,
enterpriseServices *enterprise.Services,
) error {
g := gitserver.NewClientDeprecatedNeedsDB()
g := gitserver.NewClient(db)
enterpriseServices.OwnResolver = resolvers.New(db, g, observationCtx.Logger.Scoped("own", "Code ownership"))
return nil
}

2
enterprise/cmd/worker/internal/batches/reconciler_job.go
View File

@ -47,7 +47,7 @@ func (j *reconcilerJob) Routines(_ context.Context, observationCtx *observation.
observationCtx,
bstore,
reconcilerStore,
gitserver.NewClientDeprecatedNeedsDB(),
gitserver.NewClient(bstore.DatabaseDB()),
sources.NewSourcer(httpcli.NewExternalClientFactory(
httpcli.NewLoggingMiddleware(observationCtx.Logger.Scoped("sourcer", "batches sourcer")),
)),

2
enterprise/cmd/worker/internal/embeddings/repo/worker.go
View File

@ -60,7 +60,7 @@ func (s *repoEmbeddingJob) Routines(_ context.Context, observationCtx *observati
repoembeddingsbg.NewRepoEmbeddingJobWorkerStore(observationCtx, db.Handle()),
db,
uploadStore,
gitserver.NewClientDeprecatedNeedsDB(),
gitserver.NewClient(db),
services.ContextService,
repoembeddingsbg.NewRepoEmbeddingJobsStore(db),
),

2
internal/authz/providers/authz.go
View File

@ -164,7 +164,7 @@ func ProvidersFromConfig(
initResult := github.NewAuthzProviders(ctx, db, gitHubConns, cfg.SiteConfig().AuthProviders, enableGithubInternalRepoVisibility)
initResult.Append(gitlab.NewAuthzProviders(db, cfg.SiteConfig(), gitLabConns))
initResult.Append(bitbucketserver.NewAuthzProviders(bitbucketServerConns))
initResult.Append(perforce.NewAuthzProviders(perforceConns))
initResult.Append(perforce.NewAuthzProviders(db, perforceConns))
initResult.Append(bitbucketcloud.NewAuthzProviders(db, bitbucketCloudConns, cfg.SiteConfig().AuthProviders))
initResult.Append(gerrit.NewAuthzProviders(gerritConns, cfg.SiteConfig().AuthProviders))
initResult.Append(azuredevops.NewAuthzProviders(db, azuredevopsConns))

3
internal/authz/providers/perforce/BUILD.bazel
View File

@ -13,6 +13,7 @@ go_library(
deps = [
"//internal/authz",
"//internal/authz/types",
"//internal/database",
"//internal/extsvc",
"//internal/extsvc/perforce",
"//internal/gitserver",
@ -44,9 +45,11 @@ go_test(
"//internal/authz",
"//internal/authz/subrepoperms",
"//internal/conf",
"//internal/database",
"//internal/encryption/testing",
"//internal/extsvc",
"//internal/extsvc/perforce",
"//internal/gitserver",
"//internal/types",
"//schema",
"@com_github_google_go_cmp//cmp",

9
internal/authz/providers/perforce/authz.go
View File

@ -5,6 +5,8 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/authz"
@ -23,10 +25,10 @@ import (
// This constructor does not and should not directly check connectivity to external services - if
// desired, callers should use `(*Provider).ValidateConnection` directly to get warnings related
// to connection issues.
func NewAuthzProviders(conns []*types.PerforceConnection) *atypes.ProviderInitResult {
func NewAuthzProviders(db database.DB, conns []*types.PerforceConnection) *atypes.ProviderInitResult {
initResults := &atypes.ProviderInitResult{}
for _, c := range conns {
p, err := newAuthzProvider(c.URN, c.Authorization, c.P4Port, c.P4User, c.P4Passwd, c.Depots)
p, err := newAuthzProvider(db, c.URN, c.Authorization, c.P4Port, c.P4User, c.P4Passwd, c.Depots)
if err != nil {
initResults.InvalidConnections = append(initResults.InvalidConnections, extsvc.TypePerforce)
initResults.Problems = append(initResults.Problems, err.Error())
@ -39,6 +41,7 @@ func NewAuthzProviders(conns []*types.PerforceConnection) *atypes.ProviderInitRe
}
func newAuthzProvider(
db database.DB,
urn string,
a *schema.PerforceAuthorization,
host, user, password string,
@ -67,7 +70,7 @@ func newAuthzProvider(
}
}
return NewProvider(logger, urn, host, user, password, depotIDs), nil
return NewProvider(logger, gitserver.NewClient(db), urn, host, user, password, depotIDs), nil
}
// ValidateAuthz validates the authorization fields of the given Perforce

5
internal/authz/providers/perforce/perforce.go
View File

@ -17,7 +17,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
"github.com/sourcegraph/sourcegraph/internal/extsvc/perforce"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/errors"
@ -62,7 +61,7 @@ type p4Execer interface {
// host, user and password to talk to a Perforce Server that is the source of
// truth for permissions. It assumes emails of Sourcegraph accounts match 1-1
// with emails of Perforce Server users.
func NewProvider(logger log.Logger, urn, host, user, password string, depots []extsvc.RepoID) *Provider {
func NewProvider(logger log.Logger, p4Execer p4Execer, urn, host, user, password string, depots []extsvc.RepoID) *Provider {
baseURL, _ := url.Parse(host)
return &Provider{
logger: logger,
@ -72,7 +71,7 @@ func NewProvider(logger log.Logger, urn, host, user, password string, depots []e
host: host,
user: user,
password: password,
p4Execer: gitserver.NewClientDeprecatedNeedsDB(),
p4Execer: p4Execer,
cachedGroupMembers: make(map[string][]string),
}
}

16
internal/authz/providers/perforce/perforce_test.go
View File

@ -15,9 +15,11 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/database"
et "github.com/sourcegraph/sourcegraph/internal/encryption/testing"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
"github.com/sourcegraph/sourcegraph/internal/extsvc/perforce"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/types"
)
@ -85,10 +87,11 @@ cindy <cindy@example.com> (Cindy) accessed 2020/12/04
func TestProvider_FetchUserPerms(t *testing.T) {
ctx := context.Background()
db := database.NewMockDB()
t.Run("nil account", func(t *testing.T) {
logger := logtest.Scoped(t)
p := NewProvider(logger, "", "ssl:111.222.333.444:1666", "admin", "password", nil)
p := NewProvider(logger, gitserver.NewClient(db), "", "ssl:111.222.333.444:1666", "admin", "password", nil)
_, err := p.FetchUserPerms(ctx, nil, authz.FetchPermsOptions{})
want := "no account provided"
got := fmt.Sprintf("%v", err)
@ -99,7 +102,7 @@ func TestProvider_FetchUserPerms(t *testing.T) {
t.Run("not the code host of the account", func(t *testing.T) {
logger := logtest.Scoped(t)
p := NewProvider(logger, "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
p := NewProvider(logger, gitserver.NewClient(db), "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
_, err := p.FetchUserPerms(context.Background(),
&extsvc.Account{
AccountSpec: extsvc.AccountSpec{
@ -118,7 +121,7 @@ func TestProvider_FetchUserPerms(t *testing.T) {
t.Run("no user found in account data", func(t *testing.T) {
logger := logtest.Scoped(t)
p := NewProvider(logger, "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
p := NewProvider(logger, gitserver.NewClient(db), "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
_, err := p.FetchUserPerms(ctx,
&extsvc.Account{
AccountSpec: extsvc.AccountSpec{
@ -340,9 +343,10 @@ read user alice * -//Sourcegraph/Security/...
func TestProvider_FetchRepoPerms(t *testing.T) {
logger := logtest.Scoped(t)
ctx := context.Background()
db := database.NewMockDB()
t.Run("nil repository", func(t *testing.T) {
p := NewProvider(logger, "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
p := NewProvider(logger, gitserver.NewClient(db), "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
_, err := p.FetchRepoPerms(ctx, nil, authz.FetchPermsOptions{})
want := "no repository provided"
got := fmt.Sprintf("%v", err)
@ -352,7 +356,7 @@ func TestProvider_FetchRepoPerms(t *testing.T) {
})
t.Run("not the code host of the repository", func(t *testing.T) {
p := NewProvider(logger, "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
p := NewProvider(logger, gitserver.NewClient(db), "", "ssl:111.222.333.444:1666", "admin", "password", []extsvc.RepoID{})
_, err := p.FetchRepoPerms(ctx,
&extsvc.Repository{
URI: "gitlab.com/user/repo",
@ -438,7 +442,7 @@ Users:
}
func NewTestProvider(logger log.Logger, urn, host, user, password string, execer p4Execer) *Provider {
p := NewProvider(logger, urn, host, user, password, []extsvc.RepoID{})
p := NewProvider(logger, gitserver.NewClient(database.NewMockDB()), urn, host, user, password, []extsvc.RepoID{})
p.p4Execer = execer
return p
}

4
internal/batches/processor/bulk_processor.go
View File

@ -188,7 +188,7 @@ func (b *bulkProcessor) mergeChangeset(ctx context.Context, job *btypes.Changese
b.logger.Error("Events", log.Error(err))
return nil, errcode.MakeNonRetryable(err)
}
state.SetDerivedState(ctx, b.tx.Repos(), gitserver.NewClientDeprecatedNeedsDB(), cs.Changeset, events)
state.SetDerivedState(ctx, b.tx.Repos(), gitserver.NewClient(b.tx.DatabaseDB()), cs.Changeset, events)
if err := b.tx.UpsertChangesetEvents(ctx, events...); err != nil {
b.logger.Error("UpsertChangesetEvents", log.Error(err))
@ -224,7 +224,7 @@ func (b *bulkProcessor) closeChangeset(ctx context.Context) (afterDone func(*sto
b.logger.Error("Events", log.Error(err))
return nil, errcode.MakeNonRetryable(err)
}
state.SetDerivedState(ctx, b.tx.Repos(), gitserver.NewClientDeprecatedNeedsDB(), cs.Changeset, events)
state.SetDerivedState(ctx, b.tx.Repos(), gitserver.NewClient(b.tx.DatabaseDB()), cs.Changeset, events)
if err := b.tx.UpsertChangesetEvents(ctx, events...); err != nil {
b.logger.Error("UpsertChangesetEvents", log.Error(err))

2
internal/batches/service/workspace_resolver.go
View File

@ -77,7 +77,7 @@ func NewWorkspaceResolver(s *store.Store) WorkspaceResolver {
return &workspaceResolver{
store: s,
logger: log.Scoped("batches.workspaceResolver", "The batch changes execution workspace resolver"),
gitserverClient: gitserver.NewClientDeprecatedNeedsDB(),
gitserverClient: gitserver.NewClient(s.DatabaseDB()),
frontendInternalURL: internalapi.Client.URL + "/.internal",
}
}

1
internal/batches/sources/BUILD.bazel
View File

@ -51,6 +51,7 @@ go_library(
"//schema",
"@com_github_inconshreveable_log15//:log15",
"@com_github_masterminds_semver//:semver",
"@com_github_sourcegraph_log//:log",
],
)

115
internal/batches/sources/mocks_test.go generated
View File

@ -2984,6 +2984,9 @@ func (c ForkableChangesetSourceWithAuthenticatorFuncCall) Results() []interface{
// github.com/sourcegraph/sourcegraph/internal/batches/sources) used for
// unit testing.
type MockSourcerStore struct {
// DatabaseDBFunc is an instance of a mock function object controlling
// the behavior of the method DatabaseDB.
DatabaseDBFunc *SourcerStoreDatabaseDBFunc
// ExternalServicesFunc is an instance of a mock function object
// controlling the behavior of the method ExternalServices.
ExternalServicesFunc *SourcerStoreExternalServicesFunc
@ -3011,6 +3014,11 @@ type MockSourcerStore struct {
// methods return zero values for all results, unless overwritten.
func NewMockSourcerStore() *MockSourcerStore {
return &MockSourcerStore{
DatabaseDBFunc: &SourcerStoreDatabaseDBFunc{
defaultHook: func() (r0 database.DB) {
return
},
},
ExternalServicesFunc: &SourcerStoreExternalServicesFunc{
defaultHook: func() (r0 database.ExternalServiceStore) {
return
@ -3053,6 +3061,11 @@ func NewMockSourcerStore() *MockSourcerStore {
// interface. All methods panic on invocation, unless overwritten.
func NewStrictMockSourcerStore() *MockSourcerStore {
return &MockSourcerStore{
DatabaseDBFunc: &SourcerStoreDatabaseDBFunc{
defaultHook: func() database.DB {
panic("unexpected invocation of MockSourcerStore.DatabaseDB")
},
},
ExternalServicesFunc: &SourcerStoreExternalServicesFunc{
defaultHook: func() database.ExternalServiceStore {
panic("unexpected invocation of MockSourcerStore.ExternalServices")
@ -3096,6 +3109,9 @@ func NewStrictMockSourcerStore() *MockSourcerStore {
// overwritten.
func NewMockSourcerStoreFrom(i SourcerStore) *MockSourcerStore {
return &MockSourcerStore{
DatabaseDBFunc: &SourcerStoreDatabaseDBFunc{
defaultHook: i.DatabaseDB,
},
ExternalServicesFunc: &SourcerStoreExternalServicesFunc{
defaultHook: i.ExternalServices,
},
@ -3120,6 +3136,105 @@ func NewMockSourcerStoreFrom(i SourcerStore) *MockSourcerStore {
}
}
// SourcerStoreDatabaseDBFunc describes the behavior when the DatabaseDB
// method of the parent MockSourcerStore instance is invoked.
type SourcerStoreDatabaseDBFunc struct {
defaultHook func() database.DB
hooks []func() database.DB
history []SourcerStoreDatabaseDBFuncCall
mutex sync.Mutex
}
// DatabaseDB delegates to the next hook function in the queue and stores
// the parameter and result values of this invocation.
func (m *MockSourcerStore) DatabaseDB() database.DB {
r0 := m.DatabaseDBFunc.nextHook()()
m.DatabaseDBFunc.appendCall(SourcerStoreDatabaseDBFuncCall{r0})
return r0
}
// SetDefaultHook sets function that is called when the DatabaseDB method of
// the parent MockSourcerStore instance is invoked and the hook queue is
// empty.
func (f *SourcerStoreDatabaseDBFunc) SetDefaultHook(hook func() database.DB) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// DatabaseDB method of the parent MockSourcerStore instance invokes the
// hook at the front of the queue and discards it. After the queue is empty,
// the default hook function is invoked for any future action.
func (f *SourcerStoreDatabaseDBFunc) PushHook(hook func() database.DB) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *SourcerStoreDatabaseDBFunc) SetDefaultReturn(r0 database.DB) {
f.SetDefaultHook(func() database.DB {
return r0
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *SourcerStoreDatabaseDBFunc) PushReturn(r0 database.DB) {
f.PushHook(func() database.DB {
return r0
})
}
func (f *SourcerStoreDatabaseDBFunc) nextHook() func() database.DB {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *SourcerStoreDatabaseDBFunc) appendCall(r0 SourcerStoreDatabaseDBFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of SourcerStoreDatabaseDBFuncCall objects
// describing the invocations of this function.
func (f *SourcerStoreDatabaseDBFunc) History() []SourcerStoreDatabaseDBFuncCall {
f.mutex.Lock()
history := make([]SourcerStoreDatabaseDBFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// SourcerStoreDatabaseDBFuncCall is an object that describes an invocation
// of method DatabaseDB on an instance of MockSourcerStore.
type SourcerStoreDatabaseDBFuncCall struct {
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 database.DB
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c SourcerStoreDatabaseDBFuncCall) Args() []interface{} {
return []interface{}{}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c SourcerStoreDatabaseDBFuncCall) Results() []interface{} {
return []interface{}{c.Result0}
}
// SourcerStoreExternalServicesFunc describes the behavior when the
// ExternalServices method of the parent MockSourcerStore instance is
// invoked.

7
internal/batches/sources/perforce.go
View File

@ -22,7 +22,7 @@ type PerforceSource struct {
perforceCreds *gitserver.PerforceCredentials
}
func NewPerforceSource(ctx context.Context, svc *types.ExternalService, _ *httpcli.Factory) (*PerforceSource, error) {
func NewPerforceSource(ctx context.Context, gitserverClient gitserver.Client, svc *types.ExternalService, _ *httpcli.Factory) (*PerforceSource, error) {
rawConfig, err := svc.Config.Decrypt(ctx)
if err != nil {
return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err)
@ -32,7 +32,10 @@ func NewPerforceSource(ctx context.Context, svc *types.ExternalService, _ *httpc
return nil, errors.Wrapf(err, "external service id=%d", svc.ID)
}
return &PerforceSource{server: c, gitServerClient: gitserver.NewClientDeprecatedNeedsDB()}, nil
return &PerforceSource{
server: c,
gitServerClient: gitserverClient,
}, nil
}
// GitserverPushConfig returns an authenticated push config used for pushing commits to the code host.

11
internal/batches/sources/sources.go
View File

@ -7,6 +7,7 @@ import (
"sort"
"strings"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/batches/store"
btypes "github.com/sourcegraph/sourcegraph/internal/batches/types"
"github.com/sourcegraph/sourcegraph/internal/database"
@ -17,6 +18,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/extsvc/github"
ghaauth "github.com/sourcegraph/sourcegraph/internal/github_apps/auth"
ghastore "github.com/sourcegraph/sourcegraph/internal/github_apps/store"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/protocol"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/internal/types"
@ -74,6 +76,7 @@ const (
)
type SourcerStore interface {
DatabaseDB() database.DB
GetBatchChange(ctx context.Context, opts store.GetBatchChangeOpts) (*btypes.BatchChange, error)
GetSiteCredential(ctx context.Context, opts store.GetSiteCredentialOpts) (*btypes.SiteCredential, error)
GetExternalServiceIDs(ctx context.Context, opts store.GetExternalServiceIDsOpts) ([]int64, error)
@ -118,12 +121,14 @@ func NewSourcer(cf *httpcli.Factory) Sourcer {
type changesetSourceFactory func(ctx context.Context, tx SourcerStore, cf *httpcli.Factory, extSvc *types.ExternalService) (ChangesetSource, error)
type sourcer struct {
logger log.Logger
cf *httpcli.Factory
newSource changesetSourceFactory
}
func newSourcer(cf *httpcli.Factory, csf changesetSourceFactory) Sourcer {
return &sourcer{
logger: log.Scoped("sourcer", "logger scoped to sources.sourcer"),
cf: cf,
newSource: csf,
}
@ -215,7 +220,7 @@ func (s *sourcer) ForExternalService(ctx context.Context, tx SourcerStore, au au
}
func loadBatchesSource(ctx context.Context, tx SourcerStore, cf *httpcli.Factory, extSvc *types.ExternalService) (ChangesetSource, error) {
css, err := buildChangesetSource(ctx, cf, extSvc)
css, err := buildChangesetSource(ctx, tx, cf, extSvc)
if err != nil {
return nil, errors.Wrap(err, "building changeset source")
}
@ -431,7 +436,7 @@ func loadExternalService(ctx context.Context, s database.ExternalServiceStore, o
// buildChangesetSource builds a ChangesetSource for the given repo to load the
// changeset state from.
func buildChangesetSource(ctx context.Context, cf *httpcli.Factory, externalService *types.ExternalService) (ChangesetSource, error) {
func buildChangesetSource(ctx context.Context, tx SourcerStore, cf *httpcli.Factory, externalService *types.ExternalService) (ChangesetSource, error) {
switch externalService.Kind {
case extsvc.KindGitHub:
return NewGitHubSource(ctx, externalService, cf)
@ -446,7 +451,7 @@ func buildChangesetSource(ctx context.Context, cf *httpcli.Factory, externalServ
case extsvc.KindGerrit:
return NewGerritSource(ctx, externalService, cf)
case extsvc.KindPerforce:
return NewPerforceSource(ctx, externalService, cf)
return NewPerforceSource(ctx, gitserver.NewClient(tx.DatabaseDB()), externalService, cf)
default:
return nil, errors.Errorf("unsupported external service type %q", extsvc.KindToType(externalService.Kind))
}

9
internal/batches/store/store.go
View File

@ -66,8 +66,9 @@ func RandomID() (string, error) {
// Store exposes methods to read and write batches domain models
// from persistent storage.
type Store struct {
logger log.Logger
*basestore.Store
logger log.Logger
key encryption.Key
now func() time.Time
operations *operations
@ -101,15 +102,15 @@ func (s *Store) GitHubAppsStore() store.GitHubAppsStore {
return store.GitHubAppsWith(s.Store).WithEncryptionKey(keyring.Default().GitHubAppKey)
}
// Clock returns the clock used by the Store.
func (s *Store) Clock() func() time.Time { return s.now }
// DatabaseDB returns a database.DB with the same handle that this Store was
// instantiated with.
// It's here for legacy reason to pass the database.DB to a repos.Store while
// repos.Store doesn't accept a basestore.TransactableHandle yet.
func (s *Store) DatabaseDB() database.DB { return database.NewDBWith(s.logger, s) }
// Clock returns the clock used by the Store.
func (s *Store) Clock() func() time.Time { return s.now }
var _ basestore.ShareableStore = &Store{}
// With creates a new Store with the given basestore.Shareable store as the

3
internal/batches/syncer/syncer.go
View File

@ -485,6 +485,7 @@ func (s *changesetSyncer) SyncChangeset(ctx context.Context, id int64) error {
return err
}
db := s.syncStore.DatabaseDB()
srcer := sources.NewSourcer(s.httpFactory)
source, err := srcer.ForChangeset(ctx, s.syncStore, cs, sources.AuthenticationStrategyUserCredential)
if err != nil {
@ -495,7 +496,7 @@ func (s *changesetSyncer) SyncChangeset(ctx context.Context, id int64) error {
return err
}
return SyncChangeset(ctx, s.syncStore, gitserver.NewClientDeprecatedNeedsDB(), source, repo, cs)
return SyncChangeset(ctx, s.syncStore, gitserver.NewClient(db), source, repo, cs)
}
// SyncChangeset refreshes the metadata of the given changeset and

2
internal/codeintel/autoindexing/init.go
View File

@ -29,7 +29,7 @@ func NewService(
) *Service {
store := autoindexingstore.New(scopedContext("store", observationCtx), db)
repoUpdater := repoupdater.DefaultClient
inferenceSvc := inference.NewService()
inferenceSvc := inference.NewService(db)
return newService(
scopedContext("service", observationCtx),

1
internal/codeintel/autoindexing/internal/inference/BUILD.bazel
View File

@ -23,6 +23,7 @@ go_library(
"//internal/codeintel/autoindexing/shared",
"//internal/codeintel/dependencies",
"//internal/conf/reposource",
"//internal/database",
"//internal/env",
"//internal/gitserver",
"//internal/gitserver/gitdomain",

5
internal/codeintel/autoindexing/internal/inference/iface.go
View File

@ -6,6 +6,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/luasandbox"
@ -25,14 +26,14 @@ type gitService struct {
client gitserver.Client
}
func NewDefaultGitService(checker authz.SubRepoPermissionChecker) GitService {
func NewDefaultGitService(db database.DB, checker authz.SubRepoPermissionChecker) GitService {
if checker == nil {
checker = authz.DefaultSubRepoPermsChecker
}
return &gitService{
checker: checker,
client: gitserver.NewClientDeprecatedNeedsDB(),
client: gitserver.NewClient(db),
}
}

5
internal/codeintel/autoindexing/internal/inference/init.go
View File

@ -5,6 +5,7 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/luasandbox"
"github.com/sourcegraph/sourcegraph/internal/observation"
@ -17,13 +18,13 @@ var (
maximumFileWithContentSizeBytes = env.MustGetInt("CODEINTEL_AUTOINDEXING_INFERENCE_MAXIMUM_FILE_WITH_CONTENT_SIZE_BYTES", 1024*1024, "The maximum size of the content of a single file requested by the inference script. Inference operations exceeding this limit will fail.")
)
func NewService() *Service {
func NewService(db database.DB) *Service {
observationCtx := observation.NewContext(log.Scoped("inference.service", "inference service"))
return newService(
observationCtx,
luasandbox.NewService(),
NewDefaultGitService(nil),
NewDefaultGitService(db, nil),
ratelimit.NewInstrumentedLimiter("InferenceService", rate.NewLimiter(rate.Limit(gitserverRequestRateLimit), 1)),
maximumFilesWithContentCount,
maximumFileWithContentSizeBytes,

18
internal/gitserver/client.go
View File

@ -82,24 +82,6 @@ type ClientSource interface {
Addresses() []AddressWithClient
}
// NewClientDeprecatedNeedsDB means this call site needs to be updated to use
// NewClient and pass in a DB.
//
// Right now the DB is not used so this is not a bug. This variable was
// introduced to make it possible to break out the updates to NewClient happen
// over multiple PRs.
//
// If you are introducing DB, prefer changing your code so that gitserver
// client is injected from a higher layer rather than constructed at your call
// site.
//
// Shout at Keegan and Indradhanush if you still see this call in August 2023.
// Context: we want to make looking up which shard to speak to stateful. First
// target is deduplicating storage for forks.
func NewClientDeprecatedNeedsDB() Client {
return NewClient(nil)
}
// NewClient returns a new gitserver.Client.
func NewClient(_ database.DB) Client {
return &clientImplementor{

71
internal/gitserver/commands_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/errcode"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/types"
@ -149,7 +150,7 @@ func TestDiffWithSubRepoFiltering(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.label, func(t *testing.T) {
repo := MakeGitRepository(t, append(cmds, tc.extraGitCommands...)...)
c := NewClientDeprecatedNeedsDB()
c := NewClient(database.NewMockDB())
commits, err := c.Commits(ctx, nil, repo, CommitsOptions{})
if err != nil {
t.Fatalf("err fetching commits: %s", err)
@ -203,7 +204,7 @@ func TestDiff(t *testing.T) {
".foo",
} {
t.Run("invalid base: "+input, func(t *testing.T) {
i, err := NewClientDeprecatedNeedsDB().Diff(ctx, nil, DiffOptions{Base: input})
i, err := NewClient(database.NewMockDB()).Diff(ctx, nil, DiffOptions{Base: input})
if i != nil {
t.Errorf("unexpected non-nil iterator: %+v", i)
}
@ -454,7 +455,7 @@ func TestRepository_BlameFile(t *testing.T) {
},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
for label, test := range tests {
newestCommitID, err := client.ResolveRevision(ctx, test.repo, string(test.opt.NewestCommit), ResolveRevisionOptions{})
if err != nil {
@ -496,7 +497,7 @@ func runBlameFileTest(ctx context.Context, t *testing.T, repo api.RepoName, path
checker authz.SubRepoPermissionChecker, label string, wantHunks []*Hunk,
) {
t.Helper()
hunks, err := NewClientDeprecatedNeedsDB().BlameFile(ctx, checker, repo, path, opt)
hunks, err := NewClient(database.NewMockDB()).BlameFile(ctx, checker, repo, path, opt)
if err != nil {
t.Errorf("%s: BlameFile(%s, %+v): %s", label, path, opt, err)
return
@ -541,7 +542,7 @@ func TestRepository_ResolveBranch(t *testing.T) {
}
for label, test := range tests {
commitID, err := NewClientDeprecatedNeedsDB().ResolveRevision(context.Background(), test.repo, test.branch, ResolveRevisionOptions{})
commitID, err := NewClient(database.NewMockDB()).ResolveRevision(context.Background(), test.repo, test.branch, ResolveRevisionOptions{})
if err != nil {
t.Errorf("%s: ResolveRevision: %s", label, err)
continue
@ -573,7 +574,7 @@ func TestRepository_ResolveBranch_error(t *testing.T) {
}
for label, test := range tests {
commitID, err := NewClientDeprecatedNeedsDB().ResolveRevision(context.Background(), test.repo, test.branch, ResolveRevisionOptions{})
commitID, err := NewClient(database.NewMockDB()).ResolveRevision(context.Background(), test.repo, test.branch, ResolveRevisionOptions{})
if !test.wantErr(err) {
t.Errorf("%s: ResolveRevision: %s", label, err)
continue
@ -606,7 +607,7 @@ func TestRepository_ResolveTag(t *testing.T) {
}
for label, test := range tests {
commitID, err := NewClientDeprecatedNeedsDB().ResolveRevision(context.Background(), test.repo, test.tag, ResolveRevisionOptions{})
commitID, err := NewClient(database.NewMockDB()).ResolveRevision(context.Background(), test.repo, test.tag, ResolveRevisionOptions{})
if err != nil {
t.Errorf("%s: ResolveRevision: %s", label, err)
continue
@ -638,7 +639,7 @@ func TestRepository_ResolveTag_error(t *testing.T) {
}
for label, test := range tests {
commitID, err := NewClientDeprecatedNeedsDB().ResolveRevision(context.Background(), test.repo, test.tag, ResolveRevisionOptions{})
commitID, err := NewClient(database.NewMockDB()).ResolveRevision(context.Background(), test.repo, test.tag, ResolveRevisionOptions{})
if !test.wantErr(err) {
t.Errorf("%s: ResolveRevision: %s", label, err)
continue
@ -653,7 +654,7 @@ func TestRepository_ResolveTag_error(t *testing.T) {
func TestLsFiles(t *testing.T) {
ClientMocks.LocalGitserver = true
defer ResetClientMocks()
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
runFileListingTest(t, func(ctx context.Context, checker authz.SubRepoPermissionChecker, repo api.RepoName, commit string) ([]string, error) {
return client.LsFiles(ctx, checker, repo, api.CommitID(commit))
})
@ -796,7 +797,7 @@ func TestCleanDirectoriesForLsTree(t *testing.T) {
func TestListDirectoryChildren(t *testing.T) {
ClientMocks.LocalGitserver = true
defer ResetClientMocks()
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
gitCommands := []string{
"mkdir -p dir{1..3}/sub{1..3}",
"touch dir1/sub1/file",
@ -883,7 +884,7 @@ func TestListTags(t *testing.T) {
{Name: "t3", CommitID: "afeafc4a918c144329807df307e68899e6b65018", CreatorDate: MustParseTime(time.RFC3339, "2006-01-02T15:04:05Z")},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
tags, err := client.ListTags(context.Background(), repo)
require.Nil(t, err)
@ -944,7 +945,7 @@ func TestMerger_MergeBase(t *testing.T) {
defer ResetClientMocks()
ctx := context.Background()
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
// TODO(sqs): implement for hg
// TODO(sqs): make a more complex test case
@ -1031,7 +1032,7 @@ func TestRepository_FileSystem_Symlinks(t *testing.T) {
dir := InitGitRepository(t, gitCommands...)
repo := api.RepoName(filepath.Base(dir))
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
commitID := api.CommitID(ComputeCommitHash(dir, true))
@ -1117,7 +1118,7 @@ func TestStat(t *testing.T) {
dir := InitGitRepository(t, gitCommands...)
repo := api.RepoName(filepath.Base(dir))
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
commitID := api.CommitID(ComputeCommitHash(dir, true))
@ -1197,7 +1198,7 @@ func TestRepository_GetCommit(t *testing.T) {
revisionNotFoundError bool
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
runGetCommitTests := func(checker authz.SubRepoPermissionChecker, tests map[string]testCase) {
for label, test := range tests {
t.Run(label, func(t *testing.T) {
@ -1366,7 +1367,7 @@ func TestRepository_HasCommitAfter(t *testing.T) {
},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
t.Run("basic", func(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.label, func(t *testing.T) {
@ -1445,7 +1446,7 @@ func TestRepository_FirstEverCommit(t *testing.T) {
want: "2007-01-02T15:04:05Z",
},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
t.Run("basic", func(t *testing.T) {
for _, tc := range testCases {
gitCommands := make([]string, len(tc.commitDates))
@ -1525,7 +1526,7 @@ func TestCommitExists(t *testing.T) {
ctx := actor.WithActor(context.Background(), &actor.Actor{
UID: 1,
})
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
testCommitExists := func(label string, gitCommands []string, commitID, nonExistentCommitID api.CommitID, checker authz.SubRepoPermissionChecker) {
t.Run(label, func(t *testing.T) {
repo := MakeGitRepository(t, gitCommands...)
@ -1602,7 +1603,7 @@ func TestRepository_Commits(t *testing.T) {
wantTotal: 2,
},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
runCommitsTests := func(checker authz.SubRepoPermissionChecker) {
for label, test := range tests {
t.Run(label, func(t *testing.T) {
@ -1694,7 +1695,7 @@ func TestCommits_SubRepoPerms(t *testing.T) {
for label, test := range tests {
t.Run(label, func(t *testing.T) {
checker := getTestSubRepoPermsChecker(test.noAccessPaths...)
commits, err := NewClientDeprecatedNeedsDB().Commits(ctx, checker, repo, test.opt)
commits, err := NewClient(database.NewMockDB()).Commits(ctx, checker, repo, test.opt)
if err != nil {
t.Errorf("%s: Commits(): %s", label, err)
return
@ -1786,7 +1787,7 @@ func TestCommits_SubRepoPerms_ReturnNCommits(t *testing.T) {
},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
for label, test := range tests {
t.Run(label, func(t *testing.T) {
checker := getTestSubRepoPermsChecker(test.noAccessPaths...)
@ -1883,7 +1884,7 @@ func TestRepository_Commits_options(t *testing.T) {
repo := MakeGitRepository(t)
before := ""
after := time.Date(2022, 11, 11, 12, 10, 0, 4, time.UTC).Format(time.RFC3339)
_, err := NewClientDeprecatedNeedsDB().Commits(ctx, checker, repo, CommitsOptions{N: 0, DateOrder: true, NoEnsureRevision: true, After: after, Before: before})
_, err := NewClient(database.NewMockDB()).Commits(ctx, checker, repo, CommitsOptions{N: 0, DateOrder: true, NoEnsureRevision: true, After: after, Before: before})
if err == nil {
t.Error("expected error, got nil")
}
@ -2178,7 +2179,7 @@ func TestFilterRefDescriptions(t *testing.T) { // KEEP
}
checker := getTestSubRepoPermsChecker("file3")
client := NewClientDeprecatedNeedsDB().(*clientImplementor)
client := NewClient(database.NewMockDB()).(*clientImplementor)
filtered := client.filterRefDescriptions(ctx, repo, refDescriptions, checker)
expectedRefDescriptions := map[string][]gitdomain.RefDescription{
"d38233a79e037d2ab8170b0d0bc0aa438473e6da": {},
@ -2196,7 +2197,7 @@ func TestRefDescriptions(t *testing.T) { // KEEP
ctx := actor.WithActor(context.Background(), &actor.Actor{
UID: 1,
})
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
gitCommands := append(getGitCommandsWithFiles("file1", "file2"), "git checkout -b my-other-branch")
gitCommands = append(gitCommands, getGitCommandsWithFiles("file1-b2", "file2-b2")...)
gitCommands = append(gitCommands, "git checkout -b my-branch-no-access")
@ -2244,7 +2245,7 @@ func TestCommitsUniqueToBranch(t *testing.T) {
ctx := actor.WithActor(context.Background(), &actor.Actor{
UID: 1,
})
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
gitCommands := append([]string{"git checkout -b my-branch"}, getGitCommandsWithFiles("file1", "file2")...)
gitCommands = append(gitCommands, getGitCommandsWithFiles("file3", "file-with-no-access")...)
repo := MakeGitRepository(t, gitCommands...)
@ -2288,7 +2289,7 @@ func TestCommitDate(t *testing.T) {
ctx := actor.WithActor(context.Background(), &actor.Actor{
UID: 1,
})
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
gitCommands := getGitCommandsWithFiles("file1", "file2")
repo := MakeGitRepository(t, gitCommands...)
@ -2322,7 +2323,7 @@ func TestCommitDate(t *testing.T) {
func testCommits(ctx context.Context, label string, repo api.RepoName, opt CommitsOptions, checker authz.SubRepoPermissionChecker, wantCommits []*gitdomain.Commit, t *testing.T) {
t.Helper()
client := NewClientDeprecatedNeedsDB().(*clientImplementor)
client := NewClient(database.NewMockDB()).(*clientImplementor)
commits, err := client.Commits(ctx, checker, repo, opt)
if err != nil {
t.Errorf("%s: Commits(): %s", label, err)
@ -2456,7 +2457,7 @@ func TestArchiveReaderForRepoWithSubRepoPermissions(t *testing.T) {
Treeish: commitID,
Pathspecs: []gitdomain.Pathspec{"."},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
if _, err := client.ArchiveReader(context.Background(), checker, repo.Name, opts); err == nil {
t.Error("Error should not be null because ArchiveReader is invoked for a repo with sub-repo permissions")
}
@ -2491,7 +2492,7 @@ func TestArchiveReaderForRepoWithoutSubRepoPermissions(t *testing.T) {
Treeish: commitID,
Pathspecs: []gitdomain.Pathspec{"."},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
readCloser, err := client.ArchiveReader(context.Background(), checker, repo.Name, opts)
if err != nil {
t.Error("Error should not be thrown because ArchiveReader is invoked for a repo without sub-repo permissions")
@ -2564,7 +2565,7 @@ func TestRead(t *testing.T) {
},
}
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
ClientMocks.LocalGitserver = true
t.Cleanup(func() {
ResetClientMocks()
@ -2662,7 +2663,7 @@ func TestRead(t *testing.T) {
func runNewFileReaderTest(ctx context.Context, t *testing.T, repo api.RepoName, commitID api.CommitID, file string,
checker authz.SubRepoPermissionChecker, checkFn func(*testing.T, error, []byte)) {
t.Helper()
rc, err := NewClientDeprecatedNeedsDB().NewFileReader(ctx, checker, repo, commitID, file)
rc, err := NewClient(database.NewMockDB()).NewFileReader(ctx, checker, repo, commitID, file)
if err != nil {
checkFn(t, err, nil)
return
@ -2728,7 +2729,7 @@ func TestRepository_Branches_MergedInto(t *testing.T) {
repo := MakeGitRepository(t, gitCommands...)
wantBranches := gitBranches
for branch, mergedInto := range wantBranches {
branches, err := NewClientDeprecatedNeedsDB().ListBranches(context.Background(), repo, BranchesOptions{MergedInto: branch})
branches, err := NewClient(database.NewMockDB()).ListBranches(context.Background(), repo, BranchesOptions{MergedInto: branch})
require.Nil(t, err)
if diff := cmp.Diff(mergedInto, branches); diff != "" {
t.Fatalf("branch mismatch (-want +got):\n%s", diff)
@ -2759,7 +2760,7 @@ func TestRepository_Branches_ContainsCommit(t *testing.T) {
repo := MakeGitRepository(t, gitCommands...)
commitToWantBranches := gitWantBranches
for commit, wantBranches := range commitToWantBranches {
branches, err := NewClientDeprecatedNeedsDB().ListBranches(context.Background(), repo, BranchesOptions{ContainsCommit: commit})
branches, err := NewClient(database.NewMockDB()).ListBranches(context.Background(), repo, BranchesOptions{ContainsCommit: commit})
require.Nil(t, err)
sort.Sort(gitdomain.Branches(branches))
@ -2841,7 +2842,7 @@ func testBranches(t *testing.T, gitCommands []string, wantBranches []*gitdomain.
t.Helper()
repo := MakeGitRepository(t, gitCommands...)
gotBranches, err := NewClientDeprecatedNeedsDB().ListBranches(context.Background(), repo, options)
gotBranches, err := NewClient(database.NewMockDB()).ListBranches(context.Background(), repo, options)
require.Nil(t, err)
sort.Sort(gitdomain.Branches(wantBranches))
@ -3271,7 +3272,7 @@ func Test_CommitLog(t *testing.T) {
for label, test := range tests {
t.Run(label, func(t *testing.T) {
repo := MakeGitRepository(t, test.extraGitCommands...)
logResults, err := NewClientDeprecatedNeedsDB().CommitLog(context.Background(), repo, time.Time{})
logResults, err := NewClient(database.NewMockDB()).CommitLog(context.Background(), repo, time.Time{})
if err != nil {
require.ErrorContains(t, err, test.wantErr)
}

5
internal/gitserver/grpc_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc"
"github.com/sourcegraph/sourcegraph/schema"
@ -76,7 +77,7 @@ func TestClient_GRPCRouting(t *testing.T) {
},
})
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
_, _ = client.ResolveRevision(context.Background(), "a", "HEAD", ResolveRevisionOptions{})
if !(m1.called && !m2.called) {
@ -92,7 +93,7 @@ func TestClient_GRPCRouting(t *testing.T) {
}
func TestClient_AddrForRepo_UsesConfToRead_PinnedRepos(t *testing.T) {
client := NewClientDeprecatedNeedsDB()
client := NewClient(database.NewMockDB())
cfg := newConfig(
[]string{"gitserver1", "gitserver2"},

7
internal/insights/background/background.go
View File

@ -15,6 +15,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/database"
edb "github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/basestore"
internalGitserver "github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/insights/background/limiter"
"github.com/sourcegraph/sourcegraph/internal/insights/background/pings"
@ -69,6 +70,8 @@ func GetBackgroundJobs(ctx context.Context, logger log.Logger, mainAppDB databas
NewLicenseCheckJob(ctx, mainAppDB, insightsDB),
}
gitserverClient := internalGitserver.NewClient(mainAppDB)
// Register the background goroutine which discovers historical gaps in data and enqueues
// work to fill them - if not disabled.
disableHistorical, _ := strconv.ParseBool(os.Getenv("DISABLE_CODE_INSIGHTS_HISTORICAL"))
@ -76,10 +79,10 @@ func GetBackgroundJobs(ctx context.Context, logger log.Logger, mainAppDB databas
searchRateLimiter := limiter.SearchQueryRate()
historicRateLimiter := limiter.HistoricalWorkRate()
backfillConfig := pipeline.BackfillerConfig{
CompressionPlan: compression.NewGitserverFilter(logger),
CompressionPlan: compression.NewGitserverFilter(logger, gitserverClient),
SearchHandlers: queryrunner.GetSearchHandlers(),
InsightStore: insightsStore,
CommitClient: gitserver.NewGitCommitClient(),
CommitClient: gitserver.NewGitCommitClient(gitserverClient),
SearchPlanWorkerLimit: 1,
SearchRunnerWorkerLimit: 1, // TODO: this can scale with the number of searcher endpoints
SearchRateLimiter: searchRateLimiter,

1
internal/insights/compression/BUILD.bazel
View File

@ -8,6 +8,7 @@ go_library(
visibility = ["//:__subpackages__"],
deps = [
"//internal/api",
"//internal/gitserver",
"//internal/gitserver/gitdomain",
"//internal/insights/gitserver",
"//internal/insights/store",

5
internal/insights/compression/compression.go
View File

@ -29,6 +29,7 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/api"
internalGitserver "github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/insights/gitserver"
"github.com/sourcegraph/sourcegraph/internal/insights/store"
@ -45,8 +46,8 @@ type commitFetcher interface {
RecentCommits(ctx context.Context, repoName api.RepoName, target time.Time, revision string) ([]*gitdomain.Commit, error)
}
func NewGitserverFilter(logger log.Logger) DataFrameFilter {
return &gitserverFilter{commitFetcher: gitserver.NewGitCommitClient(), logger: logger}
func NewGitserverFilter(logger log.Logger, gitserverClient internalGitserver.Client) DataFrameFilter {
return &gitserverFilter{commitFetcher: gitserver.NewGitCommitClient(gitserverClient), logger: logger}
}
type gitserverFilter struct {

6
internal/insights/gitserver/client.go
View File

@ -10,10 +10,10 @@ import (
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
)
func NewGitCommitClient() *GitCommitClient {
func NewGitCommitClient(gitserverClient gitserver.Client) *GitCommitClient {
return &GitCommitClient{
cachedFirstCommit: NewCachedGitFirstEverCommit(),
gitserverClient: gitserver.NewClientDeprecatedNeedsDB(),
gitserverClient: gitserverClient,
}
}
@ -23,7 +23,7 @@ type GitCommitClient struct {
}
func (g *GitCommitClient) FirstCommit(ctx context.Context, repoName api.RepoName) (*gitdomain.Commit, error) {
return g.cachedFirstCommit.GitFirstEverCommit(ctx, repoName)
return g.cachedFirstCommit.GitFirstEverCommit(ctx, g.gitserverClient, repoName)
}
func (g *GitCommitClient) RecentCommits(ctx context.Context, repoName api.RepoName, target time.Time, revision string) ([]*gitdomain.Commit, error) {
options := gitserver.CommitsOptions{N: 1, Before: target.Format(time.RFC3339), DateOrder: true}

10
internal/insights/gitserver/first_commit.go
View File

@ -29,8 +29,8 @@ func isFirstCommitEmptyRepoError(err error) bool {
return false
}
func GitFirstEverCommit(ctx context.Context, repoName api.RepoName) (*gitdomain.Commit, error) {
commit, err := gitserver.NewClientDeprecatedNeedsDB().FirstEverCommit(ctx, authz.DefaultSubRepoPermsChecker, repoName)
func GitFirstEverCommit(ctx context.Context, gitserverClient gitserver.Client, repoName api.RepoName) (*gitdomain.Commit, error) {
commit, err := gitserverClient.FirstEverCommit(ctx, authz.DefaultSubRepoPermsChecker, repoName)
if err != nil && isFirstCommitEmptyRepoError(err) {
return nil, errors.Wrap(EmptyRepoErr, err.Error())
}
@ -47,13 +47,13 @@ func NewCachedGitFirstEverCommit() *CachedGitFirstEverCommit {
// using a map, and entries are never evicted because they are expected to be small and in general
// unchanging.
type CachedGitFirstEverCommit struct {
impl func(ctx context.Context, repoName api.RepoName) (*gitdomain.Commit, error)
impl func(ctx context.Context, gitserverClient gitserver.Client, repoName api.RepoName) (*gitdomain.Commit, error)
mu sync.Mutex
cache map[api.RepoName]*gitdomain.Commit
}
func (c *CachedGitFirstEverCommit) GitFirstEverCommit(ctx context.Context, repoName api.RepoName) (*gitdomain.Commit, error) {
func (c *CachedGitFirstEverCommit) GitFirstEverCommit(ctx context.Context, gitserverClient gitserver.Client, repoName api.RepoName) (*gitdomain.Commit, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.cache == nil {
@ -62,7 +62,7 @@ func (c *CachedGitFirstEverCommit) GitFirstEverCommit(ctx context.Context, repoN
if cached, ok := c.cache[repoName]; ok {
return cached, nil
}
entry, err := c.impl(ctx, repoName)
entry, err := c.impl(ctx, gitserverClient, repoName)
if err != nil {
return nil, err
}

7
internal/insights/query/capture_group_executor.go
View File

@ -10,6 +10,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
internalGitserver "github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/insights/compression"
"github.com/sourcegraph/sourcegraph/internal/insights/gitserver"
"github.com/sourcegraph/sourcegraph/internal/insights/query/querybuilder"
@ -19,6 +20,7 @@ import (
)
type CaptureGroupExecutor struct {
gitserverClient internalGitserver.Client
previewExecutor
computeSearch func(ctx context.Context, query string) ([]GroupedResults, error)
@ -27,6 +29,7 @@ type CaptureGroupExecutor struct {
func NewCaptureGroupExecutor(postgres database.DB, clock func() time.Time) *CaptureGroupExecutor {
return &CaptureGroupExecutor{
gitserverClient: internalGitserver.NewClient(postgres),
previewExecutor: previewExecutor{
repoStore: postgres.Repos(),
// filter: compression.NewHistoricalFilter(true, clock().Add(time.Hour*24*365*-1), insightsDb),
@ -68,7 +71,7 @@ func (c *CaptureGroupExecutor) Execute(ctx context.Context, query string, reposi
pivoted := make(map[string]timeCounts)
for _, repository := range repositories {
firstCommit, err := gitserver.GitFirstEverCommit(ctx, api.RepoName(repository))
firstCommit, err := gitserver.GitFirstEverCommit(ctx, c.gitserverClient, api.RepoName(repository))
if err != nil {
if errors.Is(err, gitserver.EmptyRepoErr) {
continue
@ -88,7 +91,7 @@ func (c *CaptureGroupExecutor) Execute(ctx context.Context, query string, reposi
// since we are using uncompressed plans (to avoid this problem and others) right now, each execution is standalone
continue
}
commits, err := gitserver.NewGitCommitClient().RecentCommits(ctx, api.RepoName(repository), execution.RecordingTime, "")
commits, err := gitserver.NewGitCommitClient(c.gitserverClient).RecentCommits(ctx, api.RepoName(repository), execution.RecordingTime, "")
if err != nil {
return nil, errors.Wrap(err, "git.Commits")
} else if len(commits) < 1 {

7
internal/insights/query/streaming_query_executor.go
View File

@ -10,6 +10,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
internalGitserver "github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/insights/compression"
"github.com/sourcegraph/sourcegraph/internal/insights/gitserver"
"github.com/sourcegraph/sourcegraph/internal/insights/query/querybuilder"
@ -20,6 +21,7 @@ import (
)
type StreamingQueryExecutor struct {
gitserverClient internalGitserver.Client
previewExecutor
logger log.Logger
@ -27,6 +29,7 @@ type StreamingQueryExecutor struct {
func NewStreamingExecutor(postgres database.DB, clock func() time.Time) *StreamingQueryExecutor {
return &StreamingQueryExecutor{
gitserverClient: internalGitserver.NewClient(postgres),
previewExecutor: previewExecutor{
repoStore: postgres.Repos(),
filter: &compression.NoopFilter{},
@ -52,7 +55,7 @@ func (c *StreamingQueryExecutor) Execute(ctx context.Context, query string, seri
timeDataPoints := []TimeDataPoint{}
for _, repository := range repositories {
firstCommit, err := gitserver.GitFirstEverCommit(ctx, api.RepoName(repository))
firstCommit, err := gitserver.GitFirstEverCommit(ctx, c.gitserverClient, api.RepoName(repository))
if err != nil {
if errors.Is(err, gitserver.EmptyRepoErr) {
continue
@ -72,7 +75,7 @@ func (c *StreamingQueryExecutor) Execute(ctx context.Context, query string, seri
// since we are using uncompressed plans (to avoid this problem and others) right now, each execution is standalone
continue
}
commits, err := gitserver.NewGitCommitClient().RecentCommits(ctx, api.RepoName(repository), execution.RecordingTime, "")
commits, err := gitserver.NewGitCommitClient(c.gitserverClient).RecentCommits(ctx, api.RepoName(repository), execution.RecordingTime, "")
if err != nil {
return nil, errors.Wrap(err, "git.Commits")
} else if len(commits) < 1 {