artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 13:11:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

gitserver: Framework to support integration testing against gitserver (#62801)

Browse Source 
This PR tinkers a bit with building a test helper to run integration
tests that are still ~lightweight against a real gitserver.
The caller can either clone a real repo to disk / embed it in the git
repo, or can create a small repo on the fly, and then get a running
gitserver gRPC server that returns all the data required.

These tests should only exist outside of cmd/ and internal/, as there is
a big potential to do cross-cmd imports from here, which can cause bad
coupling. But for just these tests, that should be fine.

The most trivial rockskip indexing job that I put in here to POC this
runs in 6.3s, including all setup and teardown. That seems very
reasonable to me.

Test plan:

The POC test passes.
This commit is contained in:
Erik Seliger 2024-06-07 17:01:12 +02:00 committed by GitHub
parent 1284536eed
commit 1287243cae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 436 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/gitserver/shared/BUILD.bazel
View File

@ -8,6 +8,7 @@ go_library(
"debug.go",
"service.go",
"shared.go",
"testserver.go",
],
importpath = "github.com/sourcegraph/sourcegraph/cmd/gitserver/shared",
tags = [TAG_PLATFORM_SOURCE],

158
cmd/gitserver/shared/shared.go
View File

@ -11,6 +11,13 @@ import (
"path/filepath"
"strings"
"github.com/sourcegraph/sourcegraph/internal/actor"
internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
"github.com/sourcegraph/sourcegraph/internal/instrumentation"
"github.com/sourcegraph/sourcegraph/internal/requestclient"
"github.com/sourcegraph/sourcegraph/internal/requestinteraction"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/vcs"
"github.com/sourcegraph/log"
@ -25,7 +32,6 @@ import (
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git/gitcli"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/gitserverfs"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/vcssyncer"
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/authz/subrepoperms"
@ -40,16 +46,10 @@ import (
proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/goroutine/recorder"
internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc"
"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
"github.com/sourcegraph/sourcegraph/internal/instrumentation"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
"github.com/sourcegraph/sourcegraph/internal/requestclient"
"github.com/sourcegraph/sourcegraph/internal/requestinteraction"
"github.com/sourcegraph/sourcegraph/internal/service"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/internal/wrexec"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -94,53 +94,19 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
backendSource := func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
return git.NewObservableBackend(gitcli.NewBackend(logger, recordingCommandFactory, dir, repoName))
}
gitserver := server.NewServer(&server.ServerOpts{
Logger: logger,
GitBackendSource: backendSource,
GetRemoteURLFunc: func(ctx context.Context, repo api.RepoName) (string, error) {
gitserver := makeServer(
observationCtx,
fs,
db,
recordingCommandFactory,
backendSource,
hostname,
config.CoursierCacheDir,
locker,
func(ctx context.Context, repo api.RepoName) (string, error) {
return getRemoteURLFunc(ctx, db, repo)
},
GetVCSSyncer: func(ctx context.Context, repo api.RepoName) (vcssyncer.VCSSyncer, error) {
return vcssyncer.NewVCSSyncer(ctx, &vcssyncer.NewVCSSyncerOpts{
ExternalServiceStore: db.ExternalServices(),
RepoStore: db.Repos(),
DepsSvc: dependencies.NewService(observationCtx, db),
Repo: repo,
CoursierCacheDir: config.CoursierCacheDir,
RecordingCommandFactory: recordingCommandFactory,
Logger: logger,
FS: fs,
GetRemoteURLSource: func(ctx context.Context, repo api.RepoName) (vcssyncer.RemoteURLSource, error) {
return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) {
rawURL, err := getRemoteURLFunc(ctx, db, repo)
if err != nil {
return nil, errors.Wrapf(err, "getting remote URL for %q", repo)
}
u, err := vcs.ParseURL(rawURL)
if err != nil {
// TODO@ggilmore: Note that we can't redact the URL here because we can't
// parse it to know where the sensitive information is.
return nil, errors.Wrapf(err, "parsing remote URL %q", rawURL)
}
return u, nil
}), nil
},
})
},
FS: fs,
Hostname: hostname,
DB: db,
RecordingCommandFactory: recordingCommandFactory,
Locker: locker,
RPSLimiter: ratelimit.NewInstrumentedLimiter(
ratelimit.GitRPSLimiterBucketName,
ratelimit.NewGlobalRateLimiter(logger, ratelimit.GitRPSLimiterBucketName),
),
})
)
// Make sure we watch for config updates that affect the recordingCommandFactory.
go conf.Watch(func() {
@ -156,21 +122,11 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
internal.RegisterEchoMetric(logger.Scoped("echoMetricReporter"))
handler := internal.NewHTTPHandler(logger, fs)
handler = actor.HTTPMiddleware(logger, handler)
handler = requestclient.InternalHTTPMiddleware(handler)
handler = requestinteraction.HTTPMiddleware(handler)
handler = trace.HTTPMiddleware(logger, handler)
handler = instrumentation.HTTPMiddleware("", handler)
handler = internalgrpc.MultiplexHandlers(makeGRPCServer(logger, gitserver, config), handler)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
routines := []goroutine.BackgroundRoutine{
httpserver.NewFromAddr(config.ListenAddress, &http.Server{
Handler: handler,
}),
makeHTTPServer(logger, fs, makeGRPCServer(logger, gitserver, config), config.ListenAddress),
server.NewRepoStateSyncer(
ctx,
logger,
@ -236,6 +192,82 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
return nil
}
// makeServer creates a new gitserver.Server instance.
func makeServer(
observationCtx *observation.Context,
fs gitserverfs.FS,
db database.DB,
recordingCommandFactory *wrexec.RecordingCommandFactory,
backendSource func(dir common.GitDir, repoName api.RepoName) git.GitBackend,
hostname string,
coursierCacheDir string,
locker internal.RepositoryLocker,
getRemoteURLFunc func(ctx context.Context, repo api.RepoName) (string, error),
) *internal.Server {
return server.NewServer(&server.ServerOpts{
Logger: observationCtx.Logger,
GitBackendSource: backendSource,
GetRemoteURLFunc: getRemoteURLFunc,
GetVCSSyncer: func(ctx context.Context, repo api.RepoName) (vcssyncer.VCSSyncer, error) {
return vcssyncer.NewVCSSyncer(ctx, &vcssyncer.NewVCSSyncerOpts{
ExternalServiceStore: db.ExternalServices(),
RepoStore: db.Repos(),
DepsSvc: dependencies.NewService(observationCtx, db),
Repo: repo,
CoursierCacheDir: coursierCacheDir,
RecordingCommandFactory: recordingCommandFactory,
Logger: observationCtx.Logger,
FS: fs,
GetRemoteURLSource: func(ctx context.Context, repo api.RepoName) (vcssyncer.RemoteURLSource, error) {
return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) {
rawURL, err := getRemoteURLFunc(ctx, repo)
if err != nil {
return nil, errors.Wrapf(err, "getting remote URL for %q", repo)
}
u, err := vcs.ParseURL(rawURL)
if err != nil {
// TODO@ggilmore: Note that we can't redact the URL here because we can't
// parse it to know where the sensitive information is.
return nil, errors.Wrapf(err, "parsing remote URL %q", rawURL)
}
return u, nil
}), nil
},
})
},
FS: fs,
Hostname: hostname,
DB: db,
RecordingCommandFactory: recordingCommandFactory,
Locker: locker,
RPSLimiter: ratelimit.NewInstrumentedLimiter(
ratelimit.GitRPSLimiterBucketName,
ratelimit.NewGlobalRateLimiter(observationCtx.Logger, ratelimit.GitRPSLimiterBucketName),
),
})
}
// makeHTTPServer creates a new *http.Server for the gitserver endpoints and registers
// it with methods on the given server. It multiplexes HTTP requests and gRPC requests
// from a single port.
func makeHTTPServer(logger log.Logger, fs gitserverfs.FS, grpcServer *grpc.Server, listenAddress string) goroutine.BackgroundRoutine {
handler := internal.NewHTTPHandler(logger, fs)
handler = actor.HTTPMiddleware(logger, handler)
handler = requestclient.InternalHTTPMiddleware(handler)
handler = requestinteraction.HTTPMiddleware(handler)
handler = trace.HTTPMiddleware(logger, handler)
handler = instrumentation.HTTPMiddleware("", handler)
handler = internalgrpc.MultiplexHandlers(grpcServer, handler)
return httpserver.NewFromAddr(listenAddress, &http.Server{
Handler: handler,
})
}
// makeGRPCServer creates a new *grpc.Server for the gitserver endpoints and registers
// it with methods on the given server.
func makeGRPCServer(logger log.Logger, s *server.Server, c *Config) *grpc.Server {

63
cmd/gitserver/shared/testserver.go Normal file
View File

@ -0,0 +1,63 @@
package shared
import (
"context"
server "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/common"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git/gitcli"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/gitserverfs"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/wrexec"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
// TestAPIServer returns a new gitserver API server for testing. Do not use this
// in a production workload.
func TestAPIServer(ctx context.Context, observationCtx *observation.Context, db database.DB, config *Config, getRemoteURLFunc func(ctx context.Context, repo api.RepoName) (string, error)) (goroutine.BackgroundRoutine, error) {
logger := observationCtx.Logger
// Load and validate configuration.
if err := config.Validate(); err != nil {
return nil, errors.Wrap(err, "failed to validate configuration")
}
// Prepare the file system.
fs := gitserverfs.New(observationCtx, config.ReposDir)
if err := fs.Initialize(); err != nil {
return nil, err
}
backendSource := func(dir common.GitDir, repoName api.RepoName) git.GitBackend {
return git.NewObservableBackend(gitcli.NewBackend(logger, wrexec.NewNoOpRecordingCommandFactory(), dir, repoName))
}
gitserver := makeServer(observationCtx, fs, db, wrexec.NewNoOpRecordingCommandFactory(), backendSource, config.ExternalAddress, config.CoursierCacheDir, server.NewRepositoryLocker(), getRemoteURLFunc)
httpServer := makeHTTPServer(logger, fs, makeGRPCServer(logger, gitserver, config), config.ListenAddress)
return &testServerRoutine{start: httpServer.Start, stop: func() {
_ = httpServer.Stop(context.Background())
gitserver.Stop()
}}, nil
}
type testServerRoutine struct {
start func()
stop func()
}
func (t *testServerRoutine) Name() string {
return "gitserver-test"
}
func (t *testServerRoutine) Start() {
t.start()
}
func (t *testServerRoutine) Stop(context.Context) error {
t.stop()
return nil
}

1
cmd/symbols/gitserver/BUILD.bazel
View File

@ -13,7 +13,6 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//internal/api",
"//internal/database",
"//internal/gitserver",
"//internal/metrics",
"//internal/observation",

5
cmd/symbols/gitserver/client.go
View File

@ -8,7 +8,6 @@ import (
"go.opentelemetry.io/otel/attribute"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/types"
@ -45,9 +44,9 @@ type gitserverClient struct {
operations *operations
}
func NewClient(observationCtx *observation.Context, db database.DB) GitserverClient {
func NewClient(observationCtx *observation.Context, inner gitserver.Client) GitserverClient {
return &gitserverClient{
innerClient: gitserver.NewClient("symbols"),
innerClient: inner,
operations: newOperations(observationCtx),
}
}

1
cmd/symbols/shared/BUILD.bazel
View File

@ -30,6 +30,7 @@ go_library(
"//internal/debugserver",
"//internal/diskcache",
"//internal/env",
"//internal/gitserver",
"//internal/goroutine",
"//internal/honey",
"//internal/httpserver",

7
cmd/symbols/shared/main.go
View File

@ -13,7 +13,7 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher"
"github.com/sourcegraph/sourcegraph/cmd/symbols/gitserver"
symbolsgitserver "github.com/sourcegraph/sourcegraph/cmd/symbols/gitserver"
"github.com/sourcegraph/sourcegraph/cmd/symbols/internal/api"
sqlite "github.com/sourcegraph/sourcegraph/cmd/symbols/internal/database"
"github.com/sourcegraph/sourcegraph/cmd/symbols/types"
@ -23,6 +23,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/honey"
"github.com/sourcegraph/sourcegraph/internal/httpserver"
@ -44,7 +45,7 @@ var (
const addr = ":3184"
type SetupFunc func(observationCtx *observation.Context, db database.DB, gitserverClient gitserver.GitserverClient, repositoryFetcher fetcher.RepositoryFetcher) (types.SearchFunc, func(http.ResponseWriter, *http.Request), []goroutine.BackgroundRoutine, error)
type SetupFunc func(observationCtx *observation.Context, db database.DB, gitserverClient symbolsgitserver.GitserverClient, repositoryFetcher fetcher.RepositoryFetcher) (types.SearchFunc, func(http.ResponseWriter, *http.Request), []goroutine.BackgroundRoutine, error)
func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, setup SetupFunc) error {
logger := observationCtx.Logger
@ -78,7 +79,7 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
db := database.NewDB(logger, sqlDB)
// Run setup
gitserverClient := gitserver.NewClient(observationCtx, db)
gitserverClient := symbolsgitserver.NewClient(observationCtx, gitserver.NewClient("symbols"))
repositoryFetcher := fetcher.NewRepositoryFetcher(observationCtx, gitserverClient, RepositoryFetcherConfig.MaxTotalPathsLength, int64(RepositoryFetcherConfig.MaxFileSizeKb)*1000)
searchFunc, handleStatus, newRoutines, err := setup(observationCtx, db, gitserverClient, repositoryFetcher)
if err != nil {

22
dev/gitserverintegration/BUILD.bazel Normal file
View File

@ -0,0 +1,22 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "gitserverintegration",
srcs = ["testtools.go"],
importpath = "github.com/sourcegraph/sourcegraph/dev/gitserverintegration",
visibility = ["//visibility:public"],
deps = [
"//cmd/gitserver/shared",
"//internal/api",
"//internal/database/dbmocks",
"//internal/gitserver",
"//internal/gitserver/v1:gitserver",
"//internal/grpc/defaults",
"//internal/observation",
"//internal/types",
"//lib/errors",
"@com_github_sourcegraph_log//logtest",
"@com_github_stretchr_testify//require",
"@org_golang_google_grpc//:go_default_library",
],
)

109
dev/gitserverintegration/testtools.go Normal file
View File

@ -0,0 +1,109 @@
// package gitserverintegration provides utilities for testing against a real gitserver
// in integration testing.
package gitserverintegration
import (
"context"
"testing"
"github.com/sourcegraph/log/logtest"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"github.com/sourcegraph/sourcegraph/cmd/gitserver/shared"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/database/dbmocks"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
v1 "github.com/sourcegraph/sourcegraph/internal/gitserver/v1"
"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
// NewTestGitserverWithRepos spawns a new gitserver with the given repos cloned,
// the map holds the repo name to path on disk mappings. The repos will be cloned
// from the location on disk into gitserver, for a most realistic setup.
// Two clients will be returned to interact with the gitserver.
func NewTestGitserverWithRepos(t *testing.T, repos map[api.RepoName]string) (gitserver.Client, v1.GitserverRepositoryServiceClient) {
// Create supporting infrastructure:
ctx := context.Background()
logger := logtest.Scoped(t)
obsCtx := observation.TestContextTB(t)
reposDir := t.TempDir()
// Create a mock database:
db := dbmocks.NewMockDB()
repoStore := dbmocks.NewMockRepoStore()
repoStore.GetByNameFunc.SetDefaultHook(func(ctx context.Context, rn api.RepoName) (*types.Repo, error) {
if _, ok := repos[rn]; !ok {
return nil, errors.New("repo not found")
}
return &types.Repo{ID: 1, Name: rn}, nil
})
db.ReposFunc.SetDefaultReturn(repoStore)
db.GitserverReposFunc.SetDefaultReturn(dbmocks.NewMockGitserverRepoStore())
// Spawn a test gitserver on a pseudo random port:
testAddr := "127.0.0.1:29484"
routine, err := shared.TestAPIServer(ctx, obsCtx, db, &shared.Config{
ReposDir: reposDir,
ExhaustiveRequestLoggingEnabled: true,
ListenAddress: testAddr,
}, func(ctx context.Context, repo api.RepoName) (string, error) {
if _, ok := repos[repo]; !ok {
return "", errors.New("invalid repo name passed to getRemoteURL func")
}
// We make gitserver clone the repo from the local dir where we create our test repo:
return repos[repo], nil
})
require.NoError(t, err)
// Start the gitserver up and make sure we shut down cleanly on test exit:
go routine.Start()
t.Cleanup(func() {
require.NoError(t, routine.Stop(context.Background()))
})
// Create a gitserver.Client to talk to the gitserver:
gs := gitserver.NewTestClient(t).WithClientSource(gitserver.NewTestClientSource(t, []string{testAddr}, func(o *gitserver.TestClientSourceOptions) {
o.ClientFunc = func(conn *grpc.ClientConn) v1.GitserverServiceClient {
return v1.NewGitserverServiceClient(conn)
}
}))
// Also create a GitserverRepositoryServiceClient to talk to the gitserver:
conn, err := defaults.Dial(testAddr, logger)
require.NoError(t, err)
rs := v1.NewGitserverRepositoryServiceClient(conn)
// Ensure all the requested repos are cloned into the gitserver:
for repo := range repos {
_, err = rs.FetchRepository(ctx, &v1.FetchRepositoryRequest{
RepoName: string(repo),
})
require.NoError(t, err)
}
return gs, rs
}
// RepoWithCommands is a helper method to create a git repo with the given commands.
// The repo will be created in a temporary directory, and can be passed to NewTestGitserverWithRepos.
func RepoWithCommands(t *testing.T, cmds ...string) string {
tmpDir := t.TempDir()
// Prepare repo state:
for _, cmd := range append(
append([]string{"git init --initial-branch=master ."}, cmds...),
// Promote the repo to a bare repo.
"git config --bool core.bare true",
) {
out, err := gitserver.CreateGitCommand(tmpDir, "bash", "-c", cmd).CombinedOutput()
if err != nil {
t.Fatalf("Failed to run git command %v. Output was:\n\n%s", cmd, out)
}
}
return tmpDir
}

37
dev/rockskipintegration/BUILD.bazel Normal file
View File

@ -0,0 +1,37 @@
load("//dev:go_defs.bzl", "go_test")
go_test(
name = "rockskipintegration_test",
srcs = ["main_test.go"],
data = ["//dev/tools:universal-ctags"],
env = {
"CTAGS_RLOCATIONPATH": "$(rlocationpath //dev/tools:universal-ctags)",
},
tags = [
TAG_PLATFORM_SEARCH,
# Test requires talking to real gitserver over network
"requires-network",
],
deps = [
"//cmd/symbols/fetcher",
"//cmd/symbols/gitserver",
"//cmd/symbols/parser",
"//cmd/symbols/types",
"//dev/gitserverintegration",
"//internal/api",
"//internal/ctags_config",
"//internal/database",
"//internal/database/dbtest",
"//internal/env",
"//internal/observation",
"//internal/rockskip",
"//internal/search",
"//internal/search/result",
"//internal/types",
"@com_github_sourcegraph_go_ctags//:go-ctags",
"@com_github_sourcegraph_log//:log",
"@com_github_sourcegraph_log//logtest",
"@com_github_stretchr_testify//require",
"@io_bazel_rules_go//go/runfiles:go_default_library",
],
)

101
dev/rockskipintegration/main_test.go Normal file
View File

@ -0,0 +1,101 @@
package main
import (
"context"
"os"
"os/exec"
"testing"
"github.com/bazelbuild/rules_go/go/runfiles"
"github.com/sourcegraph/go-ctags"
"github.com/sourcegraph/log"
"github.com/sourcegraph/log/logtest"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher"
symbolsgitserver "github.com/sourcegraph/sourcegraph/cmd/symbols/gitserver"
symbolsParser "github.com/sourcegraph/sourcegraph/cmd/symbols/parser"
symbolstypes "github.com/sourcegraph/sourcegraph/cmd/symbols/types"
"github.com/sourcegraph/sourcegraph/dev/gitserverintegration"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/ctags_config"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/dbtest"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/rockskip"
"github.com/sourcegraph/sourcegraph/internal/search"
"github.com/sourcegraph/sourcegraph/internal/search/result"
"github.com/sourcegraph/sourcegraph/internal/types"
)
func TestRockskipIntegration(t *testing.T) {
gs, _ := gitserverintegration.NewTestGitserverWithRepos(t, map[api.RepoName]string{
"github.com/sourcegraph/rockskiptest": gitserverintegration.RepoWithCommands(t,
"echo '# Title' > README.md",
"git add README.md",
"git commit -m commit --author='Foo Author <foo@sourcegraph.com>'",
),
})
ctx := context.Background()
observationCtx := observation.TestContextTB(t)
// Verify gitserver cloned correctly:
head, headSHA, err := gs.GetDefaultBranch(ctx, "github.com/sourcegraph/rockskiptest", false)
require.NoError(t, err)
require.Equal(t, "refs/heads/master", head)
db := dbtest.NewDB(t)
require.NoError(t, database.NewDB(logtest.Scoped(t), db).Repos().Create(ctx, &types.Repo{Name: "github.com/sourcegraph/rockskiptest"}))
_, err = db.ExecContext(ctx, "INSERT INTO rockskip_repos (repo, last_accessed_at) VALUES ($1, NOW())", "github.com/sourcegraph/rockskiptest")
require.NoError(t, err)
sgs := symbolsgitserver.NewClient(observationCtx, gs)
ctagsConfig := symbolstypes.LoadCtagsConfig(env.BaseConfig{})
// Try to find the universal ctags binary. In bazel, it will be provided by bazel.
// Outside of bazel, we rely on the system.
if os.Getenv("BAZEL_TEST") != "" {
ctagsConfig.UniversalCommand, _ = runfiles.Rlocation(os.Getenv("CTAGS_RLOCATIONPATH"))
} else {
_, err = exec.LookPath(ctagsConfig.UniversalCommand)
if err != nil {
// universal-ctags installed with brew is called ctags, try that next:
_, err = exec.LookPath("ctags")
if err == nil {
ctagsConfig.UniversalCommand = "ctags"
// In bazel, we expose the path to ctags via an environment variable.
}
}
}
svc, err := rockskip.NewService(
observationCtx,
db,
sgs,
fetcher.NewRepositoryFetcher(observationCtx, sgs, 100000, 1000),
func() (ctags.Parser, error) {
return symbolsParser.SpawnCtags(log.Scoped("parser"), ctagsConfig, ctags_config.UniversalCtags)
},
// TODO: Adjust these numbers as needed:
1, 1, true, 1, 1024, 1024, true,
)
require.NoError(t, err)
require.NoError(t, svc.Index(ctx, "github.com/sourcegraph/rockskiptest", string(headSHA)))
// TODO: Properly validate rockskip data here:
res, err := svc.Search(ctx, search.SymbolsParameters{
Repo: "github.com/sourcegraph/rockskiptest",
CommitID: api.CommitID(headSHA),
})
require.NoError(t, err)
require.Equal(t, []result.Symbol{
{
Name: "Title",
Path: "README.md",
Line: 0,
Character: 2,
Kind: "chapter",
},
}, res)
}

1
internal/gitserver/v1/BUILD.bazel
View File

@ -31,6 +31,7 @@ go_library(
visibility = [
"//cmd/gitserver:__subpackages__",
"//cmd/repo-updater/internal/gitserver:__pkg__",
"//dev/gitserverintegration:__pkg__",
"//internal/api:__pkg__",
"//internal/extsvc/gitolite:__pkg__",
"//internal/gitserver:__pkg__",