From 1287243cae0bed1c6f8949065e11eeab3e297844 Mon Sep 17 00:00:00 2001 From: Erik Seliger Date: Fri, 7 Jun 2024 17:01:12 +0200 Subject: [PATCH] gitserver: Framework to support integration testing against gitserver (#62801) This PR tinkers a bit with building a test helper to run integration tests that are still ~lightweight against a real gitserver. The caller can either clone a real repo to disk / embed it in the git repo, or can create a small repo on the fly, and then get a running gitserver gRPC server that returns all the data required. These tests should only exist outside of cmd/ and internal/, as there is a big potential to do cross-cmd imports from here, which can cause bad coupling. But for just these tests, that should be fine. The most trivial rockskip indexing job that I put in here to POC this runs in 6.3s, including all setup and teardown. That seems very reasonable to me. Test plan: The POC test passes. --- cmd/gitserver/shared/BUILD.bazel | 1 + cmd/gitserver/shared/shared.go | 158 ++++++++++++++++---------- cmd/gitserver/shared/testserver.go | 63 ++++++++++ cmd/symbols/gitserver/BUILD.bazel | 1 - cmd/symbols/gitserver/client.go | 5 +- cmd/symbols/shared/BUILD.bazel | 1 + cmd/symbols/shared/main.go | 7 +- dev/gitserverintegration/BUILD.bazel | 22 ++++ dev/gitserverintegration/testtools.go | 109 ++++++++++++++++++ dev/rockskipintegration/BUILD.bazel | 37 ++++++ dev/rockskipintegration/main_test.go | 101 ++++++++++++++++ internal/gitserver/v1/BUILD.bazel | 1 + 12 files changed, 436 insertions(+), 70 deletions(-) create mode 100644 cmd/gitserver/shared/testserver.go create mode 100644 dev/gitserverintegration/BUILD.bazel create mode 100644 dev/gitserverintegration/testtools.go create mode 100644 dev/rockskipintegration/BUILD.bazel create mode 100644 dev/rockskipintegration/main_test.go diff --git a/cmd/gitserver/shared/BUILD.bazel b/cmd/gitserver/shared/BUILD.bazel index 60cc867877d..18749fb0bdd 100644 --- a/cmd/gitserver/shared/BUILD.bazel +++ b/cmd/gitserver/shared/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "debug.go", "service.go", "shared.go", + "testserver.go", ], importpath = "github.com/sourcegraph/sourcegraph/cmd/gitserver/shared", tags = [TAG_PLATFORM_SOURCE], diff --git a/cmd/gitserver/shared/shared.go b/cmd/gitserver/shared/shared.go index d0545c3bab9..b0026c3d75d 100644 --- a/cmd/gitserver/shared/shared.go +++ b/cmd/gitserver/shared/shared.go @@ -11,6 +11,13 @@ import ( "path/filepath" "strings" + "github.com/sourcegraph/sourcegraph/internal/actor" + internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc" + "github.com/sourcegraph/sourcegraph/internal/httpserver" + "github.com/sourcegraph/sourcegraph/internal/instrumentation" + "github.com/sourcegraph/sourcegraph/internal/requestclient" + "github.com/sourcegraph/sourcegraph/internal/requestinteraction" + "github.com/sourcegraph/sourcegraph/internal/trace" "github.com/sourcegraph/sourcegraph/internal/vcs" "github.com/sourcegraph/log" @@ -25,7 +32,6 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git/gitcli" "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/gitserverfs" "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/vcssyncer" - "github.com/sourcegraph/sourcegraph/internal/actor" "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/authz" "github.com/sourcegraph/sourcegraph/internal/authz/subrepoperms" @@ -40,16 +46,10 @@ import ( proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" "github.com/sourcegraph/sourcegraph/internal/goroutine" "github.com/sourcegraph/sourcegraph/internal/goroutine/recorder" - internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpserver" - "github.com/sourcegraph/sourcegraph/internal/instrumentation" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/ratelimit" - "github.com/sourcegraph/sourcegraph/internal/requestclient" - "github.com/sourcegraph/sourcegraph/internal/requestinteraction" "github.com/sourcegraph/sourcegraph/internal/service" - "github.com/sourcegraph/sourcegraph/internal/trace" "github.com/sourcegraph/sourcegraph/internal/wrexec" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -94,53 +94,19 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic backendSource := func(dir common.GitDir, repoName api.RepoName) git.GitBackend { return git.NewObservableBackend(gitcli.NewBackend(logger, recordingCommandFactory, dir, repoName)) } - gitserver := server.NewServer(&server.ServerOpts{ - Logger: logger, - GitBackendSource: backendSource, - GetRemoteURLFunc: func(ctx context.Context, repo api.RepoName) (string, error) { + gitserver := makeServer( + observationCtx, + fs, + db, + recordingCommandFactory, + backendSource, + hostname, + config.CoursierCacheDir, + locker, + func(ctx context.Context, repo api.RepoName) (string, error) { return getRemoteURLFunc(ctx, db, repo) }, - GetVCSSyncer: func(ctx context.Context, repo api.RepoName) (vcssyncer.VCSSyncer, error) { - return vcssyncer.NewVCSSyncer(ctx, &vcssyncer.NewVCSSyncerOpts{ - ExternalServiceStore: db.ExternalServices(), - RepoStore: db.Repos(), - DepsSvc: dependencies.NewService(observationCtx, db), - Repo: repo, - CoursierCacheDir: config.CoursierCacheDir, - RecordingCommandFactory: recordingCommandFactory, - Logger: logger, - FS: fs, - GetRemoteURLSource: func(ctx context.Context, repo api.RepoName) (vcssyncer.RemoteURLSource, error) { - return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) { - rawURL, err := getRemoteURLFunc(ctx, db, repo) - if err != nil { - return nil, errors.Wrapf(err, "getting remote URL for %q", repo) - - } - - u, err := vcs.ParseURL(rawURL) - if err != nil { - // TODO@ggilmore: Note that we can't redact the URL here because we can't - // parse it to know where the sensitive information is. - return nil, errors.Wrapf(err, "parsing remote URL %q", rawURL) - } - - return u, nil - - }), nil - }, - }) - }, - FS: fs, - Hostname: hostname, - DB: db, - RecordingCommandFactory: recordingCommandFactory, - Locker: locker, - RPSLimiter: ratelimit.NewInstrumentedLimiter( - ratelimit.GitRPSLimiterBucketName, - ratelimit.NewGlobalRateLimiter(logger, ratelimit.GitRPSLimiterBucketName), - ), - }) + ) // Make sure we watch for config updates that affect the recordingCommandFactory. go conf.Watch(func() { @@ -156,21 +122,11 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic internal.RegisterEchoMetric(logger.Scoped("echoMetricReporter")) - handler := internal.NewHTTPHandler(logger, fs) - handler = actor.HTTPMiddleware(logger, handler) - handler = requestclient.InternalHTTPMiddleware(handler) - handler = requestinteraction.HTTPMiddleware(handler) - handler = trace.HTTPMiddleware(logger, handler) - handler = instrumentation.HTTPMiddleware("", handler) - handler = internalgrpc.MultiplexHandlers(makeGRPCServer(logger, gitserver, config), handler) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() routines := []goroutine.BackgroundRoutine{ - httpserver.NewFromAddr(config.ListenAddress, &http.Server{ - Handler: handler, - }), + makeHTTPServer(logger, fs, makeGRPCServer(logger, gitserver, config), config.ListenAddress), server.NewRepoStateSyncer( ctx, logger, @@ -236,6 +192,82 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic return nil } +// makeServer creates a new gitserver.Server instance. +func makeServer( + observationCtx *observation.Context, + fs gitserverfs.FS, + db database.DB, + recordingCommandFactory *wrexec.RecordingCommandFactory, + backendSource func(dir common.GitDir, repoName api.RepoName) git.GitBackend, + hostname string, + coursierCacheDir string, + locker internal.RepositoryLocker, + getRemoteURLFunc func(ctx context.Context, repo api.RepoName) (string, error), +) *internal.Server { + return server.NewServer(&server.ServerOpts{ + Logger: observationCtx.Logger, + GitBackendSource: backendSource, + GetRemoteURLFunc: getRemoteURLFunc, + GetVCSSyncer: func(ctx context.Context, repo api.RepoName) (vcssyncer.VCSSyncer, error) { + return vcssyncer.NewVCSSyncer(ctx, &vcssyncer.NewVCSSyncerOpts{ + ExternalServiceStore: db.ExternalServices(), + RepoStore: db.Repos(), + DepsSvc: dependencies.NewService(observationCtx, db), + Repo: repo, + CoursierCacheDir: coursierCacheDir, + RecordingCommandFactory: recordingCommandFactory, + Logger: observationCtx.Logger, + FS: fs, + GetRemoteURLSource: func(ctx context.Context, repo api.RepoName) (vcssyncer.RemoteURLSource, error) { + return vcssyncer.RemoteURLSourceFunc(func(ctx context.Context) (*vcs.URL, error) { + rawURL, err := getRemoteURLFunc(ctx, repo) + if err != nil { + return nil, errors.Wrapf(err, "getting remote URL for %q", repo) + + } + + u, err := vcs.ParseURL(rawURL) + if err != nil { + // TODO@ggilmore: Note that we can't redact the URL here because we can't + // parse it to know where the sensitive information is. + return nil, errors.Wrapf(err, "parsing remote URL %q", rawURL) + } + + return u, nil + + }), nil + }, + }) + }, + FS: fs, + Hostname: hostname, + DB: db, + RecordingCommandFactory: recordingCommandFactory, + Locker: locker, + RPSLimiter: ratelimit.NewInstrumentedLimiter( + ratelimit.GitRPSLimiterBucketName, + ratelimit.NewGlobalRateLimiter(observationCtx.Logger, ratelimit.GitRPSLimiterBucketName), + ), + }) +} + +// makeHTTPServer creates a new *http.Server for the gitserver endpoints and registers +// it with methods on the given server. It multiplexes HTTP requests and gRPC requests +// from a single port. +func makeHTTPServer(logger log.Logger, fs gitserverfs.FS, grpcServer *grpc.Server, listenAddress string) goroutine.BackgroundRoutine { + handler := internal.NewHTTPHandler(logger, fs) + handler = actor.HTTPMiddleware(logger, handler) + handler = requestclient.InternalHTTPMiddleware(handler) + handler = requestinteraction.HTTPMiddleware(handler) + handler = trace.HTTPMiddleware(logger, handler) + handler = instrumentation.HTTPMiddleware("", handler) + handler = internalgrpc.MultiplexHandlers(grpcServer, handler) + + return httpserver.NewFromAddr(listenAddress, &http.Server{ + Handler: handler, + }) +} + // makeGRPCServer creates a new *grpc.Server for the gitserver endpoints and registers // it with methods on the given server. func makeGRPCServer(logger log.Logger, s *server.Server, c *Config) *grpc.Server { diff --git a/cmd/gitserver/shared/testserver.go b/cmd/gitserver/shared/testserver.go new file mode 100644 index 00000000000..72a48674bab --- /dev/null +++ b/cmd/gitserver/shared/testserver.go @@ -0,0 +1,63 @@ +package shared + +import ( + "context" + + server "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal" + "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/common" + "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git" + "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git/gitcli" + "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/gitserverfs" + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/goroutine" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/internal/wrexec" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +// TestAPIServer returns a new gitserver API server for testing. Do not use this +// in a production workload. +func TestAPIServer(ctx context.Context, observationCtx *observation.Context, db database.DB, config *Config, getRemoteURLFunc func(ctx context.Context, repo api.RepoName) (string, error)) (goroutine.BackgroundRoutine, error) { + logger := observationCtx.Logger + + // Load and validate configuration. + if err := config.Validate(); err != nil { + return nil, errors.Wrap(err, "failed to validate configuration") + } + + // Prepare the file system. + fs := gitserverfs.New(observationCtx, config.ReposDir) + if err := fs.Initialize(); err != nil { + return nil, err + } + + backendSource := func(dir common.GitDir, repoName api.RepoName) git.GitBackend { + return git.NewObservableBackend(gitcli.NewBackend(logger, wrexec.NewNoOpRecordingCommandFactory(), dir, repoName)) + } + gitserver := makeServer(observationCtx, fs, db, wrexec.NewNoOpRecordingCommandFactory(), backendSource, config.ExternalAddress, config.CoursierCacheDir, server.NewRepositoryLocker(), getRemoteURLFunc) + httpServer := makeHTTPServer(logger, fs, makeGRPCServer(logger, gitserver, config), config.ListenAddress) + + return &testServerRoutine{start: httpServer.Start, stop: func() { + _ = httpServer.Stop(context.Background()) + gitserver.Stop() + }}, nil +} + +type testServerRoutine struct { + start func() + stop func() +} + +func (t *testServerRoutine) Name() string { + return "gitserver-test" +} + +func (t *testServerRoutine) Start() { + t.start() +} + +func (t *testServerRoutine) Stop(context.Context) error { + t.stop() + return nil +} diff --git a/cmd/symbols/gitserver/BUILD.bazel b/cmd/symbols/gitserver/BUILD.bazel index 66d3799a12d..1a2502992ed 100644 --- a/cmd/symbols/gitserver/BUILD.bazel +++ b/cmd/symbols/gitserver/BUILD.bazel @@ -13,7 +13,6 @@ go_library( visibility = ["//visibility:public"], deps = [ "//internal/api", - "//internal/database", "//internal/gitserver", "//internal/metrics", "//internal/observation", diff --git a/cmd/symbols/gitserver/client.go b/cmd/symbols/gitserver/client.go index 9e37ab72b7f..97adabd7ebf 100644 --- a/cmd/symbols/gitserver/client.go +++ b/cmd/symbols/gitserver/client.go @@ -8,7 +8,6 @@ import ( "go.opentelemetry.io/otel/attribute" "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/types" @@ -45,9 +44,9 @@ type gitserverClient struct { operations *operations } -func NewClient(observationCtx *observation.Context, db database.DB) GitserverClient { +func NewClient(observationCtx *observation.Context, inner gitserver.Client) GitserverClient { return &gitserverClient{ - innerClient: gitserver.NewClient("symbols"), + innerClient: inner, operations: newOperations(observationCtx), } } diff --git a/cmd/symbols/shared/BUILD.bazel b/cmd/symbols/shared/BUILD.bazel index 4faa692ad06..d318342e583 100644 --- a/cmd/symbols/shared/BUILD.bazel +++ b/cmd/symbols/shared/BUILD.bazel @@ -30,6 +30,7 @@ go_library( "//internal/debugserver", "//internal/diskcache", "//internal/env", + "//internal/gitserver", "//internal/goroutine", "//internal/honey", "//internal/httpserver", diff --git a/cmd/symbols/shared/main.go b/cmd/symbols/shared/main.go index dbea6a8c15c..b9bbd7ac450 100644 --- a/cmd/symbols/shared/main.go +++ b/cmd/symbols/shared/main.go @@ -13,7 +13,7 @@ import ( "github.com/sourcegraph/log" "github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher" - "github.com/sourcegraph/sourcegraph/cmd/symbols/gitserver" + symbolsgitserver "github.com/sourcegraph/sourcegraph/cmd/symbols/gitserver" "github.com/sourcegraph/sourcegraph/cmd/symbols/internal/api" sqlite "github.com/sourcegraph/sourcegraph/cmd/symbols/internal/database" "github.com/sourcegraph/sourcegraph/cmd/symbols/types" @@ -23,6 +23,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/database" connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live" "github.com/sourcegraph/sourcegraph/internal/env" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/goroutine" "github.com/sourcegraph/sourcegraph/internal/honey" "github.com/sourcegraph/sourcegraph/internal/httpserver" @@ -44,7 +45,7 @@ var ( const addr = ":3184" -type SetupFunc func(observationCtx *observation.Context, db database.DB, gitserverClient gitserver.GitserverClient, repositoryFetcher fetcher.RepositoryFetcher) (types.SearchFunc, func(http.ResponseWriter, *http.Request), []goroutine.BackgroundRoutine, error) +type SetupFunc func(observationCtx *observation.Context, db database.DB, gitserverClient symbolsgitserver.GitserverClient, repositoryFetcher fetcher.RepositoryFetcher) (types.SearchFunc, func(http.ResponseWriter, *http.Request), []goroutine.BackgroundRoutine, error) func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, setup SetupFunc) error { logger := observationCtx.Logger @@ -78,7 +79,7 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic db := database.NewDB(logger, sqlDB) // Run setup - gitserverClient := gitserver.NewClient(observationCtx, db) + gitserverClient := symbolsgitserver.NewClient(observationCtx, gitserver.NewClient("symbols")) repositoryFetcher := fetcher.NewRepositoryFetcher(observationCtx, gitserverClient, RepositoryFetcherConfig.MaxTotalPathsLength, int64(RepositoryFetcherConfig.MaxFileSizeKb)*1000) searchFunc, handleStatus, newRoutines, err := setup(observationCtx, db, gitserverClient, repositoryFetcher) if err != nil { diff --git a/dev/gitserverintegration/BUILD.bazel b/dev/gitserverintegration/BUILD.bazel new file mode 100644 index 00000000000..3043eaa05f5 --- /dev/null +++ b/dev/gitserverintegration/BUILD.bazel @@ -0,0 +1,22 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "gitserverintegration", + srcs = ["testtools.go"], + importpath = "github.com/sourcegraph/sourcegraph/dev/gitserverintegration", + visibility = ["//visibility:public"], + deps = [ + "//cmd/gitserver/shared", + "//internal/api", + "//internal/database/dbmocks", + "//internal/gitserver", + "//internal/gitserver/v1:gitserver", + "//internal/grpc/defaults", + "//internal/observation", + "//internal/types", + "//lib/errors", + "@com_github_sourcegraph_log//logtest", + "@com_github_stretchr_testify//require", + "@org_golang_google_grpc//:go_default_library", + ], +) diff --git a/dev/gitserverintegration/testtools.go b/dev/gitserverintegration/testtools.go new file mode 100644 index 00000000000..98fdca39997 --- /dev/null +++ b/dev/gitserverintegration/testtools.go @@ -0,0 +1,109 @@ +// package gitserverintegration provides utilities for testing against a real gitserver +// in integration testing. +package gitserverintegration + +import ( + "context" + "testing" + + "github.com/sourcegraph/log/logtest" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + "github.com/sourcegraph/sourcegraph/cmd/gitserver/shared" + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/database/dbmocks" + "github.com/sourcegraph/sourcegraph/internal/gitserver" + v1 "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" + "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/internal/types" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +// NewTestGitserverWithRepos spawns a new gitserver with the given repos cloned, +// the map holds the repo name to path on disk mappings. The repos will be cloned +// from the location on disk into gitserver, for a most realistic setup. +// Two clients will be returned to interact with the gitserver. +func NewTestGitserverWithRepos(t *testing.T, repos map[api.RepoName]string) (gitserver.Client, v1.GitserverRepositoryServiceClient) { + // Create supporting infrastructure: + ctx := context.Background() + logger := logtest.Scoped(t) + obsCtx := observation.TestContextTB(t) + reposDir := t.TempDir() + + // Create a mock database: + db := dbmocks.NewMockDB() + repoStore := dbmocks.NewMockRepoStore() + repoStore.GetByNameFunc.SetDefaultHook(func(ctx context.Context, rn api.RepoName) (*types.Repo, error) { + if _, ok := repos[rn]; !ok { + return nil, errors.New("repo not found") + } + return &types.Repo{ID: 1, Name: rn}, nil + }) + db.ReposFunc.SetDefaultReturn(repoStore) + db.GitserverReposFunc.SetDefaultReturn(dbmocks.NewMockGitserverRepoStore()) + + // Spawn a test gitserver on a pseudo random port: + testAddr := "127.0.0.1:29484" + routine, err := shared.TestAPIServer(ctx, obsCtx, db, &shared.Config{ + ReposDir: reposDir, + ExhaustiveRequestLoggingEnabled: true, + ListenAddress: testAddr, + }, func(ctx context.Context, repo api.RepoName) (string, error) { + if _, ok := repos[repo]; !ok { + return "", errors.New("invalid repo name passed to getRemoteURL func") + } + + // We make gitserver clone the repo from the local dir where we create our test repo: + return repos[repo], nil + }) + require.NoError(t, err) + + // Start the gitserver up and make sure we shut down cleanly on test exit: + go routine.Start() + t.Cleanup(func() { + require.NoError(t, routine.Stop(context.Background())) + }) + + // Create a gitserver.Client to talk to the gitserver: + gs := gitserver.NewTestClient(t).WithClientSource(gitserver.NewTestClientSource(t, []string{testAddr}, func(o *gitserver.TestClientSourceOptions) { + o.ClientFunc = func(conn *grpc.ClientConn) v1.GitserverServiceClient { + return v1.NewGitserverServiceClient(conn) + } + })) + + // Also create a GitserverRepositoryServiceClient to talk to the gitserver: + conn, err := defaults.Dial(testAddr, logger) + require.NoError(t, err) + rs := v1.NewGitserverRepositoryServiceClient(conn) + + // Ensure all the requested repos are cloned into the gitserver: + for repo := range repos { + _, err = rs.FetchRepository(ctx, &v1.FetchRepositoryRequest{ + RepoName: string(repo), + }) + require.NoError(t, err) + } + + return gs, rs +} + +// RepoWithCommands is a helper method to create a git repo with the given commands. +// The repo will be created in a temporary directory, and can be passed to NewTestGitserverWithRepos. +func RepoWithCommands(t *testing.T, cmds ...string) string { + tmpDir := t.TempDir() + // Prepare repo state: + for _, cmd := range append( + append([]string{"git init --initial-branch=master ."}, cmds...), + // Promote the repo to a bare repo. + "git config --bool core.bare true", + ) { + out, err := gitserver.CreateGitCommand(tmpDir, "bash", "-c", cmd).CombinedOutput() + if err != nil { + t.Fatalf("Failed to run git command %v. Output was:\n\n%s", cmd, out) + } + } + + return tmpDir +} diff --git a/dev/rockskipintegration/BUILD.bazel b/dev/rockskipintegration/BUILD.bazel new file mode 100644 index 00000000000..8fa5d130a50 --- /dev/null +++ b/dev/rockskipintegration/BUILD.bazel @@ -0,0 +1,37 @@ +load("//dev:go_defs.bzl", "go_test") + +go_test( + name = "rockskipintegration_test", + srcs = ["main_test.go"], + data = ["//dev/tools:universal-ctags"], + env = { + "CTAGS_RLOCATIONPATH": "$(rlocationpath //dev/tools:universal-ctags)", + }, + tags = [ + TAG_PLATFORM_SEARCH, + # Test requires talking to real gitserver over network + "requires-network", + ], + deps = [ + "//cmd/symbols/fetcher", + "//cmd/symbols/gitserver", + "//cmd/symbols/parser", + "//cmd/symbols/types", + "//dev/gitserverintegration", + "//internal/api", + "//internal/ctags_config", + "//internal/database", + "//internal/database/dbtest", + "//internal/env", + "//internal/observation", + "//internal/rockskip", + "//internal/search", + "//internal/search/result", + "//internal/types", + "@com_github_sourcegraph_go_ctags//:go-ctags", + "@com_github_sourcegraph_log//:log", + "@com_github_sourcegraph_log//logtest", + "@com_github_stretchr_testify//require", + "@io_bazel_rules_go//go/runfiles:go_default_library", + ], +) diff --git a/dev/rockskipintegration/main_test.go b/dev/rockskipintegration/main_test.go new file mode 100644 index 00000000000..e19a63d4afd --- /dev/null +++ b/dev/rockskipintegration/main_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "os" + "os/exec" + "testing" + + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/sourcegraph/go-ctags" + "github.com/sourcegraph/log" + "github.com/sourcegraph/log/logtest" + "github.com/stretchr/testify/require" + + "github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher" + symbolsgitserver "github.com/sourcegraph/sourcegraph/cmd/symbols/gitserver" + symbolsParser "github.com/sourcegraph/sourcegraph/cmd/symbols/parser" + symbolstypes "github.com/sourcegraph/sourcegraph/cmd/symbols/types" + "github.com/sourcegraph/sourcegraph/dev/gitserverintegration" + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/ctags_config" + "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/database/dbtest" + "github.com/sourcegraph/sourcegraph/internal/env" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/internal/rockskip" + "github.com/sourcegraph/sourcegraph/internal/search" + "github.com/sourcegraph/sourcegraph/internal/search/result" + "github.com/sourcegraph/sourcegraph/internal/types" +) + +func TestRockskipIntegration(t *testing.T) { + gs, _ := gitserverintegration.NewTestGitserverWithRepos(t, map[api.RepoName]string{ + "github.com/sourcegraph/rockskiptest": gitserverintegration.RepoWithCommands(t, + "echo '# Title' > README.md", + "git add README.md", + "git commit -m commit --author='Foo Author '", + ), + }) + + ctx := context.Background() + observationCtx := observation.TestContextTB(t) + + // Verify gitserver cloned correctly: + head, headSHA, err := gs.GetDefaultBranch(ctx, "github.com/sourcegraph/rockskiptest", false) + require.NoError(t, err) + require.Equal(t, "refs/heads/master", head) + + db := dbtest.NewDB(t) + require.NoError(t, database.NewDB(logtest.Scoped(t), db).Repos().Create(ctx, &types.Repo{Name: "github.com/sourcegraph/rockskiptest"})) + _, err = db.ExecContext(ctx, "INSERT INTO rockskip_repos (repo, last_accessed_at) VALUES ($1, NOW())", "github.com/sourcegraph/rockskiptest") + require.NoError(t, err) + + sgs := symbolsgitserver.NewClient(observationCtx, gs) + ctagsConfig := symbolstypes.LoadCtagsConfig(env.BaseConfig{}) + // Try to find the universal ctags binary. In bazel, it will be provided by bazel. + // Outside of bazel, we rely on the system. + if os.Getenv("BAZEL_TEST") != "" { + ctagsConfig.UniversalCommand, _ = runfiles.Rlocation(os.Getenv("CTAGS_RLOCATIONPATH")) + } else { + _, err = exec.LookPath(ctagsConfig.UniversalCommand) + if err != nil { + // universal-ctags installed with brew is called ctags, try that next: + _, err = exec.LookPath("ctags") + if err == nil { + ctagsConfig.UniversalCommand = "ctags" + // In bazel, we expose the path to ctags via an environment variable. + } + } + } + svc, err := rockskip.NewService( + observationCtx, + db, + sgs, + fetcher.NewRepositoryFetcher(observationCtx, sgs, 100000, 1000), + func() (ctags.Parser, error) { + return symbolsParser.SpawnCtags(log.Scoped("parser"), ctagsConfig, ctags_config.UniversalCtags) + }, + // TODO: Adjust these numbers as needed: + 1, 1, true, 1, 1024, 1024, true, + ) + require.NoError(t, err) + + require.NoError(t, svc.Index(ctx, "github.com/sourcegraph/rockskiptest", string(headSHA))) + + // TODO: Properly validate rockskip data here: + res, err := svc.Search(ctx, search.SymbolsParameters{ + Repo: "github.com/sourcegraph/rockskiptest", + CommitID: api.CommitID(headSHA), + }) + require.NoError(t, err) + require.Equal(t, []result.Symbol{ + { + Name: "Title", + Path: "README.md", + Line: 0, + Character: 2, + Kind: "chapter", + }, + }, res) +} diff --git a/internal/gitserver/v1/BUILD.bazel b/internal/gitserver/v1/BUILD.bazel index 9e7a4e57735..2617b208ca7 100644 --- a/internal/gitserver/v1/BUILD.bazel +++ b/internal/gitserver/v1/BUILD.bazel @@ -31,6 +31,7 @@ go_library( visibility = [ "//cmd/gitserver:__subpackages__", "//cmd/repo-updater/internal/gitserver:__pkg__", + "//dev/gitserverintegration:__pkg__", "//internal/api:__pkg__", "//internal/extsvc/gitolite:__pkg__", "//internal/gitserver:__pkg__",