From bb09a4ac1fdd40a69c63eb052679043935694e9e Mon Sep 17 00:00:00 2001 From: Erik Seliger Date: Thu, 11 Jan 2024 19:46:32 +0100 Subject: [PATCH] Remove HTTP for inter-service RPC (#59093) In the upcoming release, we will only support gRPC going forward. This PR removes the old HTTP client and server implementations and a few leftovers from the transition. --- cmd/frontend/backend/external_services.go | 3 +- cmd/frontend/graphqlbackend/BUILD.bazel | 1 - .../graphqlbackend/external_service.go | 2 + .../graphqlbackend/graphqlbackend_test.go | 68 +- cmd/frontend/internal/httpapi/httpapi.go | 7 +- cmd/frontend/internal/httpapi/internal.go | 1 + .../internal/httpapi/releasecache/http.go | 2 +- cmd/frontend/internal/httpapi/search.go | 138 -- cmd/frontend/internal/httpapi/search_test.go | 507 ++---- .../internal/repos/webhooks/BUILD.bazel | 1 - .../internal/repos/webhooks/handlers_test.go | 9 - cmd/gitserver/internal/BUILD.bazel | 6 +- cmd/gitserver/internal/disk.go | 22 - .../internal/integration_tests/BUILD.bazel | 1 - .../integration_tests/archivereader_test.go | 114 +- .../internal/integration_tests/object_test.go | 40 +- .../internal/integration_tests/test_utils.go | 3 +- cmd/gitserver/internal/list_gitolite.go | 14 - cmd/gitserver/internal/list_gitolite_test.go | 68 - cmd/gitserver/internal/p4exec.go | 79 - cmd/gitserver/internal/p4exec_test.go | 244 +-- cmd/gitserver/internal/patch.go | 21 - cmd/gitserver/internal/repo_info.go | 38 - cmd/gitserver/internal/repo_info_test.go | 124 ++ cmd/gitserver/internal/server.go | 771 +-------- cmd/gitserver/internal/server_grpc.go | 9 +- cmd/gitserver/internal/server_test.go | 600 +++---- cmd/gitserver/internal/serverutil.go | 108 -- cmd/gitserver/internal/serverutil_test.go | 45 - .../internal/repoupdater/BUILD.bazel | 1 - .../internal/repoupdater/grpc_server.go | 2 +- .../internal/repoupdater/server.go | 101 -- .../internal/repoupdater/server_test.go | 105 -- cmd/repo-updater/shared/main.go | 2 +- cmd/searcher/internal/search/BUILD.bazel | 4 +- cmd/searcher/internal/search/hybrid_test.go | 5 +- cmd/searcher/internal/search/search.go | 82 - cmd/searcher/internal/search/search_grpc.go | 30 +- cmd/searcher/internal/search/search_test.go | 441 +++--- cmd/searcher/internal/search/store_test.go | 10 - cmd/searcher/shared/BUILD.bazel | 3 - cmd/searcher/shared/shared.go | 10 +- cmd/symbols/internal/api/BUILD.bazel | 2 - cmd/symbols/internal/api/handler.go | 39 - cmd/symbols/internal/api/handler_cgo.go | 17 +- cmd/symbols/internal/api/handler_nocgo.go | 22 +- cmd/symbols/internal/api/handler_test.go | 2 - cmd/symbols/squirrel/BUILD.bazel | 2 - cmd/symbols/squirrel/http_handlers.go | 128 -- .../internal/embeddings/repo/BUILD.bazel | 5 +- .../embeddings/repo/document_ranks.go | 61 - .../internal/embeddings/repo/handler.go | 4 +- cmd/worker/internal/embeddings/repo/worker.go | 4 + dev/run-server-image.sh | 2 - doc/admin/observability/alerts.md | 290 ---- doc/admin/observability/dashboards.md | 404 ++--- internal/api/internalapi/BUILD.bazel | 5 - internal/api/internalapi/client.go | 120 +- internal/batches/sources/BUILD.bazel | 1 + internal/batches/sources/mocks_test.go | 128 ++ internal/conf/BUILD.bazel | 2 - internal/conf/grpc.go | 20 - internal/conf/grpc_test.go | 45 - internal/embeddings/client.go | 3 +- internal/extsvc/versions/BUILD.bazel | 1 + internal/extsvc/versions/sync.go | 3 +- internal/gitserver/BUILD.bazel | 8 +- internal/gitserver/client.go | 1398 ++++------------- internal/gitserver/client_test.go | 513 +----- internal/gitserver/commands.go | 172 +- internal/gitserver/git_command.go | 45 - internal/gitserver/gitolite.go | 85 - internal/gitserver/mock.go | 875 +++++++++++ internal/gitserver/mocks_temp.go | 125 ++ internal/gitserver/proxy.go | 56 - internal/gitserver/search/search_test.go | 2 +- internal/repos/BUILD.bazel | 1 + internal/repos/gitolite.go | 26 +- internal/repos/gitolite_test.go | 22 +- internal/repos/sources.go | 12 +- internal/repos/sources_test.go | 48 +- internal/repos/sources_test_utils.go | 13 - .../excluded-repos-are-never-yielded.yaml | 22 - ...yPathPattern-determines-the-repo-name.yaml | 22 - .../yielded-repos-are-always-enabled.yaml | 22 - internal/repoupdater/BUILD.bazel | 10 - internal/repoupdater/client.go | 227 +-- internal/repoupdater/client_test.go | 14 - internal/search/alert/BUILD.bazel | 1 + internal/search/alert/observer.go | 12 +- internal/search/backend/BUILD.bazel | 2 - internal/search/backend/grpc.go | 41 - internal/search/backend/horizontal.go | 7 - internal/search/backend/zoekt.go | 38 +- internal/search/commit/commit.go | 2 +- internal/search/job/jobutil/BUILD.bazel | 1 + internal/search/job/jobutil/alert.go | 13 +- internal/search/job/jobutil/exhaustive_job.go | 2 +- .../job/jobutil/filter_file_contains.go | 16 +- .../job/jobutil/filter_file_contains_test.go | 4 +- internal/search/job/jobutil/repo_pager_job.go | 2 +- internal/search/job/jobutil/repos.go | 2 +- internal/search/repos/BUILD.bazel | 2 + internal/search/repos/repos.go | 47 +- internal/search/repos/repos_test.go | 19 +- internal/search/searcher/BUILD.bazel | 4 - internal/search/searcher/client.go | 148 +- internal/search/searcher/client_grpc.go | 125 -- internal/search/searcher/search.go | 81 +- internal/search/searcher/stream.go | 56 - internal/search/structural/structural.go | 2 +- internal/symbols/BUILD.bazel | 2 - internal/symbols/client.go | 174 +- mockgen.temp.yaml | 1 + monitoring/definitions/embeddings.go | 1 - monitoring/definitions/frontend.go | 11 - monitoring/definitions/git_server.go | 1 - .../definitions/precise_code_intel_worker.go | 1 - monitoring/definitions/repo_updater.go | 1 - monitoring/definitions/searcher.go | 1 - monitoring/definitions/shared/BUILD.bazel | 1 - monitoring/definitions/shared/frontend.go | 52 - monitoring/definitions/symbols.go | 1 - monitoring/definitions/worker.go | 1 - schema/schema.go | 3 - schema/site.schema.json | 8 - sg.config.yaml | 1 - testing/BUILD.bazel | 46 - testing/tools/integration_runner.sh | 8 +- 129 files changed, 2879 insertions(+), 6932 deletions(-) create mode 100644 cmd/gitserver/internal/repo_info_test.go delete mode 100644 cmd/gitserver/internal/serverutil_test.go delete mode 100644 cmd/symbols/squirrel/http_handlers.go delete mode 100644 cmd/worker/internal/embeddings/repo/document_ranks.go delete mode 100644 internal/conf/grpc.go delete mode 100644 internal/conf/grpc_test.go delete mode 100644 internal/gitserver/gitolite.go delete mode 100644 internal/gitserver/proxy.go delete mode 100644 internal/repos/testdata/sources/GITOLITE/excluded-repos-are-never-yielded.yaml delete mode 100644 internal/repos/testdata/sources/GITOLITE/repositoryPathPattern-determines-the-repo-name.yaml delete mode 100644 internal/repos/testdata/sources/GITOLITE/yielded-repos-are-always-enabled.yaml delete mode 100644 internal/repoupdater/client_test.go delete mode 100644 internal/search/searcher/client_grpc.go delete mode 100644 internal/search/searcher/stream.go delete mode 100644 monitoring/definitions/shared/frontend.go diff --git a/cmd/frontend/backend/external_services.go b/cmd/frontend/backend/external_services.go index ead0414720e..3bc3dc3a159 100644 --- a/cmd/frontend/backend/external_services.go +++ b/cmd/frontend/backend/external_services.go @@ -20,6 +20,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/extsvc/github" "github.com/sourcegraph/sourcegraph/internal/extsvc/gitlab" "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/observation" internalrepos "github.com/sourcegraph/sourcegraph/internal/repos" @@ -440,5 +441,5 @@ func newGenericSourcer(logger log.Logger, db database.DB) internalrepos.Sourcer db = database.NewDBWith(sourcerLogger.Scoped("db"), db) dependenciesService := dependencies.NewService(observation.NewContext(logger), db) cf := httpcli.NewExternalClientFactory(httpcli.NewLoggingMiddleware(sourcerLogger)) - return internalrepos.NewSourcer(sourcerLogger, db, cf, internalrepos.WithDependenciesService(dependenciesService)) + return internalrepos.NewSourcer(sourcerLogger, db, cf, gitserver.NewClient("backend.external-services"), internalrepos.WithDependenciesService(dependenciesService)) } diff --git a/cmd/frontend/graphqlbackend/BUILD.bazel b/cmd/frontend/graphqlbackend/BUILD.bazel index 475c8f10276..ef0049137f1 100644 --- a/cmd/frontend/graphqlbackend/BUILD.bazel +++ b/cmd/frontend/graphqlbackend/BUILD.bazel @@ -486,7 +486,6 @@ go_test( "//internal/authz/permssync", "//internal/binary", "//internal/conf", - "//internal/conf/conftypes", "//internal/database", "//internal/database/basestore", "//internal/database/dbmocks", diff --git a/cmd/frontend/graphqlbackend/external_service.go b/cmd/frontend/graphqlbackend/external_service.go index a83aadf4b71..b1a18cea9a6 100644 --- a/cmd/frontend/graphqlbackend/external_service.go +++ b/cmd/frontend/graphqlbackend/external_service.go @@ -17,6 +17,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/errcode" "github.com/sourcegraph/sourcegraph/internal/extsvc" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gqlutil" "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/ratelimit" @@ -267,6 +268,7 @@ func (r *externalServiceResolver) CheckConnection(ctx context.Context) (*externa r.db, r.externalService, httpcli.ExternalClientFactory, + gitserver.NewClient("graphql.check-connection"), ) if err != nil { return nil, errors.Wrap(err, "failed to create source") diff --git a/cmd/frontend/graphqlbackend/graphqlbackend_test.go b/cmd/frontend/graphqlbackend/graphqlbackend_test.go index b23a34befb5..9ce016b0b8d 100644 --- a/cmd/frontend/graphqlbackend/graphqlbackend_test.go +++ b/cmd/frontend/graphqlbackend/graphqlbackend_test.go @@ -1,33 +1,30 @@ package graphqlbackend import ( + "context" "encoding/base64" - "encoding/json" "flag" "fmt" "io" "log" //nolint:logging // TODO move all logging to sourcegraph/log - "net/http" - "net/http/httptest" - "net/url" "os" "reflect" - "sync/atomic" "testing" + mockassert "github.com/derision-test/go-mockgen/testutil/assert" "github.com/grafana/regexp" + "github.com/graph-gophers/graphql-go" "github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log sglog "github.com/sourcegraph/log" "github.com/sourcegraph/log/logtest" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/sourcegraph/sourcegraph/cmd/frontend/backend" - "github.com/sourcegraph/sourcegraph/internal/conf" - "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/database/dbmocks" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/protocol" "github.com/sourcegraph/sourcegraph/internal/types" - "github.com/sourcegraph/sourcegraph/schema" ) func TestMain(m *testing.M) { @@ -91,28 +88,6 @@ func TestRepository(t *testing.T) { func TestRecloneRepository(t *testing.T) { resetMocks() - var gitserverCalled atomic.Bool - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - resp := protocol.RepoUpdateResponse{} - gitserverCalled.Store(true) - json.NewEncoder(w).Encode(&resp) - })) - defer srv.Close() - - serverURL, err := url.Parse(srv.URL) - assert.Nil(t, err) - conf.Mock(&conf.Unified{ - ServiceConnectionConfig: conftypes.ServiceConnections{ - GitServers: []string{serverURL.Host}, - }, SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, - }, - }) - defer conf.Mock(nil) - repos := dbmocks.NewMockRepoStore() repos.GetFunc.SetDefaultReturn(&types.Repo{ID: 1, Name: "github.com/gorilla/mux"}, nil) @@ -127,32 +102,19 @@ func TestRecloneRepository(t *testing.T) { db.UsersFunc.SetDefaultReturn(users) db.GitserverReposFunc.SetDefaultReturn(gitserverRepos) - called := backend.Mocks.Repos.MockDeleteRepositoryFromDisk(t, 1) + repoID := MarshalRepositoryID(1) - repoID := base64.StdEncoding.EncodeToString([]byte("Repository:1")) + gc := gitserver.NewMockClient() + gc.RequestRepoCloneFunc.SetDefaultReturn(&protocol.RepoCloneResponse{}, nil) + r := newSchemaResolver(db, gc) - RunTests(t, []*Test{ - { - Schema: mustParseGraphQLSchema(t, db), - Query: fmt.Sprintf(` - mutation { - recloneRepository(repo: "%s") { - alwaysNil - } - } - `, repoID), - ExpectedResult: ` - { - "recloneRepository": { - "alwaysNil": null - } - } - `, - }, - }) + _, err := r.RecloneRepository(context.Background(), &struct{ Repo graphql.ID }{Repo: repoID}) + require.NoError(t, err) - assert.True(t, *called) - assert.True(t, gitserverCalled.Load()) + // To reclone, we first make a request to delete the repository, followed by a request + // to clone the repository again. + mockassert.CalledN(t, gc.RemoveFunc, 1) + mockassert.CalledN(t, gc.RequestRepoCloneFunc, 1) } func TestDeleteRepositoryFromDisk(t *testing.T) { diff --git a/cmd/frontend/internal/httpapi/httpapi.go b/cmd/frontend/internal/httpapi/httpapi.go index ff437bac612..021579a76b1 100644 --- a/cmd/frontend/internal/httpapi/httpapi.go +++ b/cmd/frontend/internal/httpapi/httpapi.go @@ -247,11 +247,10 @@ func RegisterInternalServices( gitService := &gitServiceHandler{Gitserver: gsClient} m.Path("/git/{RepoName:.*}/info/refs").Methods("GET").Name(gitInfoRefs).Handler(trace.Route(handler(gitService.serveInfoRefs()))) m.Path("/git/{RepoName:.*}/git-upload-pack").Methods("GET", "POST").Name(gitUploadPack).Handler(trace.Route(handler(gitService.serveGitUploadPack()))) - m.Path("/repos/index").Methods("POST").Handler(trace.Route(handler(indexer.serveList))) + + // TODO: Can be removed after 5.3 is cut. m.Path("/configuration").Methods("POST").Handler(trace.Route(handler(serveConfiguration))) - m.Path("/ranks/{RepoName:.*}/documents").Methods("GET").Handler(trace.Route(handler(indexer.serveDocumentRanks))) - m.Path("/search/configuration").Methods("GET", "POST").Handler(trace.Route(handler(indexer.serveConfiguration))) - m.Path("/search/index-status").Methods("POST").Handler(trace.Route(handler(indexer.handleIndexStatusUpdate))) + m.Path("/lsif/upload").Methods("POST").Handler(trace.Route(newCodeIntelUploadHandler(false))) m.Path("/scip/upload").Methods("POST").Handler(trace.Route(newCodeIntelUploadHandler(false))) m.Path("/scip/upload").Methods("HEAD").Handler(trace.Route(noopHandler)) diff --git a/cmd/frontend/internal/httpapi/internal.go b/cmd/frontend/internal/httpapi/internal.go index c58aa53c4be..d6ded988fd4 100644 --- a/cmd/frontend/internal/httpapi/internal.go +++ b/cmd/frontend/internal/httpapi/internal.go @@ -15,6 +15,7 @@ import ( "github.com/sourcegraph/sourcegraph/lib/errors" ) +// TODO: Can be removed after 5.3 is cut. func serveConfiguration(w http.ResponseWriter, _ *http.Request) error { raw := conf.Raw() err := json.NewEncoder(w).Encode(raw) diff --git a/cmd/frontend/internal/httpapi/releasecache/http.go b/cmd/frontend/internal/httpapi/releasecache/http.go index b3d0ac18f43..8ffc3b942dc 100644 --- a/cmd/frontend/internal/httpapi/releasecache/http.go +++ b/cmd/frontend/internal/httpapi/releasecache/http.go @@ -135,7 +135,7 @@ func (h *handler) handleBranch(w http.ResponseWriter, branch string) { } w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Write(raw) + _, _ = w.Write(raw) } func (h *handler) handleWebhook(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/frontend/internal/httpapi/search.go b/cmd/frontend/internal/httpapi/search.go index e5252216c54..fb50393375e 100644 --- a/cmd/frontend/internal/httpapi/search.go +++ b/cmd/frontend/internal/httpapi/search.go @@ -1,15 +1,10 @@ package httpapi import ( - "bytes" "context" - "encoding/json" - "fmt" "net/http" - "strconv" "time" - "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sourcegraph/log" @@ -183,72 +178,6 @@ type searchIndexerServer struct { MinLastChangedDisabled bool } -// serveConfiguration is _only_ used by the zoekt index server. Zoekt does -// not depend on frontend and therefore does not have access to `conf.Watch`. -// Additionally, it only cares about certain search specific settings so this -// search specific endpoint is used rather than serving the entire site settings -// from /.internal/configuration. -// -// This endpoint also supports batch requests to avoid managing concurrency in -// zoekt. On vertically scaled instances we have observed zoekt requesting -// this endpoint concurrently leading to socket starvation. -func (h *searchIndexerServer) serveConfiguration(w http.ResponseWriter, r *http.Request) error { - ctx := r.Context() - if err := r.ParseForm(); err != nil { - return err - } - - indexedIDs := make([]api.RepoID, 0, len(r.Form["repoID"])) - for _, idStr := range r.Form["repoID"] { - id, err := strconv.Atoi(idStr) - if err != nil { - http.Error(w, fmt.Sprintf("invalid repo id %s: %s", idStr, err), http.StatusBadRequest) - return nil - } - indexedIDs = append(indexedIDs, api.RepoID(id)) - } - - var clientFingerprint searchbackend.ConfigFingerprint - err := clientFingerprint.FromHeaders(r.Header) - if err != nil { - http.Error(w, fmt.Sprintf("invalid fingerprint: %s", err), http.StatusBadRequest) - return nil - } - - response, err := h.doSearchConfiguration(ctx, searchConfigurationParameters{ - repoIDs: indexedIDs, - fingerprint: clientFingerprint, - }) - - if err != nil { - var parameterErr *parameterError - code := http.StatusInternalServerError - - if errors.As(err, ¶meterErr) { - code = http.StatusBadRequest - } - - http.Error(w, err.Error(), code) - return nil - } - - response.fingerprint.ToHeaders(w.Header()) - - jsonOptions := make([][]byte, 0, len(response.options)) - for _, opt := range response.options { - marshalled, err := json.Marshal(opt) - if err != nil { - _, _ = w.Write([]byte(err.Error())) - } - - jsonOptions = append(jsonOptions, marshalled) - } - - _, _ = w.Write(bytes.Join(jsonOptions, []byte("\n"))) - - return nil -} - func (h *searchIndexerServer) doSearchConfiguration(ctx context.Context, parameters searchConfigurationParameters) (*searchConfigurationResponse, error) { siteConfig := conf.Get().SiteConfiguration @@ -387,32 +316,6 @@ type searchConfigurationResponse struct { fingerprint searchbackend.ConfigFingerprint } -// serveList is used by zoekt to get the list of repositories for it to index. -func (h *searchIndexerServer) serveList(w http.ResponseWriter, r *http.Request) error { - var parameters listParameters - err := json.NewDecoder(r.Body).Decode(¶meters) - if err != nil { - return err - } - - repoIDs, err := h.doList(r.Context(), ¶meters) - if err != nil { - return err - } - - // TODO: Avoid batching up so much in memory by: - // 1. Changing the schema from object of arrays to array of objects. - // 2. Stream out each object marshalled rather than marshall the full list in memory. - - data := struct { - RepoIDs []api.RepoID - }{ - RepoIDs: repoIDs, - } - - return json.NewEncoder(w).Encode(&data) -} - func (h *searchIndexerServer) doList(ctx context.Context, parameters *listParameters) (repoIDS []api.RepoID, err error) { indexable, err := h.ListIndexable(ctx) if err != nil { @@ -460,47 +363,6 @@ var metricGetVersion = promauto.NewCounter(prometheus.CounterOpts{ Help: "The total number of times we poll gitserver for the version of a indexable branch.", }) -func (h *searchIndexerServer) serveDocumentRanks(w http.ResponseWriter, r *http.Request) error { - return serveRank(h.Ranking.GetDocumentRanks, w, r) -} - -func serveRank[T []float64 | citypes.RepoPathRanks]( - f func(ctx context.Context, name api.RepoName) (r T, err error), - w http.ResponseWriter, - r *http.Request, -) error { - ctx := r.Context() - - repoName := api.RepoName(mux.Vars(r)["RepoName"]) - - rank, err := f(ctx, repoName) - if err != nil { - if errcode.IsNotFound(err) { - http.Error(w, err.Error(), http.StatusNotFound) - return nil - } - return err - } - - b, err := json.Marshal(rank) - if err != nil { - return err - } - - _, _ = w.Write(b) - return nil -} - -func (h *searchIndexerServer) handleIndexStatusUpdate(_ http.ResponseWriter, r *http.Request) error { - var args indexStatusUpdateArgs - - if err := json.NewDecoder(r.Body).Decode(&args); err != nil { - return errors.Wrap(err, "failed to decode request args") - } - - return h.doIndexStatusUpdate(r.Context(), &args) -} - func (h *searchIndexerServer) doIndexStatusUpdate(ctx context.Context, args *indexStatusUpdateArgs) error { var ( ids = make([]int32, len(args.Repositories)) diff --git a/cmd/frontend/internal/httpapi/search_test.go b/cmd/frontend/internal/httpapi/search_test.go index 1aebd990315..1414a1fa071 100644 --- a/cmd/frontend/internal/httpapi/search_test.go +++ b/cmd/frontend/internal/httpapi/search_test.go @@ -1,13 +1,7 @@ package httpapi import ( - "bytes" "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "net/url" "sort" "strings" "testing" @@ -55,203 +49,141 @@ func TestServeConfiguration(t *testing.T) { } rankingService := &fakeRankingService{} - t.Run("gRPC", func(t *testing.T) { - - // Set up the GRPC server - grpcServer := searchIndexerGRPCServer{ - server: &searchIndexerServer{ - RepoStore: repoStore, - gitserverClient: gsClient, - Ranking: rankingService, - SearchContextsRepoRevs: searchContextRepoRevsFunc, - }, - } - - // Setup: create a request for repos 5 and 6, and the non-existent repo 1 - requestedRepoIDs := []int32{1, 5, 6} - - // Execute the first request (no fingerprint) - var initialRequest proto.SearchConfigurationRequest - initialRequest.RepoIds = requestedRepoIDs - initialRequest.Fingerprint = nil - - initialResponse, err := grpcServer.SearchConfiguration(context.Background(), &initialRequest) - if err != nil { - t.Fatalf("SearchConfiguration: %s", err) - } - - // Verify: Check to see that the response contains an error - // for the non-existent repo 1 - var responseRepo1 *proto.ZoektIndexOptions - foundRepo1 := false - - var receivedRepositories []*proto.ZoektIndexOptions - - for _, repo := range initialResponse.GetUpdatedOptions() { - if repo.RepoId == 1 { - responseRepo1 = repo - foundRepo1 = true - continue - } - - sort.Slice(repo.LanguageMap, func(i, j int) bool { - return repo.LanguageMap[i].Language > repo.LanguageMap[j].Language - }) - receivedRepositories = append(receivedRepositories, repo) - } - - if !foundRepo1 { - t.Errorf("expected to find repo ID 1 in response: %v", receivedRepositories) - } - - if foundRepo1 && !strings.Contains(responseRepo1.Error, "repo not found") { - t.Errorf("expected to find repo not found error in repo 1: %v", responseRepo1) - } - - languageMap := make([]*proto.LanguageMapping, 0) - for lang, engine := range ctags_config.DefaultEngines { - languageMap = append(languageMap, &proto.LanguageMapping{Language: lang, Ctags: proto.CTagsParserType(engine)}) - } - - sort.Slice(languageMap, func(i, j int) bool { - return languageMap[i].Language > languageMap[j].Language - }) - - // Verify: Check to see that the response the expected repos 5 and 6 - expectedRepo5 := &proto.ZoektIndexOptions{ - RepoId: 5, - Name: "5", - Priority: 5, - Public: true, - Symbols: true, - Branches: []*proto.ZoektRepositoryBranch{{Name: "HEAD", Version: "!HEAD"}}, - LanguageMap: languageMap, - } - - expectedRepo6 := &proto.ZoektIndexOptions{ - RepoId: 6, - Name: "6", - Priority: 6, - Public: true, - Symbols: true, - Branches: []*proto.ZoektRepositoryBranch{ - {Name: "HEAD", Version: "!HEAD"}, - {Name: "a", Version: "!a"}, - {Name: "b", Version: "!b"}, - }, - LanguageMap: languageMap, - } - - expectedRepos := []*proto.ZoektIndexOptions{ - expectedRepo5, - expectedRepo6, - } - - sort.Slice(receivedRepositories, func(i, j int) bool { - return receivedRepositories[i].RepoId < receivedRepositories[j].RepoId - }) - sort.Slice(expectedRepos, func(i, j int) bool { - return expectedRepos[i].RepoId < expectedRepos[j].RepoId - }) - - if diff := cmp.Diff(expectedRepos, receivedRepositories, protocmp.Transform()); diff != "" { - t.Fatalf("mismatch in response repositories (-want, +got):\n%s", diff) - } - - if initialResponse.GetFingerprint() == nil { - t.Fatalf("expected fingerprint to be set in initial response") - } - - // Setup: run a second request with the fingerprint from the first response - // Note: when fingerprint is set we only return a subset. We simulate this by setting RepoStore to only list repo number 5 - grpcServer.server.RepoStore = &fakeRepoStore{Repos: repos[:1]} - - var fingerprintedRequest proto.SearchConfigurationRequest - fingerprintedRequest.RepoIds = requestedRepoIDs - fingerprintedRequest.Fingerprint = initialResponse.GetFingerprint() - - // Execute the seconds request - fingerprintedResponse, err := grpcServer.SearchConfiguration(context.Background(), &fingerprintedRequest) - if err != nil { - t.Fatalf("SearchConfiguration: %s", err) - } - - fingerprintedResponses := fingerprintedResponse.GetUpdatedOptions() - - for _, res := range fingerprintedResponses { - sort.Slice(res.LanguageMap, func(i, j int) bool { - return res.LanguageMap[i].Language > res.LanguageMap[j].Language - }) - } - - // Verify that the response contains the expected repo 5 - if diff := cmp.Diff(fingerprintedResponses, []*proto.ZoektIndexOptions{expectedRepo5}, protocmp.Transform()); diff != "" { - t.Errorf("mismatch in fingerprinted repositories (-want, +got):\n%s", diff) - } - - if fingerprintedResponse.GetFingerprint() == nil { - t.Fatalf("expected fingerprint to be set in fingerprinted response") - } - }) - - t.Run("REST", func(t *testing.T) { - srv := &searchIndexerServer{ + // Set up the GRPC server + grpcServer := searchIndexerGRPCServer{ + server: &searchIndexerServer{ RepoStore: repoStore, gitserverClient: gsClient, Ranking: rankingService, SearchContextsRepoRevs: searchContextRepoRevsFunc, + }, + } + + // Setup: create a request for repos 5 and 6, and the non-existent repo 1 + requestedRepoIDs := []int32{1, 5, 6} + + // Execute the first request (no fingerprint) + var initialRequest proto.SearchConfigurationRequest + initialRequest.RepoIds = requestedRepoIDs + initialRequest.Fingerprint = nil + + initialResponse, err := grpcServer.SearchConfiguration(context.Background(), &initialRequest) + if err != nil { + t.Fatalf("SearchConfiguration: %s", err) + } + + // Verify: Check to see that the response contains an error + // for the non-existent repo 1 + var responseRepo1 *proto.ZoektIndexOptions + foundRepo1 := false + + var receivedRepositories []*proto.ZoektIndexOptions + + for _, repo := range initialResponse.GetUpdatedOptions() { + if repo.RepoId == 1 { + responseRepo1 = repo + foundRepo1 = true + continue } - data := url.Values{ - "repoID": []string{"1", "5", "6"}, - } - req := httptest.NewRequest("POST", "/", strings.NewReader(data.Encode())) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - w := httptest.NewRecorder() - if err := srv.serveConfiguration(w, req); err != nil { - t.Fatal(err) - } + sort.Slice(repo.LanguageMap, func(i, j int) bool { + return repo.LanguageMap[i].Language > repo.LanguageMap[j].Language + }) + receivedRepositories = append(receivedRepositories, repo) + } - resp := w.Result() - body, _ := io.ReadAll(resp.Body) + if !foundRepo1 { + t.Errorf("expected to find repo ID 1 in response: %v", receivedRepositories) + } - // This is a very fragile test since it will depend on changes to - // searchbackend.GetIndexOptions. If this becomes a problem we can make it - // more robust by shifting around responsibilities. - want := `{"Name":"","RepoID":1,"Public":false,"Fork":false,"Archived":false,"LargeFiles":null,"Symbols":false,"Error":"repo not found: id=1","LanguageMap":null} -{"Name":"5","RepoID":5,"Public":true,"Fork":false,"Archived":false,"LargeFiles":null,"Symbols":true,"Branches":[{"Name":"HEAD","Version":"!HEAD"}],"Priority":5,"LanguageMap":{"c_sharp":3,"go":3,"javascript":3,"kotlin":3,"python":3,"ruby":3,"rust":3,"scala":3,"typescript":3,"zig":3}} -{"Name":"6","RepoID":6,"Public":true,"Fork":false,"Archived":false,"LargeFiles":null,"Symbols":true,"Branches":[{"Name":"HEAD","Version":"!HEAD"},{"Name":"a","Version":"!a"},{"Name":"b","Version":"!b"}],"Priority":6,"LanguageMap":{"c_sharp":3,"go":3,"javascript":3,"kotlin":3,"python":3,"ruby":3,"rust":3,"scala":3,"typescript":3,"zig":3}}` + if foundRepo1 && !strings.Contains(responseRepo1.Error, "repo not found") { + t.Errorf("expected to find repo not found error in repo 1: %v", responseRepo1) + } - if d := cmp.Diff(want, string(body)); d != "" { - t.Fatalf("mismatch (-want, +got):\n%s", d) - } + languageMap := make([]*proto.LanguageMapping, 0) + for lang, engine := range ctags_config.DefaultEngines { + languageMap = append(languageMap, &proto.LanguageMapping{Language: lang, Ctags: proto.CTagsParserType(engine)}) + } - // when fingerprint is set we only return a subset. We simulate this by setting RepoStore to only list repo number 5 - srv.RepoStore = &fakeRepoStore{Repos: repos[:1]} - req = httptest.NewRequest("POST", "/", strings.NewReader(data.Encode())) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("X-Sourcegraph-Config-Fingerprint", resp.Header.Get("X-Sourcegraph-Config-Fingerprint")) - - w = httptest.NewRecorder() - if err := srv.serveConfiguration(w, req); err != nil { - t.Fatal(err) - } - - resp = w.Result() - body, _ = io.ReadAll(resp.Body) - - // We want the same as before, except we only want to get back 5. - // - // This is a very fragile test since it will depend on changes to - // searchbackend.GetIndexOptions. If this becomes a problem we can make it - // more robust by shifting around responsibilities. - want = `{"Name":"5","RepoID":5,"Public":true,"Fork":false,"Archived":false,"LargeFiles":null,"Symbols":true,"Branches":[{"Name":"HEAD","Version":"!HEAD"}],"Priority":5,"LanguageMap":{"c_sharp":3,"go":3,"javascript":3,"kotlin":3,"python":3,"ruby":3,"rust":3,"scala":3,"typescript":3,"zig":3}}` - - if d := cmp.Diff(want, string(body)); d != "" { - t.Fatalf("mismatch (-want, +got):\n%s", d) - } + sort.Slice(languageMap, func(i, j int) bool { + return languageMap[i].Language > languageMap[j].Language }) + // Verify: Check to see that the response the expected repos 5 and 6 + expectedRepo5 := &proto.ZoektIndexOptions{ + RepoId: 5, + Name: "5", + Priority: 5, + Public: true, + Symbols: true, + Branches: []*proto.ZoektRepositoryBranch{{Name: "HEAD", Version: "!HEAD"}}, + LanguageMap: languageMap, + } + + expectedRepo6 := &proto.ZoektIndexOptions{ + RepoId: 6, + Name: "6", + Priority: 6, + Public: true, + Symbols: true, + Branches: []*proto.ZoektRepositoryBranch{ + {Name: "HEAD", Version: "!HEAD"}, + {Name: "a", Version: "!a"}, + {Name: "b", Version: "!b"}, + }, + LanguageMap: languageMap, + } + + expectedRepos := []*proto.ZoektIndexOptions{ + expectedRepo5, + expectedRepo6, + } + + sort.Slice(receivedRepositories, func(i, j int) bool { + return receivedRepositories[i].RepoId < receivedRepositories[j].RepoId + }) + sort.Slice(expectedRepos, func(i, j int) bool { + return expectedRepos[i].RepoId < expectedRepos[j].RepoId + }) + + if diff := cmp.Diff(expectedRepos, receivedRepositories, protocmp.Transform()); diff != "" { + t.Fatalf("mismatch in response repositories (-want, +got):\n%s", diff) + } + + if initialResponse.GetFingerprint() == nil { + t.Fatalf("expected fingerprint to be set in initial response") + } + + // Setup: run a second request with the fingerprint from the first response + // Note: when fingerprint is set we only return a subset. We simulate this by setting RepoStore to only list repo number 5 + grpcServer.server.RepoStore = &fakeRepoStore{Repos: repos[:1]} + + var fingerprintedRequest proto.SearchConfigurationRequest + fingerprintedRequest.RepoIds = requestedRepoIDs + fingerprintedRequest.Fingerprint = initialResponse.GetFingerprint() + + // Execute the seconds request + fingerprintedResponse, err := grpcServer.SearchConfiguration(context.Background(), &fingerprintedRequest) + if err != nil { + t.Fatalf("SearchConfiguration: %s", err) + } + + fingerprintedResponses := fingerprintedResponse.GetUpdatedOptions() + + for _, res := range fingerprintedResponses { + sort.Slice(res.LanguageMap, func(i, j int) bool { + return res.LanguageMap[i].Language > res.LanguageMap[j].Language + }) + } + + // Verify that the response contains the expected repo 5 + if diff := cmp.Diff(fingerprintedResponses, []*proto.ZoektIndexOptions{expectedRepo5}, protocmp.Transform()); diff != "" { + t.Errorf("mismatch in fingerprinted repositories (-want, +got):\n%s", diff) + } + + if fingerprintedResponse.GetFingerprint() == nil { + t.Fatalf("expected fingerprint to be set in fingerprinted response") + } } func TestReposIndex(t *testing.T) { @@ -320,84 +252,38 @@ func TestReposIndex(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - t.Run("gRPC", func(t *testing.T) { - grpcServer := &searchIndexerGRPCServer{ - server: &searchIndexerServer{ - ListIndexable: fakeListIndexable(tc.indexable), - RepoStore: &fakeRepoStore{ - Repos: allRepos, - }, - Indexers: suffixIndexers(true), - }, - } - - resp, err := grpcServer.List(context.Background(), tc.parameters.grpcRequest) - if err != nil { - t.Fatal(err) - } - - expectedRepoIDs := make([]api.RepoID, len(tc.want)) - for i, name := range tc.want { - for _, repo := range allRepos { - if string(repo.Name) == name { - expectedRepoIDs[i] = repo.ID - } - } - } - - var receivedRepoIDs []api.RepoID - for _, id := range resp.GetRepoIds() { - receivedRepoIDs = append(receivedRepoIDs, api.RepoID(id)) - } - - if d := cmp.Diff(expectedRepoIDs, receivedRepoIDs, cmpopts.EquateEmpty()); d != "" { - t.Fatalf("ids mismatch (-want +got):\n%s", d) - } - - }) - - t.Run("REST", func(t *testing.T) { - - srv := &searchIndexerServer{ + grpcServer := &searchIndexerGRPCServer{ + server: &searchIndexerServer{ ListIndexable: fakeListIndexable(tc.indexable), RepoStore: &fakeRepoStore{ Repos: allRepos, }, Indexers: suffixIndexers(true), - } + }, + } - req := httptest.NewRequest("POST", "/", bytes.NewReader([]byte(tc.parameters.restBody))) - w := httptest.NewRecorder() - if err := srv.serveList(w, req); err != nil { - t.Fatal(err) - } + resp, err := grpcServer.List(context.Background(), tc.parameters.grpcRequest) + if err != nil { + t.Fatal(err) + } - resp := w.Result() - body, _ := io.ReadAll(resp.Body) - - if resp.StatusCode != http.StatusOK { - t.Errorf("got status %v", resp.StatusCode) - } - - var data struct { - RepoIDs []api.RepoID - } - if err := json.Unmarshal(body, &data); err != nil { - t.Fatal(err) - } - - wantIDs := make([]api.RepoID, len(tc.want)) - for i, name := range tc.want { - for _, repo := range allRepos { - if string(repo.Name) == name { - wantIDs[i] = repo.ID - } + expectedRepoIDs := make([]api.RepoID, len(tc.want)) + for i, name := range tc.want { + for _, repo := range allRepos { + if string(repo.Name) == name { + expectedRepoIDs[i] = repo.ID } } - if d := cmp.Diff(wantIDs, data.RepoIDs); d != "" { - t.Fatalf("ids mismatch (-want +got):\n%s", d) - } - }) + } + + var receivedRepoIDs []api.RepoID + for _, id := range resp.GetRepoIds() { + receivedRepoIDs = append(receivedRepoIDs, api.RepoID(id)) + } + + if d := cmp.Diff(expectedRepoIDs, receivedRepoIDs, cmpopts.EquateEmpty()); d != "" { + t.Fatalf("ids mismatch (-want +got):\n%s", d) + } }) } } @@ -511,90 +397,45 @@ func TestRepoRankFromConfig(t *testing.T) { } func TestIndexStatusUpdate(t *testing.T) { + logger := logtest.Scoped(t) - t.Run("REST", func(t *testing.T) { - logger := logtest.Scoped(t) + wantRepoID := uint32(1234) + wantBranches := []zoekt.RepositoryBranch{{Name: "main", Version: "f00b4r"}} - body := `{"Repositories": [{"RepoID": 1234, "Branches": [{"Name": "main", "Version": "f00b4r"}]}]}` - wantBranches := []zoekt.RepositoryBranch{{Name: "main", Version: "f00b4r"}} - called := false + called := false - zoektReposStore := dbmocks.NewMockZoektReposStore() - zoektReposStore.UpdateIndexStatusesFunc.SetDefaultHook(func(_ context.Context, indexed zoekt.ReposMap) error { - entry, ok := indexed[1234] - if !ok { - t.Fatalf("wrong repo ID") - } - if d := cmp.Diff(entry.Branches, wantBranches); d != "" { - t.Fatalf("ids mismatch (-want +got):\n%s", d) - } - called = true - return nil - }) - - db := dbmocks.NewMockDB() - db.ZoektReposFunc.SetDefaultReturn(zoektReposStore) - - srv := &searchIndexerServer{db: db, logger: logger} - - req := httptest.NewRequest("POST", "/", bytes.NewReader([]byte(body))) - w := httptest.NewRecorder() - - if err := srv.handleIndexStatusUpdate(w, req); err != nil { - t.Fatal(err) + zoektReposStore := dbmocks.NewMockZoektReposStore() + zoektReposStore.UpdateIndexStatusesFunc.SetDefaultHook(func(_ context.Context, indexed zoekt.ReposMap) error { + entry, ok := indexed[wantRepoID] + if !ok { + t.Fatalf("wrong repo ID") } - - resp := w.Result() - if resp.StatusCode != http.StatusOK { - t.Errorf("got status %v", resp.StatusCode) - } - - if !called { - t.Fatalf("not called") + if d := cmp.Diff(entry.Branches, wantBranches); d != "" { + t.Fatalf("ids mismatch (-want +got):\n%s", d) } + called = true + return nil }) - t.Run("gRPC", func(t *testing.T) { - logger := logtest.Scoped(t) + db := dbmocks.NewMockDB() + db.ZoektReposFunc.SetDefaultReturn(zoektReposStore) - wantRepoID := uint32(1234) - wantBranches := []zoekt.RepositoryBranch{{Name: "main", Version: "f00b4r"}} + parameters := indexStatusUpdateArgs{ + Repositories: []indexStatusUpdateRepository{ + {RepoID: wantRepoID, Branches: wantBranches}, + }, + } - called := false + srv := &searchIndexerGRPCServer{server: &searchIndexerServer{db: db, logger: logger}} - zoektReposStore := dbmocks.NewMockZoektReposStore() - zoektReposStore.UpdateIndexStatusesFunc.SetDefaultHook(func(_ context.Context, indexed zoekt.ReposMap) error { - entry, ok := indexed[wantRepoID] - if !ok { - t.Fatalf("wrong repo ID") - } - if d := cmp.Diff(entry.Branches, wantBranches); d != "" { - t.Fatalf("ids mismatch (-want +got):\n%s", d) - } - called = true - return nil - }) + _, err := srv.UpdateIndexStatus(context.Background(), parameters.ToProto()) + if err != nil { + t.Fatal(err) + } - db := dbmocks.NewMockDB() - db.ZoektReposFunc.SetDefaultReturn(zoektReposStore) - - parameters := indexStatusUpdateArgs{ - Repositories: []indexStatusUpdateRepository{ - {RepoID: wantRepoID, Branches: wantBranches}, - }, - } - - srv := &searchIndexerGRPCServer{server: &searchIndexerServer{db: db, logger: logger}} - - _, err := srv.UpdateIndexStatus(context.Background(), parameters.ToProto()) - if err != nil { - t.Fatal(err) - } - - if !called { - t.Fatalf("not called") - } - }) + if !called { + t.Fatalf("not called") + } } func TestRepoPathRanks_RoundTrip(t *testing.T) { diff --git a/cmd/frontend/internal/repos/webhooks/BUILD.bazel b/cmd/frontend/internal/repos/webhooks/BUILD.bazel index 7ece75e5cbc..dede7b2e445 100644 --- a/cmd/frontend/internal/repos/webhooks/BUILD.bazel +++ b/cmd/frontend/internal/repos/webhooks/BUILD.bazel @@ -50,7 +50,6 @@ go_test( "//internal/extsvc/gitlab/webhooks", "//internal/grpc", "//internal/grpc/defaults", - "//internal/httpcli", "//internal/repos", "//internal/repoupdater", "//internal/repoupdater/protocol", diff --git a/cmd/frontend/internal/repos/webhooks/handlers_test.go b/cmd/frontend/internal/repos/webhooks/handlers_test.go index d58484239b7..954a330d238 100644 --- a/cmd/frontend/internal/repos/webhooks/handlers_test.go +++ b/cmd/frontend/internal/repos/webhooks/handlers_test.go @@ -33,7 +33,6 @@ import ( gitlabwebhooks "github.com/sourcegraph/sourcegraph/internal/extsvc/gitlab/webhooks" internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/repos" "github.com/sourcegraph/sourcegraph/internal/repoupdater" "github.com/sourcegraph/sourcegraph/internal/repoupdater/protocol" @@ -151,15 +150,7 @@ func TestGitHubHandler(t *testing.T) { server := httptest.NewServer(internalgrpc.MultiplexHandlers(gs, mux)) defer server.Close() - cf := httpcli.NewExternalClientFactory() - opts := []httpcli.Opt{} - doer, err := cf.Doer(opts...) - if err != nil { - t.Fatal(err) - } - repoupdater.DefaultClient = repoupdater.NewClient(server.URL) - repoupdater.DefaultClient.HTTPClient = doer payload, err := os.ReadFile(filepath.Join("testdata", "github-push.json")) if err != nil { diff --git a/cmd/gitserver/internal/BUILD.bazel b/cmd/gitserver/internal/BUILD.bazel index 7643624febc..df7408f3f4b 100644 --- a/cmd/gitserver/internal/BUILD.bazel +++ b/cmd/gitserver/internal/BUILD.bazel @@ -62,7 +62,6 @@ go_library( "//internal/observation", "//internal/perforce", "//internal/ratelimit", - "//internal/search/streaming/http", "//internal/security", "//internal/trace", "//internal/types", @@ -74,13 +73,11 @@ go_library( "@com_github_mxk_go_flowrate//flowrate", "@com_github_prometheus_client_golang//prometheus", "@com_github_prometheus_client_golang//prometheus/promauto", - "@com_github_sourcegraph_conc//:conc", "@com_github_sourcegraph_log//:log", "@com_github_sourcegraph_mountinfo//:mountinfo", "@io_opentelemetry_go_otel//attribute", "@org_golang_google_grpc//codes", "@org_golang_google_grpc//status", - "@org_golang_google_protobuf//encoding/protojson", "@org_golang_x_sync//errgroup", "@org_golang_x_sync//semaphore", "@org_golang_x_time//rate", @@ -95,8 +92,8 @@ go_test( "list_gitolite_test.go", "main_test.go", "p4exec_test.go", + "repo_info_test.go", "server_test.go", - "serverutil_test.go", ], data = glob(["testdata/**"]), embed = [":internal"], @@ -116,6 +113,7 @@ go_test( "//cmd/gitserver/internal/vcssyncer", "//internal/actor", "//internal/api", + "//internal/conf", "//internal/database", "//internal/database/dbmocks", "//internal/database/dbtest", diff --git a/cmd/gitserver/internal/disk.go b/cmd/gitserver/internal/disk.go index ea301b0e519..e0f3ad150c5 100644 --- a/cmd/gitserver/internal/disk.go +++ b/cmd/gitserver/internal/disk.go @@ -1,10 +1,6 @@ package internal import ( - "net/http" - - "google.golang.org/protobuf/encoding/protojson" - "github.com/sourcegraph/sourcegraph/internal/diskusage" proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" ) @@ -25,21 +21,3 @@ func getDiskInfo(dir string) (*proto.DiskInfoResponse, error) { PercentUsed: usage.PercentUsed(), }, nil } - -func (s *Server) handleDiskInfo(w http.ResponseWriter, r *http.Request) { - resp, err := getDiskInfo(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - jsonBytes, err := protojson.Marshal(resp) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(jsonBytes) -} diff --git a/cmd/gitserver/internal/integration_tests/BUILD.bazel b/cmd/gitserver/internal/integration_tests/BUILD.bazel index 47c81a2a244..f01646dade5 100644 --- a/cmd/gitserver/internal/integration_tests/BUILD.bazel +++ b/cmd/gitserver/internal/integration_tests/BUILD.bazel @@ -20,7 +20,6 @@ go_library( "//internal/gitserver/v1:gitserver", "//internal/grpc", "//internal/grpc/defaults", - "//internal/httpcli", "//internal/observation", "//internal/ratelimit", "//internal/types", diff --git a/cmd/gitserver/internal/integration_tests/archivereader_test.go b/cmd/gitserver/internal/integration_tests/archivereader_test.go index afd9b10ee71..d1d82e2537a 100644 --- a/cmd/gitserver/internal/integration_tests/archivereader_test.go +++ b/cmd/gitserver/internal/integration_tests/archivereader_test.go @@ -22,7 +22,6 @@ import ( server "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal" "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/vcssyncer" "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" @@ -31,7 +30,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/ratelimit" "github.com/sourcegraph/sourcegraph/internal/wrexec" "github.com/sourcegraph/sourcegraph/lib/errors" - "github.com/sourcegraph/sourcegraph/schema" ) func TestClient_ArchiveReader(t *testing.T) { @@ -185,97 +183,41 @@ func TestClient_ArchiveReader(t *testing.T) { }) } - t.Run("grpc", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(true), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - for _, test := range tests { - repoName := api.RepoName(test.name) - called := false + for _, test := range tests { + repoName := api.RepoName(test.name) + called := false - mkClient := func(t *testing.T, addrs []string) gitserver.Client { - t.Helper() + mkClient := func(t *testing.T, addrs []string) gitserver.Client { + t.Helper() - source := gitserver.NewTestClientSource(t, addrs, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { + source := gitserver.NewTestClientSource(t, addrs, func(o *gitserver.TestClientSourceOptions) { + o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { + base := proto.NewGitserverServiceClient(cc) + + mockArchive := func(ctx context.Context, in *proto.ArchiveRequest, opts ...grpc.CallOption) (proto.GitserverService_ArchiveClient, error) { + called = true + return base.Archive(ctx, in, opts...) + } + mockRepoUpdate := func(ctx context.Context, in *proto.RepoUpdateRequest, opts ...grpc.CallOption) (*proto.RepoUpdateResponse, error) { base := proto.NewGitserverServiceClient(cc) - - mockArchive := func(ctx context.Context, in *proto.ArchiveRequest, opts ...grpc.CallOption) (proto.GitserverService_ArchiveClient, error) { - called = true - return base.Archive(ctx, in, opts...) - } - mockRepoUpdate := func(ctx context.Context, in *proto.RepoUpdateRequest, opts ...grpc.CallOption) (*proto.RepoUpdateResponse, error) { - base := proto.NewGitserverServiceClient(cc) - return base.RepoUpdate(ctx, in, opts...) - } - cli := gitserver.NewMockGitserverServiceClient() - cli.ArchiveFunc.SetDefaultHook(mockArchive) - cli.RepoUpdateFunc.SetDefaultHook(mockRepoUpdate) - return cli + return base.RepoUpdate(ctx, in, opts...) } - }) - - return gitserver.NewTestClient(t).WithClientSource(source) - } - - runArchiveReaderTestfunc(t, mkClient, repoName, test) - if !called { - t.Error("archiveReader: GitserverServiceClient should have been called") - } - - } - }) - - t.Run("http", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - - for _, test := range tests { - repoName := api.RepoName(test.name) - called := false - - mkClient := func(t *testing.T, addrs []string) gitserver.Client { - t.Helper() - - source := gitserver.NewTestClientSource(t, addrs, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockArchive := func(ctx context.Context, in *proto.ArchiveRequest, opts ...grpc.CallOption) (proto.GitserverService_ArchiveClient, error) { - called = true - base := proto.NewGitserverServiceClient(cc) - return base.Archive(ctx, in, opts...) - } - cli := gitserver.NewMockGitserverServiceClient() - cli.ArchiveFunc.SetDefaultHook(mockArchive) - return cli - } - }) - - return gitserver.NewTestClient(t).WithClientSource(source) - } - - runArchiveReaderTestfunc(t, mkClient, repoName, test) - if called { - t.Error("archiveReader: GitserverServiceClient should have been called") - } + cli := gitserver.NewMockGitserverServiceClient() + cli.ArchiveFunc.SetDefaultHook(mockArchive) + cli.RepoUpdateFunc.SetDefaultHook(mockRepoUpdate) + return cli + } + }) + return gitserver.NewTestClient(t).WithClientSource(source) } - }) + runArchiveReaderTestfunc(t, mkClient, repoName, test) + if !called { + t.Error("archiveReader: GitserverServiceClient should have been called") + } + + } } func createSimpleGitRepo(t *testing.T, root string) string { diff --git a/cmd/gitserver/internal/integration_tests/object_test.go b/cmd/gitserver/internal/integration_tests/object_test.go index c505e4ce0db..f79d5d41b5b 100644 --- a/cmd/gitserver/internal/integration_tests/object_test.go +++ b/cmd/gitserver/internal/integration_tests/object_test.go @@ -5,10 +5,8 @@ import ( "testing" "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" - "github.com/sourcegraph/sourcegraph/schema" ) func TestGetObject(t *testing.T) { @@ -51,37 +49,9 @@ func TestGetObject(t *testing.T) { }) } - t.Run("gRPC", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(true), - }, - }, - }) - for label, test := range tests { - source := gitserver.NewTestClientSource(t, GitserverAddresses) - cli := gitserver.NewTestClient(t).WithClientSource(source) - runTest(t, label, test, cli) - } - }) - - t.Run("HTTP", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, - }, - }) - for label, test := range tests { - source := gitserver.NewTestClientSource(t, GitserverAddresses) - cli := gitserver.NewTestClient(t).WithClientSource(source) - runTest(t, label, test, cli) - } - }) -} - -func boolPointer(b bool) *bool { - return &b + for label, test := range tests { + source := gitserver.NewTestClientSource(t, GitserverAddresses) + cli := gitserver.NewTestClient(t).WithClientSource(source) + runTest(t, label, test, cli) + } } diff --git a/cmd/gitserver/internal/integration_tests/test_utils.go b/cmd/gitserver/internal/integration_tests/test_utils.go index d26983a31fe..841dcb9f125 100644 --- a/cmd/gitserver/internal/integration_tests/test_utils.go +++ b/cmd/gitserver/internal/integration_tests/test_utils.go @@ -25,7 +25,6 @@ import ( proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/ratelimit" "github.com/sourcegraph/sourcegraph/internal/types" @@ -105,7 +104,7 @@ func InitGitserver() { serverAddress := l.Addr().String() source := gitserver.NewTestClientSource(&t, []string{serverAddress}) - testGitserverClient = gitserver.NewTestClient(&t).WithDoer(httpcli.InternalDoer).WithClientSource(source) + testGitserverClient = gitserver.NewTestClient(&t).WithClientSource(source) GitserverAddresses = []string{serverAddress} } diff --git a/cmd/gitserver/internal/list_gitolite.go b/cmd/gitserver/internal/list_gitolite.go index dbe144992bd..9835b4ef530 100644 --- a/cmd/gitserver/internal/list_gitolite.go +++ b/cmd/gitserver/internal/list_gitolite.go @@ -2,26 +2,12 @@ package internal import ( "context" - "encoding/json" - "net/http" "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" "github.com/sourcegraph/sourcegraph/internal/security" "github.com/sourcegraph/sourcegraph/lib/errors" ) -func (s *Server) handleListGitolite(w http.ResponseWriter, r *http.Request) { - repos, err := defaultGitolite.listRepos(r.Context(), r.URL.Query().Get("gitolite")) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err := json.NewEncoder(w).Encode(repos); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - var defaultGitolite = gitoliteFetcher{client: gitoliteClient{}} type gitoliteFetcher struct { diff --git a/cmd/gitserver/internal/list_gitolite_test.go b/cmd/gitserver/internal/list_gitolite_test.go index 087b775c89d..8497da305ac 100644 --- a/cmd/gitserver/internal/list_gitolite_test.go +++ b/cmd/gitserver/internal/list_gitolite_test.go @@ -2,22 +2,11 @@ package internal import ( "context" - "net/http" - "net/http/httptest" "testing" "github.com/google/go-cmp/cmp" - "github.com/sourcegraph/log/logtest" - "github.com/stretchr/testify/assert" - "golang.org/x/time/rate" - "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/vcssyncer" - "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/database/dbmocks" "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" - "github.com/sourcegraph/sourcegraph/internal/observation" - "github.com/sourcegraph/sourcegraph/internal/ratelimit" - "github.com/sourcegraph/sourcegraph/internal/wrexec" "github.com/sourcegraph/sourcegraph/schema" ) @@ -115,63 +104,6 @@ func Test_Gitolite_listRepos(t *testing.T) { } } -func TestCheckSSRFHeader(t *testing.T) { - db := dbmocks.NewMockDB() - gr := dbmocks.NewMockGitserverRepoStore() - db.GitserverReposFunc.SetDefaultReturn(gr) - s := &Server{ - Logger: logtest.Scoped(t), - ObservationCtx: observation.TestContextTB(t), - ReposDir: "/testroot", - skipCloneForTests: true, - GetRemoteURLFunc: func(ctx context.Context, name api.RepoName) (string, error) { - return "https://" + string(name) + ".git", nil - }, - GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) { - return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory()), nil - }, - DB: db, - Locker: NewRepositoryLocker(), - RPSLimiter: ratelimit.NewInstrumentedLimiter("GitserverTest", rate.NewLimiter(rate.Inf, 10)), - } - h := s.Handler() - - oldFetcher := defaultGitolite - t.Cleanup(func() { - defaultGitolite = oldFetcher - }) - defaultGitolite = gitoliteFetcher{ - client: stubGitoliteClient{ - ListRepos_: func(ctx context.Context, host string) ([]*gitolite.Repo, error) { - return []*gitolite.Repo{}, nil - }, - }, - } - - t.Run("header missing", func(t *testing.T) { - rw := httptest.NewRecorder() - r, err := http.NewRequest("GET", "/list-gitolite?gitolite=127.0.0.1", nil) - if err != nil { - t.Fatal(err) - } - h.ServeHTTP(rw, r) - - assert.Equal(t, 400, rw.Code) - }) - - t.Run("header supplied", func(t *testing.T) { - rw := httptest.NewRecorder() - r, err := http.NewRequest("GET", "/list-gitolite?gitolite=127.0.0.1", nil) - if err != nil { - t.Fatal(err) - } - r.Header.Set("X-Requested-With", "Sourcegraph") - h.ServeHTTP(rw, r) - - assert.Equal(t, 200, rw.Code) - }) -} - type stubGitoliteClient struct { ListRepos_ func(ctx context.Context, host string) ([]*gitolite.Repo, error) } diff --git a/cmd/gitserver/internal/p4exec.go b/cmd/gitserver/internal/p4exec.go index 19d0b6b1454..36408b1b2e1 100644 --- a/cmd/gitserver/internal/p4exec.go +++ b/cmd/gitserver/internal/p4exec.go @@ -3,10 +3,8 @@ package internal import ( "bytes" "context" - "encoding/json" "fmt" "io" - "net/http" "os" "os/exec" "strconv" @@ -116,83 +114,6 @@ func (gs *GRPCServer) doP4Exec(ctx context.Context, logger log.Logger, req *p4Ex return nil } -func (s *Server) handleP4Exec(w http.ResponseWriter, r *http.Request) { - var req p4ExecRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if len(req.Args) < 1 { - http.Error(w, "args must be greater than or equal to 1", http.StatusBadRequest) - return - } - - // Make sure the subcommand is explicitly allowed - allowlist := []string{"protects", "groups", "users", "group", "changes"} - allowed := false - for _, arg := range allowlist { - if req.Args[0] == arg { - allowed = true - break - } - } - if !allowed { - http.Error(w, fmt.Sprintf("subcommand %q is not allowed", req.Args[0]), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Log which actor is accessing p4-exec. - // - // p4-exec is currently only used for fetching user based permissions information - // so, we don't have a repo name. - accesslog.Record(r.Context(), "", - log.String("p4user", req.P4User), - log.String("p4port", req.P4Port), - log.Strings("args", req.Args), - ) - - // Make sure credentials are valid before heavier operation - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - s.p4execHTTP(w, r, &req) -} - -func (s *Server) p4execHTTP(w http.ResponseWriter, r *http.Request, req *p4ExecRequest) { - logger := s.Logger.Scoped("p4exec") - - // Flush writes more aggressively than standard net/http so that clients - // with a context deadline see as much partial response body as possible. - if fw := newFlushingResponseWriter(logger, w); fw != nil { - w = fw - defer fw.Close() - } - - ctx, cancel := context.WithTimeout(r.Context(), time.Minute) - defer cancel() - - w.Header().Set("Trailer", "X-Exec-Error") - w.Header().Add("Trailer", "X-Exec-Exit-Status") - w.Header().Add("Trailer", "X-Exec-Stderr") - w.WriteHeader(http.StatusOK) - - execStatus := s.p4Exec(ctx, logger, req, r.UserAgent(), w) - w.Header().Set("X-Exec-Error", errorString(execStatus.Err)) - w.Header().Set("X-Exec-Exit-Status", strconv.Itoa(execStatus.ExitStatus)) - w.Header().Set("X-Exec-Stderr", execStatus.Stderr) - -} - func (s *Server) p4Exec(ctx context.Context, logger log.Logger, req *p4ExecRequest, userAgent string, w io.Writer) execStatus { start := time.Now() var cmdStart time.Time // set once we have ensured commit diff --git a/cmd/gitserver/internal/p4exec_test.go b/cmd/gitserver/internal/p4exec_test.go index ae28136015d..bc094d49d5a 100644 --- a/cmd/gitserver/internal/p4exec_test.go +++ b/cmd/gitserver/internal/p4exec_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "net/url" "os/exec" - "strings" "testing" "github.com/google/go-cmp/cmp" @@ -77,184 +76,115 @@ func TestServer_handleP4Exec(t *testing.T) { } } - t.Run("gRPC", func(t *testing.T) { - readAll := func(execClient proto.GitserverService_P4ExecClient) ([]byte, error) { - var buf bytes.Buffer - for { - resp, err := execClient.Recv() - if errors.Is(err, io.EOF) { - return buf.Bytes(), nil - } + readAll := func(execClient proto.GitserverService_P4ExecClient) ([]byte, error) { + var buf bytes.Buffer + for { + resp, err := execClient.Recv() + if errors.Is(err, io.EOF) { + return buf.Bytes(), nil + } - if err != nil { - return buf.Bytes(), err - } + if err != nil { + return buf.Bytes(), err + } - _, err = buf.Write(resp.GetData()) - if err != nil { - t.Fatalf("failed to write data: %v", err) - } + _, err = buf.Write(resp.GetData()) + if err != nil { + t.Fatalf("failed to write data: %v", err) } } + } - t.Run("users", func(t *testing.T) { - executil.UpdateRunCommandMock(defaultMockRunCommand) + t.Run("users", func(t *testing.T) { + executil.UpdateRunCommandMock(defaultMockRunCommand) - _, client, closeFunc := startServer(t) - t.Cleanup(closeFunc) + _, client, closeFunc := startServer(t) + t.Cleanup(closeFunc) - stream, err := client.P4Exec(context.Background(), &proto.P4ExecRequest{ - Args: [][]byte{[]byte("users")}, - }) - if err != nil { - t.Fatalf("failed to call P4Exec: %v", err) - } - - data, err := readAll(stream) - s, ok := status.FromError(err) - if !ok { - t.Fatal("received non-status error from p4exec call") - } - - if diff := cmp.Diff("the answer to life the universe and everything", s.Message()); diff != "" { - t.Fatalf("unexpected error in stream (-want +got):\n%s", diff) - } - - expectedData := []byte("admin (admin) accessed 2021/01/31") - - if diff := cmp.Diff(expectedData, data); diff != "" { - t.Fatalf("unexpected data (-want +got):\n%s", diff) - } + stream, err := client.P4Exec(context.Background(), &proto.P4ExecRequest{ + Args: [][]byte{[]byte("users")}, }) + if err != nil { + t.Fatalf("failed to call P4Exec: %v", err) + } - t.Run("empty request", func(t *testing.T) { - executil.UpdateRunCommandMock(defaultMockRunCommand) + data, err := readAll(stream) + s, ok := status.FromError(err) + if !ok { + t.Fatal("received non-status error from p4exec call") + } - _, client, closeFunc := startServer(t) - t.Cleanup(closeFunc) + if diff := cmp.Diff("the answer to life the universe and everything", s.Message()); diff != "" { + t.Fatalf("unexpected error in stream (-want +got):\n%s", diff) + } - stream, err := client.P4Exec(context.Background(), &proto.P4ExecRequest{}) - if err != nil { - t.Fatalf("failed to call P4Exec: %v", err) - } - - _, err = readAll(stream) - if status.Code(err) != codes.InvalidArgument { - t.Fatalf("expected InvalidArgument error, got %v", err) - } - }) - - t.Run("disallowed command", func(t *testing.T) { - - executil.UpdateRunCommandMock(defaultMockRunCommand) - - _, client, closeFunc := startServer(t) - t.Cleanup(closeFunc) - - stream, err := client.P4Exec(context.Background(), &proto.P4ExecRequest{ - Args: [][]byte{[]byte("bad_command")}, - }) - if err != nil { - t.Fatalf("failed to call P4Exec: %v", err) - } - - _, err = readAll(stream) - if status.Code(err) != codes.InvalidArgument { - t.Fatalf("expected InvalidArgument error, got %v", err) - } - }) - - t.Run("context cancelled", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - - executil.UpdateRunCommandMock(func(ctx context.Context, _ *exec.Cmd) (int, error) { - // fake a context cancellation that occurs while the process is running - - cancel() - return 0, ctx.Err() - }) - - _, client, closeFunc := startServer(t) - t.Cleanup(closeFunc) - - stream, err := client.P4Exec(ctx, &proto.P4ExecRequest{ - Args: [][]byte{[]byte("users")}, - }) - if err != nil { - t.Fatalf("failed to call P4Exec: %v", err) - } - - _, err = readAll(stream) - if !(errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled) { - t.Fatalf("expected context cancelation error, got %v", err) - } - }) + expectedData := []byte("admin (admin) accessed 2021/01/31") + if diff := cmp.Diff(expectedData, data); diff != "" { + t.Fatalf("unexpected data (-want +got):\n%s", diff) + } }) - t.Run("HTTP", func(t *testing.T) { + t.Run("empty request", func(t *testing.T) { + executil.UpdateRunCommandMock(defaultMockRunCommand) - tests := []Test{ - { - Name: "Command", - Request: newRequest("POST", "/p4-exec", strings.NewReader(`{"args": ["users"]}`)), - ExpectedCode: http.StatusOK, - ExpectedBody: "admin (admin) accessed 2021/01/31", - ExpectedTrailers: http.Header{ - "X-Exec-Error": {"the answer to life the universe and everything"}, - "X-Exec-Exit-Status": {"42"}, - "X-Exec-Stderr": {"teststderr"}, - }, - }, - { - Name: "Error", - Request: newRequest("POST", "/p4-exec", strings.NewReader(`{"args": ["bad_command"]}`)), - ExpectedCode: http.StatusBadRequest, - ExpectedBody: "subcommand \"bad_command\" is not allowed", - }, - { - Name: "EmptyBody", - Request: newRequest("POST", "/p4-exec", nil), - ExpectedCode: http.StatusBadRequest, - ExpectedBody: `EOF`, - }, - { - Name: "EmptyInput", - Request: newRequest("POST", "/p4-exec", strings.NewReader("{}")), - ExpectedCode: http.StatusBadRequest, - ExpectedBody: `args must be greater than or equal to 1`, - }, + _, client, closeFunc := startServer(t) + t.Cleanup(closeFunc) + + stream, err := client.P4Exec(context.Background(), &proto.P4ExecRequest{}) + if err != nil { + t.Fatalf("failed to call P4Exec: %v", err) } - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - executil.UpdateRunCommandMock(defaultMockRunCommand) + _, err = readAll(stream) + if status.Code(err) != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument error, got %v", err) + } + }) - handler, _, closeFunc := startServer(t) - t.Cleanup(closeFunc) + t.Run("disallowed command", func(t *testing.T) { - w := httptest.ResponseRecorder{Body: new(bytes.Buffer)} - handler.ServeHTTP(&w, test.Request) + executil.UpdateRunCommandMock(defaultMockRunCommand) - res := w.Result() - if res.StatusCode != test.ExpectedCode { - t.Errorf("wrong status: expected %d, got %d", test.ExpectedCode, w.Code) - } + _, client, closeFunc := startServer(t) + t.Cleanup(closeFunc) - body, err := io.ReadAll(res.Body) - if err != nil { - t.Fatal(err) - } - if strings.TrimSpace(string(body)) != test.ExpectedBody { - t.Errorf("wrong body: expected %q, got %q", test.ExpectedBody, string(body)) - } + stream, err := client.P4Exec(context.Background(), &proto.P4ExecRequest{ + Args: [][]byte{[]byte("bad_command")}, + }) + if err != nil { + t.Fatalf("failed to call P4Exec: %v", err) + } - for k, v := range test.ExpectedTrailers { - if got := res.Trailer.Get(k); got != v[0] { - t.Errorf("wrong trailer %q: expected %q, got %q", k, v[0], got) - } - } - }) + _, err = readAll(stream) + if status.Code(err) != codes.InvalidArgument { + t.Fatalf("expected InvalidArgument error, got %v", err) + } + }) + + t.Run("context cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + executil.UpdateRunCommandMock(func(ctx context.Context, _ *exec.Cmd) (int, error) { + // fake a context cancellation that occurs while the process is running + + cancel() + return 0, ctx.Err() + }) + + _, client, closeFunc := startServer(t) + t.Cleanup(closeFunc) + + stream, err := client.P4Exec(ctx, &proto.P4ExecRequest{ + Args: [][]byte{[]byte("users")}, + }) + if err != nil { + t.Fatalf("failed to call P4Exec: %v", err) + } + + _, err = readAll(stream) + if !(errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled) { + t.Fatalf("expected context cancelation error, got %v", err) } }) } diff --git a/cmd/gitserver/internal/patch.go b/cmd/gitserver/internal/patch.go index c310bd08eee..8c4c7946ec9 100644 --- a/cmd/gitserver/internal/patch.go +++ b/cmd/gitserver/internal/patch.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "encoding/json" "fmt" "io/fs" "net/http" @@ -35,26 +34,6 @@ import ( var patchID uint64 -func (s *Server) handleCreateCommitFromPatchBinary(w http.ResponseWriter, r *http.Request) { - var req protocol.CreateCommitFromPatchRequest - var resp protocol.CreateCommitFromPatchResponse - var status int - - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - resp := new(protocol.CreateCommitFromPatchResponse) - resp.SetError("", "", "", errors.Wrap(err, "decoding CreateCommitFromPatchRequest")) - status = http.StatusBadRequest - } else { - status, resp = s.createCommitFromPatch(r.Context(), req) - } - - w.WriteHeader(status) - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - func (s *Server) createCommitFromPatch(ctx context.Context, req protocol.CreateCommitFromPatchRequest) (int, protocol.CreateCommitFromPatchResponse) { logger := s.Logger.Scoped("createCommitFromPatch"). With( diff --git a/cmd/gitserver/internal/repo_info.go b/cmd/gitserver/internal/repo_info.go index 7bc595870dd..738d54aab36 100644 --- a/cmd/gitserver/internal/repo_info.go +++ b/cmd/gitserver/internal/repo_info.go @@ -2,8 +2,6 @@ package internal import ( "context" - "encoding/json" - "net/http" "github.com/sourcegraph/log" @@ -28,42 +26,6 @@ func repoCloneProgress(reposDir string, locker RepositoryLocker, repo api.RepoNa return &resp } -func (s *Server) handleRepoCloneProgress(w http.ResponseWriter, r *http.Request) { - var req protocol.RepoCloneProgressRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - resp := protocol.RepoCloneProgressResponse{ - Results: make(map[api.RepoName]*protocol.RepoCloneProgress, len(req.Repos)), - } - for _, repoName := range req.Repos { - result := repoCloneProgress(s.ReposDir, s.Locker, repoName) - resp.Results[repoName] = result - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handleRepoDelete(w http.ResponseWriter, r *http.Request) { - var req protocol.RepoDeleteRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := deleteRepo(r.Context(), s.Logger, s.DB, s.Hostname, s.ReposDir, req.Repo); err != nil { - s.Logger.Error("failed to delete repository", log.String("repo", string(req.Repo)), log.Error(err)) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - s.Logger.Info("deleted repository", log.String("repo", string(req.Repo))) -} - func deleteRepo( ctx context.Context, logger log.Logger, diff --git a/cmd/gitserver/internal/repo_info_test.go b/cmd/gitserver/internal/repo_info_test.go new file mode 100644 index 00000000000..86e84e45845 --- /dev/null +++ b/cmd/gitserver/internal/repo_info_test.go @@ -0,0 +1,124 @@ +package internal + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/sourcegraph/log/logtest" + "github.com/stretchr/testify/require" + + "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/gitserverfs" + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/database/dbtest" + "github.com/sourcegraph/sourcegraph/internal/gitserver/protocol" + "github.com/sourcegraph/sourcegraph/internal/types" +) + +func TestDeleteRepo(t *testing.T) { + testDeleteRepo(t, false) +} + +func TestDeleteRepoWhenDeleteInDB(t *testing.T) { + // We also want to ensure that we can delete repo data on disk for a repo that + // has already been deleted in the DB. + testDeleteRepo(t, true) +} + +func testDeleteRepo(t *testing.T, deletedInDB bool) { + logger := logtest.Scoped(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + remote := t.TempDir() + repoName := api.RepoName("example.com/foo/bar") + db := database.NewDB(logger, dbtest.NewDB(t)) + + dbRepo := &types.Repo{ + Name: repoName, + Description: "Test", + } + + // Insert the repo into our database + if err := db.Repos().Create(ctx, dbRepo); err != nil { + t.Fatal(err) + } + + repo := remote + cmd := func(name string, arg ...string) string { + t.Helper() + return runCmd(t, repo, name, arg...) + } + _ = makeSingleCommitRepo(cmd) + // Add a bad tag + cmd("git", "tag", "HEAD") + + reposDir := t.TempDir() + + s := makeTestServer(ctx, t, reposDir, remote, db) + + // We need some of the side effects here + _ = s.Handler() + + // This will perform an initial clone + s.repoUpdate(&protocol.RepoUpdateRequest{ + Repo: repoName, + }) + + size := gitserverfs.DirSize(gitserverfs.RepoDirFromName(s.ReposDir, repoName).Path(".")) + want := &types.GitserverRepo{ + RepoID: dbRepo.ID, + ShardID: "", + CloneStatus: types.CloneStatusCloned, + RepoSizeBytes: size, + } + fromDB, err := db.GitserverRepos().GetByID(ctx, dbRepo.ID) + if err != nil { + t.Fatal(err) + } + + // We don't expect an error + if diff := cmp.Diff(want, fromDB, ignoreVolatileGitserverRepoFields); diff != "" { + t.Fatal(diff) + } + + if deletedInDB { + if err := db.Repos().Delete(ctx, dbRepo.ID); err != nil { + t.Fatal(err) + } + repos, err := db.Repos().List(ctx, database.ReposListOptions{IncludeDeleted: true, IDs: []api.RepoID{dbRepo.ID}}) + if err != nil { + t.Fatal(err) + } + if len(repos) != 1 { + t.Fatalf("Expected 1 repo, got %d", len(repos)) + } + dbRepo = repos[0] + } + + // Now we can delete it + require.NoError(t, deleteRepo(ctx, logger, db, "", reposDir, dbRepo.Name)) + + size = gitserverfs.DirSize(gitserverfs.RepoDirFromName(s.ReposDir, repoName).Path(".")) + if size != 0 { + t.Fatalf("Size should be 0, got %d", size) + } + + // Check status in gitserver_repos + want = &types.GitserverRepo{ + RepoID: dbRepo.ID, + ShardID: "", + CloneStatus: types.CloneStatusNotCloned, + RepoSizeBytes: size, + } + fromDB, err = db.GitserverRepos().GetByID(ctx, dbRepo.ID) + if err != nil { + t.Fatal(err) + } + + // We don't expect an error + if diff := cmp.Diff(want, fromDB, ignoreVolatileGitserverRepoFields); diff != "" { + t.Fatal(diff) + } +} diff --git a/cmd/gitserver/internal/server.go b/cmd/gitserver/internal/server.go index 2fe62340fc5..55f25741a62 100644 --- a/cmd/gitserver/internal/server.go +++ b/cmd/gitserver/internal/server.go @@ -6,8 +6,6 @@ import ( "bytes" "container/list" "context" - "encoding/gob" - "encoding/json" "fmt" "io" "net/http" @@ -18,7 +16,6 @@ import ( "strings" "sync" "sync/atomic" - "syscall" "time" "github.com/prometheus/client_golang/prometheus" @@ -28,7 +25,6 @@ import ( "golang.org/x/sync/semaphore" "golang.org/x/time/rate" - "github.com/sourcegraph/conc" "github.com/sourcegraph/log" "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/accesslog" @@ -55,7 +51,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/limiter" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/ratelimit" - streamhttp "github.com/sourcegraph/sourcegraph/internal/search/streaming/http" "github.com/sourcegraph/sourcegraph/internal/trace" "github.com/sourcegraph/sourcegraph/internal/types" "github.com/sourcegraph/sourcegraph/internal/vcs" @@ -225,35 +220,6 @@ func shortGitCommandSlow(args []string) time.Duration { } } -// 🚨 SECURITY: headerXRequestedWithMiddleware will ensure that the X-Requested-With -// header contains the correct value. See "What does X-Requested-With do, anyway?" in -// https://github.com/sourcegraph/sourcegraph/pull/27931. -func headerXRequestedWithMiddleware(next http.Handler) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - l := log.Scoped("gitserver") - - // Do not apply the middleware to /ping and /git endpoints. - // - // 1. /ping is used by health check services who most likely don't set this header - // at all. - // - // 2. /git may be used to run "git fetch" from another gitserver instance over - // HTTP and the fetchCommand does not set this header yet. - if strings.HasPrefix(r.URL.Path, "/ping") || strings.HasPrefix(r.URL.Path, "/git") { - next.ServeHTTP(w, r) - return - } - - if value := r.Header.Get("X-Requested-With"); value != "Sourcegraph" { - l.Error("header X-Requested-With is not set or is invalid", log.String("path", r.URL.Path)) - http.Error(w, "header X-Requested-With is not set or is invalid", http.StatusBadRequest) - return - } - - next.ServeHTTP(w, r) - } -} - // Handler returns the http.Handler that should be used to serve requests. func (s *Server) Handler() http.Handler { s.ctx, s.cancel = context.WithCancel(context.Background()) @@ -279,40 +245,7 @@ func (s *Server) Handler() http.Handler { }) mux := http.NewServeMux() - mux.HandleFunc("/archive", trace.WithRouteName("archive", accesslog.HTTPMiddleware( - s.Logger.Scoped("archive.accesslog"), - conf.DefaultClient(), - s.handleArchive, - ))) - mux.HandleFunc("/exec", trace.WithRouteName("exec", accesslog.HTTPMiddleware( - s.Logger.Scoped("exec.accesslog"), - conf.DefaultClient(), - s.handleExec, - ))) - mux.HandleFunc("/search", trace.WithRouteName("search", s.handleSearch)) - mux.HandleFunc("/batch-log", trace.WithRouteName("batch-log", s.handleBatchLog)) - mux.HandleFunc("/p4-exec", trace.WithRouteName("p4-exec", accesslog.HTTPMiddleware( - s.Logger.Scoped("p4-exec.accesslog"), - conf.DefaultClient(), - s.handleP4Exec, - ))) - mux.HandleFunc("/list-gitolite", trace.WithRouteName("list-gitolite", s.handleListGitolite)) - mux.HandleFunc("/is-repo-cloneable", trace.WithRouteName("is-repo-cloneable", s.handleIsRepoCloneable)) - mux.HandleFunc("/repo-clone-progress", trace.WithRouteName("repo-clone-progress", s.handleRepoCloneProgress)) - mux.HandleFunc("/delete", trace.WithRouteName("delete", s.handleRepoDelete)) - mux.HandleFunc("/repo-update", trace.WithRouteName("repo-update", s.handleRepoUpdate)) - mux.HandleFunc("/repo-clone", trace.WithRouteName("repo-clone", s.handleRepoClone)) - mux.HandleFunc("/create-commit-from-patch-binary", trace.WithRouteName("create-commit-from-patch-binary", s.handleCreateCommitFromPatchBinary)) - mux.HandleFunc("/disk-info", trace.WithRouteName("disk-info", s.handleDiskInfo)) - mux.HandleFunc("/is-perforce-path-cloneable", trace.WithRouteName("is-perforce-path-cloneable", s.handleIsPerforcePathCloneable)) - mux.HandleFunc("/check-perforce-credentials", trace.WithRouteName("check-perforce-credentials", s.handleCheckPerforceCredentials)) - mux.HandleFunc("/commands/get-object", trace.WithRouteName("commands/get-object", s.handleGetObject)) - mux.HandleFunc("/perforce-users", trace.WithRouteName("perforce-users", s.handlePerforceUsers)) - mux.HandleFunc("/perforce-protects-for-user", trace.WithRouteName("perforce-protects-for-user", s.handlePerforceProtectsForUser)) - mux.HandleFunc("/perforce-protects-for-depot", trace.WithRouteName("perforce-protects-for-depot", s.handlePerforceProtectsForDepot)) - mux.HandleFunc("/perforce-group-members", trace.WithRouteName("perforce-group-members", s.handlePerforceGroupMembers)) - mux.HandleFunc("/is-perforce-super-user", trace.WithRouteName("is-perforce-super-user", s.handleIsPerforceSuperUser)) - mux.HandleFunc("/perforce-get-changelist", trace.WithRouteName("perforce-get-changelist", s.handlePerforceGetChangelist)) + mux.HandleFunc("/ping", trace.WithRouteName("ping", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) @@ -332,8 +265,7 @@ func (s *Server) Handler() http.Handler { }, ))) - // 🚨 SECURITY: This must be wrapped in headerXRequestedWithMiddleware. - return headerXRequestedWithMiddleware(mux) + return mux } func addrForRepo(ctx context.Context, repoName api.RepoName, gitServerAddrs gitserver.GitserverAddresses) string { @@ -511,29 +443,6 @@ func (s *Server) acquireCloneableLimiter(ctx context.Context) (context.Context, return s.cloneableLimiter.Acquire(ctx) } -func (s *Server) handleIsRepoCloneable(w http.ResponseWriter, r *http.Request) { - var req protocol.IsRepoCloneableRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if req.Repo == "" { - http.Error(w, "no Repo given", http.StatusBadRequest) - return - } - resp, err := s.isRepoCloneable(r.Context(), req.Repo) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - func (s *Server) isRepoCloneable(ctx context.Context, repo api.RepoName) (protocol.IsRepoCloneableResponse, error) { // We use an internal actor here as the repo may be private. It is safe since all // we return is a bool indicating whether the repo is cloneable or not. Perhaps @@ -561,25 +470,6 @@ func (s *Server) isRepoCloneable(ctx context.Context, repo api.RepoName) (protoc return resp, nil } -// handleRepoUpdate is a synchronous (waits for update to complete or -// time out) method so it can yield errors. Updates are not -// unconditional; we debounce them based on the provided -// interval, to avoid spam. -func (s *Server) handleRepoUpdate(w http.ResponseWriter, r *http.Request) { - var req protocol.RepoUpdateRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - resp := s.repoUpdate(&req) - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - func (s *Server) repoUpdate(req *protocol.RepoUpdateRequest) protocol.RepoUpdateResponse { logger := s.Logger.Scoped("handleRepoUpdate") var resp protocol.RepoUpdateResponse @@ -640,169 +530,6 @@ func (s *Server) repoUpdate(req *protocol.RepoUpdateRequest) protocol.RepoUpdate return resp } -// handleRepoClone is an asynchronous (does not wait for update to complete or -// time out) call to clone a repository. -// Asynchronous errors will have to be checked in the gitserver_repos table under last_error. -func (s *Server) handleRepoClone(w http.ResponseWriter, r *http.Request) { - logger := s.Logger.Scoped("handleRepoClone") - var req protocol.RepoCloneRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - var resp protocol.RepoCloneResponse - req.Repo = protocol.NormalizeRepo(req.Repo) - - _, err := s.CloneRepo(context.Background(), req.Repo, CloneOptions{Block: false}) - if err != nil { - logger.Warn("error cloning repo", log.String("repo", string(req.Repo)), log.Error(err)) - resp.Error = err.Error() - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handleArchive(w http.ResponseWriter, r *http.Request) { - var ( - logger = s.Logger.Scoped("handleArchive") - q = r.URL.Query() - treeish = q.Get("treeish") - repo = q.Get("repo") - format = q.Get("format") - pathspecs = q["path"] - ) - - // Log which which actor is accessing the repo. - accesslog.Record(r.Context(), repo, - log.String("treeish", treeish), - log.String("format", format), - log.Strings("path", pathspecs), - ) - - if err := git.CheckSpecArgSafety(treeish); err != nil { - w.WriteHeader(http.StatusBadRequest) - s.Logger.Error("gitserver.archive.CheckSpecArgSafety", log.Error(err)) - return - } - - if repo == "" || format == "" { - w.WriteHeader(http.StatusBadRequest) - logger.Error("gitserver.archive", log.String("error", "empty repo or format")) - return - } - - req := &protocol.ExecRequest{ - Repo: api.RepoName(repo), - Args: []string{ - "archive", - - // Suppresses fatal error when the repo contains paths matching **/.git/** and instead - // includes those files (to allow archiving invalid such repos). This is unexpected - // behavior; the --worktree-attributes flag should merely let us specify a gitattributes - // file that contains `**/.git/** export-ignore`, but it actually makes everything work as - // desired. Tested by the "repo with .git dir" test case. - "--worktree-attributes", - - "--format=" + format, - }, - } - - if format == string(gitserver.ArchiveFormatZip) { - // Compression level of 0 (no compression) seems to perform the - // best overall on fast network links, but this has not been tuned - // thoroughly. - req.Args = append(req.Args, "-0") - } - - req.Args = append(req.Args, treeish, "--") - req.Args = append(req.Args, pathspecs...) - - s.execHTTP(w, r, req) -} - -func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { - logger := s.Logger.Scoped("handleSearch") - tr, ctx := trace.New(r.Context(), "handleSearch") - defer tr.End() - - // Decode the request - protocol.RegisterGob() - var args protocol.SearchRequest - if err := gob.NewDecoder(r.Body).Decode(&args); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - eventWriter, err := streamhttp.NewWriter(w) - if err != nil { - tr.SetError(err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - var matchesBufMux sync.Mutex - matchesBuf := streamhttp.NewJSONArrayBuf(8*1024, func(data []byte) error { - tr.AddEvent("flushing data", attribute.Int("data.len", len(data))) - return eventWriter.EventBytes("matches", data) - }) - - // Start a goroutine that periodically flushes the buffer - var flusherWg conc.WaitGroup - flusherCtx, flusherCancel := context.WithCancel(context.Background()) - defer flusherCancel() - flusherWg.Go(func() { - flushTicker := time.NewTicker(50 * time.Millisecond) - defer flushTicker.Stop() - - for { - select { - case <-flushTicker.C: - matchesBufMux.Lock() - matchesBuf.Flush() - matchesBufMux.Unlock() - case <-flusherCtx.Done(): - return - } - } - }) - - // Create a callback that appends the match to the buffer - var haveFlushed atomic.Bool - onMatch := func(match *protocol.CommitMatch) error { - matchesBufMux.Lock() - defer matchesBufMux.Unlock() - - err := matchesBuf.Append(match) - if err != nil { - return err - } - - // If we haven't sent any results yet, flush immediately - if !haveFlushed.Load() { - haveFlushed.Store(true) - return matchesBuf.Flush() - } - - return nil - } - - // Run the search - limitHit, searchErr := s.searchWithObservability(ctx, tr, &args, onMatch) - if writeErr := eventWriter.Event("done", protocol.NewSearchEventDone(limitHit, searchErr)); writeErr != nil { - if !errors.Is(writeErr, syscall.EPIPE) { - logger.Error("failed to send done event", log.Error(writeErr)) - } - } - - // Clean up the flusher goroutine, then do one final flush - flusherCancel() - flusherWg.Wait() - matchesBuf.Flush() -} - func (s *Server) performGitLogCommand(ctx context.Context, repoCommit api.RepoCommit, format string) (output string, isRepoCloned bool, err error) { ctx, _, endObservation := s.operations.batchLogSingle.With(ctx, &err, observation.Args{ Attrs: append( @@ -903,44 +630,6 @@ func (s *Server) batchGitLogInstrumentedHandler(ctx context.Context, req protoco return protocol.BatchLogResponse{Results: results}, nil } -func (s *Server) handleBatchLog(w http.ResponseWriter, r *http.Request) { - // 🚨 SECURITY: Only allow POST requests. - if strings.ToUpper(r.Method) != http.MethodPost { - http.Error(w, "", http.StatusMethodNotAllowed) - return - } - - s.operations = s.ensureOperations() - - // Read request body - var req protocol.BatchLogRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Validate request parameters - if len(req.RepoCommits) == 0 { - // Early exit: implicitly writes 200 OK - _ = json.NewEncoder(w).Encode(protocol.BatchLogResponse{Results: []protocol.BatchLogResult{}}) - return - } - if !strings.HasPrefix(req.Format, "--format=") { - http.Error(w, "format parameter expected to be of the form `--format=`", http.StatusUnprocessableEntity) - return - } - - // Handle unexpected error conditions - resp, err := s.batchGitLogInstrumentedHandler(r.Context(), req) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Write payload to client: implicitly writes 200 OK - _ = json.NewEncoder(w).Encode(resp) -} - // ensureOperations returns the non-nil operations value supplied to this server // via RegisterMetrics (when constructed as part of the gitserver binary), or // constructs and memoizes a no-op operations value (for use in tests). @@ -952,35 +641,6 @@ func (s *Server) ensureOperations() *operations { return s.operations } -func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { - // 🚨 SECURITY: Only allow POST requests. - // See https://github.com/sourcegraph/security-issues/issues/213. - if strings.ToUpper(r.Method) != http.MethodPost { - http.Error(w, "", http.StatusMethodNotAllowed) - return - } - - var req protocol.ExecRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Log which actor is accessing the repo. - args := req.Args - cmd := "" - if len(req.Args) > 0 { - cmd = req.Args[0] - args = args[1:] - } - accesslog.Record(r.Context(), string(req.Repo), - log.String("cmd", cmd), - log.Strings("args", args), - ) - - s.execHTTP(w, r, &req) -} - var blockedCommandExecutedCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "src_gitserver_exec_blocked_command_received", Help: "Incremented each time a command not in the allowlist for gitserver is executed", @@ -1000,6 +660,7 @@ type execStatus struct { Err error } +// TODO: eseliger // exec runs a git command. After the first write to w, it must not return an error. // TODO(@camdencheek): once gRPC is the only consumer of this, do everything with errors // because gRPC can handle trailing errors on a stream. @@ -1182,48 +843,6 @@ func (s *Server) exec(ctx context.Context, logger log.Logger, req *protocol.Exec }, nil } -// execHTTP translates the results of an exec into the expected HTTP statuses and payloads -func (s *Server) execHTTP(w http.ResponseWriter, r *http.Request, req *protocol.ExecRequest) { - logger := s.Logger.Scoped("exec").With(log.Strings("req.Args", req.Args)) - - // Flush writes more aggressively than standard net/http so that clients - // with a context deadline see as much partial response body as possible. - if fw := newFlushingResponseWriter(logger, w); fw != nil { - w = fw - defer fw.Close() - } - - ctx := r.Context() - - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Cache-Control", "no-cache") - - w.Header().Set("Trailer", "X-Exec-Error") - w.Header().Add("Trailer", "X-Exec-Exit-Status") - w.Header().Add("Trailer", "X-Exec-Stderr") - - execStatus, err := s.exec(ctx, logger, req, r.UserAgent(), w) - w.Header().Set("X-Exec-Error", errorString(execStatus.Err)) - w.Header().Set("X-Exec-Exit-Status", strconv.Itoa(execStatus.ExitStatus)) - w.Header().Set("X-Exec-Stderr", execStatus.Stderr) - if err != nil { - if v := (&NotFoundError{}); errors.As(err, &v) { - w.WriteHeader(http.StatusNotFound) - _ = json.NewEncoder(w).Encode(v.Payload) - - } else if errors.Is(err, ErrInvalidCommand) { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte("invalid command")) - - } else { - // If it's not a well-known error, send the error text - // and a generic error code. - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) - } - } -} - func setLastFetched(ctx context.Context, db database.DB, shardID string, dir common.GitDir, name api.RepoName) error { lastFetched, err := repoLastFetched(dir) if err != nil { @@ -2027,387 +1646,3 @@ func setLastChanged(logger log.Logger, dir common.GitDir) error { return nil } - -// errorString returns the error string. If err is nil it returns the empty -// string. -func errorString(err error) string { - if err == nil { - return "" - } - return err.Error() -} - -func (s *Server) handleIsPerforcePathCloneable(w http.ResponseWriter, r *http.Request) { - var req protocol.IsPerforcePathCloneableRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if req.DepotPath == "" { - http.Error(w, "no DepotPath given", http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.IsDepotPathCloneable(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd, req.DepotPath) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := json.NewEncoder(w).Encode(protocol.IsPerforcePathCloneableResponse{}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handleCheckPerforceCredentials(w http.ResponseWriter, r *http.Request) { - var req protocol.CheckPerforceCredentialsRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := json.NewEncoder(w).Encode(protocol.CheckPerforceCredentialsResponse{}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handleGetObject(w http.ResponseWriter, r *http.Request) { - var req protocol.GetObjectRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, errors.Wrap(err, "decoding body").Error(), http.StatusBadRequest) - return - } - - // Log which actor is accessing the repo. - accesslog.Record(r.Context(), string(req.Repo), log.String("objectname", req.ObjectName)) - - obj, err := git.GetObject(r.Context(), s.RecordingCommandFactory, s.ReposDir, req.Repo, req.ObjectName) - if err != nil { - http.Error(w, errors.Wrap(err, "getting object").Error(), http.StatusInternalServerError) - return - } - - resp := protocol.GetObjectResponse{ - Object: *obj, - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handlePerforceUsers(w http.ResponseWriter, r *http.Request) { - var req protocol.PerforceUsersRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - accesslog.Record( - r.Context(), - "", - log.String("p4user", req.P4User), - log.String("p4port", req.P4Port), - ) - - users, err := perforce.P4Users(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := &protocol.PerforceUsersResponse{ - Users: make([]protocol.PerforceUser, 0, len(users)), - } - - for _, user := range users { - resp.Users = append(resp.Users, protocol.PerforceUser{ - Username: user.Username, - Email: user.Email, - }) - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handlePerforceProtectsForUser(w http.ResponseWriter, r *http.Request) { - var req protocol.PerforceProtectsForUserRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - accesslog.Record( - r.Context(), - "", - log.String("p4user", req.P4User), - log.String("p4port", req.P4Port), - ) - - protects, err := perforce.P4ProtectsForUser(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd, req.Username) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - jsonProtects := make([]protocol.PerforceProtect, len(protects)) - for i, p := range protects { - jsonProtects[i] = protocol.PerforceProtect{ - Level: p.Level, - EntityType: p.EntityType, - EntityName: p.EntityName, - Match: p.Match, - IsExclusion: p.IsExclusion, - Host: p.Host, - } - } - - resp := &protocol.PerforceProtectsForUserResponse{ - Protects: jsonProtects, - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handlePerforceProtectsForDepot(w http.ResponseWriter, r *http.Request) { - var req protocol.PerforceProtectsForDepotRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - accesslog.Record( - r.Context(), - "", - log.String("p4user", req.P4User), - log.String("p4port", req.P4Port), - ) - - protects, err := perforce.P4ProtectsForDepot(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd, req.Depot) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - jsonProtects := make([]protocol.PerforceProtect, len(protects)) - for i, p := range protects { - jsonProtects[i] = protocol.PerforceProtect{ - Level: p.Level, - EntityType: p.EntityType, - EntityName: p.EntityName, - Match: p.Match, - IsExclusion: p.IsExclusion, - Host: p.Host, - } - } - - resp := &protocol.PerforceProtectsForDepotResponse{ - Protects: jsonProtects, - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - -} - -func (s *Server) handlePerforceGroupMembers(w http.ResponseWriter, r *http.Request) { - var req protocol.PerforceGroupMembersRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - accesslog.Record( - r.Context(), - "", - log.String("p4user", req.P4User), - log.String("p4port", req.P4Port), - ) - - members, err := perforce.P4GroupMembers(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd, req.Group) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := &protocol.PerforceGroupMembersResponse{ - Usernames: members, - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handleIsPerforceSuperUser(w http.ResponseWriter, r *http.Request) { - var req protocol.IsPerforceSuperUserRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - accesslog.Record( - r.Context(), - "", - log.String("p4user", req.P4User), - log.String("p4port", req.P4Port), - ) - - err = perforce.P4UserIsSuperUser(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - if err == perforce.ErrIsNotSuperUser { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := &protocol.IsPerforceSuperUserResponse{} - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} - -func (s *Server) handlePerforceGetChangelist(w http.ResponseWriter, r *http.Request) { - var req protocol.PerforceGetChangelistRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - p4home, err := gitserverfs.MakeP4HomeDir(s.ReposDir) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = perforce.P4TestWithTrust(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - accesslog.Record( - r.Context(), - "", - log.String("p4user", req.P4User), - log.String("p4port", req.P4Port), - ) - - changelist, err := perforce.GetChangelistByID(r.Context(), p4home, req.P4Port, req.P4User, req.P4Passwd, req.ChangelistID) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := &protocol.PerforceGetChangelistResponse{ - Changelist: protocol.PerforceChangelist{ - ID: changelist.ID, - CreationDate: changelist.CreationDate, - State: string(changelist.State), - Author: changelist.Author, - Title: changelist.Title, - Message: changelist.Message, - }, - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } -} diff --git a/cmd/gitserver/internal/server_grpc.go b/cmd/gitserver/internal/server_grpc.go index bb71d84f9e0..4456e4ed063 100644 --- a/cmd/gitserver/internal/server_grpc.go +++ b/cmd/gitserver/internal/server_grpc.go @@ -214,11 +214,15 @@ func (gs *GRPCServer) doExec(ctx context.Context, logger log.Logger, req *protoc } gRPCStatus := codes.Unknown - if strings.Contains(execStatus.Err.Error(), "signal: killed") { + if execStatus.Err != nil && strings.Contains(execStatus.Err.Error(), "signal: killed") { gRPCStatus = codes.Aborted } - s, err := status.New(gRPCStatus, execStatus.Err.Error()).WithDetails(&proto.ExecStatusPayload{ + var errString string + if execStatus.Err != nil { + errString = execStatus.Err.Error() + } + s, err := status.New(gRPCStatus, errString).WithDetails(&proto.ExecStatusPayload{ StatusCode: int32(execStatus.ExitStatus), Stderr: execStatus.Stderr, }) @@ -228,6 +232,7 @@ func (gs *GRPCServer) doExec(ctx context.Context, logger log.Logger, req *protoc } return s.Err() } + return nil } diff --git a/cmd/gitserver/internal/server_test.go b/cmd/gitserver/internal/server_test.go index 479945b229b..d2934bed242 100644 --- a/cmd/gitserver/internal/server_test.go +++ b/cmd/gitserver/internal/server_test.go @@ -1,14 +1,10 @@ package internal import ( - "bytes" "container/list" "context" - "encoding/json" "fmt" "io" - "net/http" - "net/http/httptest" "os" "os/exec" "path/filepath" @@ -22,6 +18,8 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/sync/semaphore" "golang.org/x/time/rate" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/sourcegraph/log/logtest" @@ -31,11 +29,13 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/perforce" "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/vcssyncer" "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/database/dbmocks" "github.com/sourcegraph/sourcegraph/internal/database/dbtest" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/protocol" + v1 "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" "github.com/sourcegraph/sourcegraph/internal/limiter" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/ratelimit" @@ -46,124 +46,101 @@ import ( ) type Test struct { - Name string - Request *http.Request - ExpectedCode int - ExpectedBody string - ExpectedTrailers http.Header -} - -func newRequest(method, path string, body io.Reader) *http.Request { - r := httptest.NewRequest(method, path, body) - r.Header.Add("X-Requested-With", "Sourcegraph") - return r + Name string + Request *v1.ExecRequest + ExpectedCode codes.Code + ExpectedBody string + ExpectedDetails []any } func TestExecRequest(t *testing.T) { + conf.Mock(&conf.Unified{}) + t.Cleanup(func() { conf.Mock(nil) }) + tests := []Test{ { - Name: "HTTP GET", - Request: newRequest("GET", "/exec", strings.NewReader("{}")), - ExpectedCode: http.StatusMethodNotAllowed, - ExpectedBody: "", - }, - { - Name: "Command", - Request: newRequest("POST", "/exec", strings.NewReader(`{"repo": "github.com/gorilla/mux", "args": ["testcommand"]}`)), - ExpectedCode: http.StatusOK, - ExpectedBody: "teststdout", - ExpectedTrailers: http.Header{ - "X-Exec-Error": {""}, - "X-Exec-Exit-Status": {"42"}, - "X-Exec-Stderr": {"teststderr"}, - }, - }, - { - Name: "CommandWithURL", - Request: newRequest("POST", "/exec", strings.NewReader(`{"repo": "my-mux", "url": "https://github.com/gorilla/mux.git", "args": ["testcommand"]}`)), - ExpectedCode: http.StatusOK, - ExpectedBody: "teststdout", - ExpectedTrailers: http.Header{ - "X-Exec-Error": {""}, - "X-Exec-Exit-Status": {"42"}, - "X-Exec-Stderr": {"teststderr"}, + Name: "Command", + Request: &v1.ExecRequest{ + Repo: "github.com/gorilla/mux", + Args: [][]byte{[]byte("testcommand")}, }, + ExpectedCode: codes.Unknown, + ExpectedDetails: []any{&v1.ExecStatusPayload{ + StatusCode: 42, + Stderr: "teststderr", + }}, }, { Name: "echo", - Request: newRequest( - "POST", "/exec", strings.NewReader( - `{"repo": "github.com/gorilla/mux", "args": ["testecho", "hi"]}`, - ), - ), - ExpectedCode: http.StatusOK, - ExpectedBody: "hi", - ExpectedTrailers: http.Header{ - "X-Exec-Error": {""}, - "X-Exec-Exit-Status": {"0"}, - "X-Exec-Stderr": {""}, + Request: &v1.ExecRequest{ + Repo: "github.com/gorilla/mux", + Args: [][]byte{[]byte("testecho"), []byte("hi")}, }, + ExpectedCode: codes.OK, + ExpectedBody: "hi", }, { Name: "stdin", - Request: newRequest( - "POST", "/exec", strings.NewReader( - `{"repo": "github.com/gorilla/mux", "args": ["testcat"], "stdin": "aGk="}`, - ), - ), - ExpectedCode: http.StatusOK, + Request: &v1.ExecRequest{ + Repo: "github.com/gorilla/mux", + Args: [][]byte{[]byte("testcat")}, + Stdin: []byte("hi"), + }, + ExpectedCode: codes.OK, ExpectedBody: "hi", - ExpectedTrailers: http.Header{ - "X-Exec-Error": {""}, - "X-Exec-Exit-Status": {"0"}, - "X-Exec-Stderr": {""}, + }, + { + Name: "NonexistingRepo", + Request: &v1.ExecRequest{ + Repo: "github.com/gorilla/doesnotexist", + Args: [][]byte{[]byte("testcommand")}, }, + ExpectedCode: codes.NotFound, + ExpectedBody: "repo not found", + ExpectedDetails: []any{&v1.NotFoundPayload{ + Repo: "github.com/gorilla/doesnotexist", + CloneInProgress: false, + }}, }, { - Name: "NonexistingRepo", - Request: newRequest("POST", "/exec", strings.NewReader(`{"repo": "github.com/gorilla/doesnotexist", "args": ["testcommand"]}`)), - ExpectedCode: http.StatusNotFound, - ExpectedBody: `{"cloneInProgress":false}`, - }, - { - Name: "NonexistingRepoWithURL", - Request: newRequest( - "POST", "/exec", strings.NewReader(`{"repo": "my-doesnotexist", "url": "https://github.com/gorilla/doesntexist.git", "args": ["testcommand"]}`)), - ExpectedCode: http.StatusNotFound, - ExpectedBody: `{"cloneInProgress":false}`, - }, - { - Name: "UnclonedRepoWithoutURL", - Request: newRequest("POST", "/exec", strings.NewReader(`{"repo": "github.com/nicksnyder/go-i18n", "args": ["testcommand"]}`)), - ExpectedCode: http.StatusNotFound, - ExpectedBody: `{"cloneInProgress":true}`, // we now fetch the URL from GetRemoteURL so it works. - }, - { - Name: "UnclonedRepoWithURL", - Request: newRequest("POST", "/exec", strings.NewReader(`{"repo": "github.com/nicksnyder/go-i18n", "url": "https://github.com/nicksnyder/go-i18n.git", "args": ["testcommand"]}`)), - ExpectedCode: http.StatusNotFound, - ExpectedBody: `{"cloneInProgress":true}`, - }, - { - Name: "Error", - Request: newRequest("POST", "/exec", strings.NewReader(`{"repo": "github.com/gorilla/mux", "args": ["testerror"]}`)), - ExpectedCode: http.StatusOK, - ExpectedTrailers: http.Header{ - "X-Exec-Error": {"testerror"}, - "X-Exec-Exit-Status": {"0"}, - "X-Exec-Stderr": {""}, + Name: "UnclonedRepo", + Request: &v1.ExecRequest{ + Repo: "github.com/nicksnyder/go-i18n", + Args: [][]byte{[]byte("testcommand")}, }, + ExpectedCode: codes.NotFound, + ExpectedBody: "repo not found", + ExpectedDetails: []any{&v1.NotFoundPayload{ + Repo: "github.com/nicksnyder/go-i18n", + CloneInProgress: true, + }}, + }, + { + Name: "Error", + Request: &v1.ExecRequest{ + Repo: "github.com/gorilla/mux", + Args: [][]byte{[]byte("testerror")}, + }, + ExpectedCode: codes.Unknown, + ExpectedBody: "testerror", + ExpectedDetails: []any{&v1.ExecStatusPayload{ + StatusCode: 1, + Stderr: "teststderr", + }}, }, { Name: "EmptyInput", - Request: newRequest("POST", "/exec", strings.NewReader("{}")), - ExpectedCode: http.StatusBadRequest, + Request: &v1.ExecRequest{}, + ExpectedCode: codes.InvalidArgument, ExpectedBody: "invalid command", }, { - Name: "BadCommand", - Request: newRequest("POST", "/exec", strings.NewReader(`{"repo":"github.com/sourcegraph/sourcegraph", "args": ["invalid-command"]}`)), - ExpectedCode: http.StatusBadRequest, + Name: "BadCommand", + Request: &v1.ExecRequest{ + Repo: "github.com/sourcegraph/sourcegraph", + Args: [][]byte{[]byte("invalid-command")}, + }, + ExpectedCode: codes.InvalidArgument, ExpectedBody: "invalid command", }, } @@ -188,7 +165,10 @@ func TestExecRequest(t *testing.T) { Locker: NewRepositoryLocker(), RPSLimiter: ratelimit.NewInstrumentedLimiter("GitserverTest", rate.NewLimiter(rate.Inf, 10)), } - h := s.Handler() + // Initialize side-effects. + _ = s.Handler() + + gs := &GRPCServer{Server: s} origRepoCloned := repoCloned repoCloned = func(dir common.GitDir) bool { @@ -211,7 +191,8 @@ func TestExecRequest(t *testing.T) { _, _ = cmd.Stderr.Write([]byte("teststderr")) return 42, nil case "testerror": - return 0, errors.New("testerror") + _, _ = cmd.Stderr.Write([]byte("teststderr")) + return 1, errors.New("testerror") case "testecho", "testcat": // We do an actual exec in this case to test that code path. exe := strings.TrimPrefix(cmd.Args[1], "test") @@ -236,37 +217,46 @@ func TestExecRequest(t *testing.T) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - w := httptest.ResponseRecorder{Body: new(bytes.Buffer)} - h.ServeHTTP(&w, test.Request) + ss := gitserver.NewMockGitserverService_ExecServer() + ss.ContextFunc.SetDefaultReturn(context.Background()) + var receivedData []byte + ss.SendFunc.SetDefaultHook(func(er *v1.ExecResponse) error { + receivedData = append(receivedData, er.GetData()...) + return nil + }) + err := gs.Exec(test.Request, ss) - res := w.Result() - if res.StatusCode != test.ExpectedCode { - t.Errorf("wrong status: expected %d, got %d", test.ExpectedCode, w.Code) - } - - body, err := io.ReadAll(res.Body) - if err != nil { + if test.ExpectedCode == codes.OK && err != nil { t.Fatal(err) } - if strings.TrimSpace(string(body)) != test.ExpectedBody { - t.Errorf("wrong body: expected %q, got %q", test.ExpectedBody, string(body)) + + if test.ExpectedCode != codes.OK { + if err == nil { + t.Fatal("expected error to be returned") + } + s, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, test.ExpectedCode, s.Code()) + + if len(test.ExpectedDetails) > 0 { + if diff := cmp.Diff(test.ExpectedDetails, s.Details(), cmpopts.IgnoreUnexported(v1.ExecStatusPayload{}, v1.NotFoundPayload{})); diff != "" { + t.Fatalf("unexpected error details (-want +got):\n%s", diff) + } + } + + if strings.TrimSpace(s.Message()) != test.ExpectedBody { + t.Errorf("wrong error body: expected %q, got %q", test.ExpectedBody, s.Message()) + } + return } - for k, v := range test.ExpectedTrailers { - if got := res.Trailer.Get(k); got != v[0] { - t.Errorf("wrong trailer %q: expected %q, got %q", k, v[0], got) - } + if strings.TrimSpace(string(receivedData)) != test.ExpectedBody { + t.Errorf("wrong body: expected %q, got %q", test.ExpectedBody, string(receivedData)) } }) } } -func staticGetRemoteURL(remote string) func(context.Context, api.RepoName) (string, error) { - return func(context.Context, api.RepoName) (string, error) { - return remote, nil - } -} - // makeSingleCommitRepo make create a new repo with a single commit and returns // the HEAD SHA func makeSingleCommitRepo(cmd func(string, ...string) string) string { @@ -305,10 +295,12 @@ func makeTestServer(ctx context.Context, t *testing.T, repoDir, remote string, d cloneQueue := NewCloneQueue(obctx, list.New()) s := &Server{ - Logger: logger, - ObservationCtx: obctx, - ReposDir: repoDir, - GetRemoteURLFunc: staticGetRemoteURL(remote), + Logger: logger, + ObservationCtx: obctx, + ReposDir: repoDir, + GetRemoteURLFunc: func(context.Context, api.RepoName) (string, error) { + return remote, nil + }, GetVCSSyncer: func(ctx context.Context, name api.RepoName) (vcssyncer.VCSSyncer, error) { return vcssyncer.NewGitRepoSyncer(logtest.Scoped(t), wrexec.NewNoOpRecordingCommandFactory()), nil }, @@ -525,130 +517,6 @@ var ignoreVolatileGitserverRepoFields = cmpopts.IgnoreFields( "CloningProgress", ) -func TestHandleRepoDelete(t *testing.T) { - testHandleRepoDelete(t, false) -} - -func TestHandleRepoDeleteWhenDeleteInDB(t *testing.T) { - // We also want to ensure that we can delete repo data on disk for a repo that - // has already been deleted in the DB. - testHandleRepoDelete(t, true) -} - -func testHandleRepoDelete(t *testing.T, deletedInDB bool) { - logger := logtest.Scoped(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - remote := t.TempDir() - repoName := api.RepoName("example.com/foo/bar") - db := database.NewDB(logger, dbtest.NewDB(t)) - - dbRepo := &types.Repo{ - Name: repoName, - Description: "Test", - } - - // Insert the repo into our database - if err := db.Repos().Create(ctx, dbRepo); err != nil { - t.Fatal(err) - } - - repo := remote - cmd := func(name string, arg ...string) string { - t.Helper() - return runCmd(t, repo, name, arg...) - } - _ = makeSingleCommitRepo(cmd) - // Add a bad tag - cmd("git", "tag", "HEAD") - - reposDir := t.TempDir() - - s := makeTestServer(ctx, t, reposDir, remote, db) - - // We need some of the side effects here - _ = s.Handler() - - rr := httptest.NewRecorder() - - updateReq := protocol.RepoUpdateRequest{ - Repo: repoName, - } - body, err := json.Marshal(updateReq) - if err != nil { - t.Fatal(err) - } - - // This will perform an initial clone - req := newRequest("GET", "/repo-update", bytes.NewReader(body)) - s.handleRepoUpdate(rr, req) - - size := gitserverfs.DirSize(gitserverfs.RepoDirFromName(s.ReposDir, repoName).Path(".")) - want := &types.GitserverRepo{ - RepoID: dbRepo.ID, - ShardID: "", - CloneStatus: types.CloneStatusCloned, - RepoSizeBytes: size, - } - fromDB, err := db.GitserverRepos().GetByID(ctx, dbRepo.ID) - if err != nil { - t.Fatal(err) - } - - // We don't expect an error - if diff := cmp.Diff(want, fromDB, ignoreVolatileGitserverRepoFields); diff != "" { - t.Fatal(diff) - } - - if deletedInDB { - if err := db.Repos().Delete(ctx, dbRepo.ID); err != nil { - t.Fatal(err) - } - repos, err := db.Repos().List(ctx, database.ReposListOptions{IncludeDeleted: true, IDs: []api.RepoID{dbRepo.ID}}) - if err != nil { - t.Fatal(err) - } - if len(repos) != 1 { - t.Fatalf("Expected 1 repo, got %d", len(repos)) - } - dbRepo = repos[0] - } - - // Now we can delete it - deleteReq := protocol.RepoDeleteRequest{ - Repo: dbRepo.Name, - } - body, err = json.Marshal(deleteReq) - if err != nil { - t.Fatal(err) - } - req = newRequest("GET", "/delete", bytes.NewReader(body)) - s.handleRepoDelete(rr, req) - - size = gitserverfs.DirSize(gitserverfs.RepoDirFromName(s.ReposDir, repoName).Path(".")) - if size != 0 { - t.Fatalf("Size should be 0, got %d", size) - } - - // Check status in gitserver_repos - want = &types.GitserverRepo{ - RepoID: dbRepo.ID, - ShardID: "", - CloneStatus: types.CloneStatusNotCloned, - RepoSizeBytes: size, - } - fromDB, err = db.GitserverRepos().GetByID(ctx, dbRepo.ID) - if err != nil { - t.Fatal(err) - } - - // We don't expect an error - if diff := cmp.Diff(want, fromDB, ignoreVolatileGitserverRepoFields); diff != "" { - t.Fatal(diff) - } -} - func TestHandleRepoUpdate(t *testing.T) { logger := logtest.Scoped(t) ctx, cancel := context.WithCancel(context.Background()) @@ -683,23 +551,14 @@ func TestHandleRepoUpdate(t *testing.T) { // We need the side effects here _ = s.Handler() - rr := httptest.NewRecorder() - - updateReq := protocol.RepoUpdateRequest{ - Repo: repoName, - } - body, err := json.Marshal(updateReq) - if err != nil { - t.Fatal(err) - } - // Confirm that failing to clone the repo stores the error oldRemoveURLFunc := s.GetRemoteURLFunc s.GetRemoteURLFunc = func(ctx context.Context, name api.RepoName) (string, error) { return "https://invalid.example.com/", nil } - req := newRequest("GET", "/repo-update", bytes.NewReader(body)) - s.handleRepoUpdate(rr, req) + s.repoUpdate(&protocol.RepoUpdateRequest{ + Repo: repoName, + }) size := gitserverfs.DirSize(gitserverfs.RepoDirFromName(s.ReposDir, repoName).Path(".")) want := &types.GitserverRepo{ @@ -728,8 +587,9 @@ func TestHandleRepoUpdate(t *testing.T) { // This will perform an initial clone s.GetRemoteURLFunc = oldRemoveURLFunc - req = newRequest("GET", "/repo-update", bytes.NewReader(body)) - s.handleRepoUpdate(rr, req) + s.repoUpdate(&protocol.RepoUpdateRequest{ + Repo: repoName, + }) size = gitserverfs.DirSize(gitserverfs.RepoDirFromName(s.ReposDir, repoName).Path(".")) want = &types.GitserverRepo{ @@ -756,8 +616,9 @@ func TestHandleRepoUpdate(t *testing.T) { t.Cleanup(func() { doBackgroundRepoUpdateMock = nil }) // This will trigger an update since the repo is already cloned - req = newRequest("GET", "/repo-update", bytes.NewReader(body)) - s.handleRepoUpdate(rr, req) + s.repoUpdate(&protocol.RepoUpdateRequest{ + Repo: repoName, + }) want = &types.GitserverRepo{ RepoID: dbRepo.ID, @@ -780,8 +641,9 @@ func TestHandleRepoUpdate(t *testing.T) { doBackgroundRepoUpdateMock = nil // This will trigger an update since the repo is already cloned - req = newRequest("GET", "/repo-update", bytes.NewReader(body)) - s.handleRepoUpdate(rr, req) + s.repoUpdate(&protocol.RepoUpdateRequest{ + Repo: repoName, + }) want = &types.GitserverRepo{ RepoID: dbRepo.ID, @@ -1060,11 +922,10 @@ func TestSyncRepoState(t *testing.T) { } type BatchLogTest struct { - Name string - Request *http.Request - ExpectedCode int - ExpectedBody string - RunCommandMock func(ctx context.Context, cmd *exec.Cmd) (int, error) + Name string + Request *v1.BatchLogRequest + ExpectedCode codes.Code + ExpectedBody *protocol.BatchLogResponse } func TestHandleBatchLog(t *testing.T) { @@ -1087,32 +948,26 @@ func TestHandleBatchLog(t *testing.T) { t.Cleanup(func() { executil.UpdateRunCommandMock(nil) }) tests := []BatchLogTest{ - { - Name: "bad request", - Request: newRequest("POST", "/batch-log", strings.NewReader(``)), - ExpectedCode: http.StatusBadRequest, - ExpectedBody: "EOF", // the particular error when parsing empty payload - }, { Name: "empty", - Request: newRequest("POST", "/batch-log", strings.NewReader(`{}`)), - ExpectedCode: http.StatusOK, - ExpectedBody: mustEncodeJSONResponse(protocol.BatchLogResponse{ + Request: &v1.BatchLogRequest{}, + ExpectedCode: codes.OK, + ExpectedBody: &protocol.BatchLogResponse{ Results: []protocol.BatchLogResult{}, - }), + }, }, { Name: "all resolved", - Request: newRequest("POST", "/batch-log", strings.NewReader(`{ - "repoCommits": [ - {"repo": "github.com/foo/bar", "commitId": "deadbeef1"}, - {"repo": "github.com/foo/baz", "commitId": "deadbeef2"}, - {"repo": "github.com/foo/bonk", "commitId": "deadbeef3"} - ], - "format": "--format=test" - }`)), - ExpectedCode: http.StatusOK, - ExpectedBody: mustEncodeJSONResponse(protocol.BatchLogResponse{ + Request: &v1.BatchLogRequest{ + RepoCommits: []*v1.RepoCommit{ + {Repo: "github.com/foo/bar", Commit: "deadbeef1"}, + {Repo: "github.com/foo/baz", Commit: "deadbeef2"}, + {Repo: "github.com/foo/bonk", Commit: "deadbeef3"}, + }, + Format: "--format=test", + }, + ExpectedCode: codes.OK, + ExpectedBody: &protocol.BatchLogResponse{ Results: []protocol.BatchLogResult{ { RepoCommit: api.RepoCommit{Repo: "github.com/foo/bar", CommitID: "deadbeef1"}, @@ -1130,20 +985,20 @@ func TestHandleBatchLog(t *testing.T) { CommandError: "", }, }, - }), + }, }, { Name: "partially resolved", - Request: newRequest("POST", "/batch-log", strings.NewReader(`{ - "repoCommits": [ - {"repo": "github.com/foo/bar", "commitId": "deadbeef1"}, - {"repo": "github.com/foo/baz", "commitId": "dumbmilk1"}, - {"repo": "github.com/foo/honk", "commitId": "deadbeef3"} - ], - "format": "--format=test" - }`)), - ExpectedCode: http.StatusOK, - ExpectedBody: mustEncodeJSONResponse(protocol.BatchLogResponse{ + Request: &v1.BatchLogRequest{ + RepoCommits: []*v1.RepoCommit{ + {Repo: "github.com/foo/bar", Commit: "deadbeef1"}, + {Repo: "github.com/foo/baz", Commit: "dumbmilk1"}, + {Repo: "github.com/foo/honk", Commit: "deadbeef3"}, + }, + Format: "--format=test", + }, + ExpectedCode: codes.OK, + ExpectedBody: &protocol.BatchLogResponse{ Results: []protocol.BatchLogResult{ { RepoCommit: api.RepoCommit{Repo: "github.com/foo/bar", CommitID: "deadbeef1"}, @@ -1163,7 +1018,7 @@ func TestHandleBatchLog(t *testing.T) { CommandError: "repo not found", }, }, - }), + }, }, } @@ -1177,129 +1032,35 @@ func TestHandleBatchLog(t *testing.T) { RecordingCommandFactory: wrexec.NewNoOpRecordingCommandFactory(), Locker: NewRepositoryLocker(), } - h := server.Handler() + // Initialize side-effects. + _ = server.Handler() - w := httptest.ResponseRecorder{Body: new(bytes.Buffer)} - h.ServeHTTP(&w, test.Request) + gs := &GRPCServer{Server: server} - res := w.Result() - if res.StatusCode != test.ExpectedCode { - t.Errorf("wrong status: expected %d, got %d", test.ExpectedCode, w.Code) - } + res, err := gs.BatchLog(context.Background(), test.Request) - body, err := io.ReadAll(res.Body) - if err != nil { + if test.ExpectedCode == codes.OK && err != nil { t.Fatal(err) } - if strings.TrimSpace(string(body)) != test.ExpectedBody { - t.Errorf("wrong body: expected %q, got %q", test.ExpectedBody, string(body)) + + if test.ExpectedCode != codes.OK { + if err == nil { + t.Fatal("expected error to be returned") + } + s, ok := status.FromError(err) + require.True(t, ok) + require.Equal(t, test.ExpectedCode, s.Code()) + return } + + var have protocol.BatchLogResponse + have.FromProto(res) + + require.Equal(t, test.ExpectedBody, &have) }) } } -func TestHeaderXRequestedWithMiddleware(t *testing.T) { - test := headerXRequestedWithMiddleware( - http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("success")) - w.WriteHeader(http.StatusOK) - }), - ) - - assertBody := func(result *http.Response, want string) { - b, err := io.ReadAll(result.Body) - if err != nil { - t.Fatalf("failed to read body: %v", err) - } - - data := string(b) - - if data != want { - t.Fatalf(`Expected body to contain %q, but found %q`, want, data) - } - } - - failureExpectation := "header X-Requested-With is not set or is invalid\n" - - t.Run("x-requested-with not set", func(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/", nil) - w := httptest.NewRecorder() - - test(w, r) - - result := w.Result() - defer result.Body.Close() - - if result.StatusCode != http.StatusBadRequest { - t.Fatalf("expected HTTP status code %d, but got %d", http.StatusBadRequest, result.StatusCode) - } - - assertBody(result, failureExpectation) - }) - - t.Run("x-requested-with invalid value", func(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/", nil) - r.Header.Add("X-Requested-With", "foo") - w := httptest.NewRecorder() - - test(w, r) - - result := w.Result() - defer result.Body.Close() - - if result.StatusCode != http.StatusBadRequest { - t.Fatalf("expected HTTP status code %d, but got %d", http.StatusBadRequest, result.StatusCode) - } - - assertBody(result, failureExpectation) - }) - - t.Run("x-requested-with correct value", func(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/", nil) - r.Header.Add("X-Requested-With", "Sourcegraph") - w := httptest.NewRecorder() - - test(w, r) - - result := w.Result() - defer result.Body.Close() - - if result.StatusCode != http.StatusOK { - t.Fatalf("expected HTTP status code %d, but got %d", http.StatusOK, result.StatusCode) - } - - assertBody(result, "success") - }) - - t.Run("check skippped for /ping", func(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/ping", nil) - w := httptest.NewRecorder() - - test(w, r) - - result := w.Result() - defer result.Body.Close() - - if result.StatusCode != http.StatusOK { - t.Fatalf("expected HTTP status code %d, but got %d", http.StatusOK, result.StatusCode) - } - }) - - t.Run("check skipped for /git", func(t *testing.T) { - r := httptest.NewRequest(http.MethodGet, "/git/foo/bar", nil) - w := httptest.NewRecorder() - - test(w, r) - - result := w.Result() - defer result.Body.Close() - - if result.StatusCode != http.StatusOK { - t.Fatalf("expected HTTP status code %d, but got %d", http.StatusOK, result.StatusCode) - } - }) -} - func TestLogIfCorrupt(t *testing.T) { logger := logtest.Scoped(t) ctx, cancel := context.WithCancel(context.Background()) @@ -1367,11 +1128,6 @@ func TestLogIfCorrupt(t *testing.T) { }) } -func mustEncodeJSONResponse(value any) string { - encoded, _ := json.Marshal(value) - return strings.TrimSpace(string(encoded)) -} - func TestStdErrIndicatesCorruption(t *testing.T) { bad := []string{ "error: packfile .git/objects/pack/pack-a.pack does not match index", diff --git a/cmd/gitserver/internal/serverutil.go b/cmd/gitserver/internal/serverutil.go index f37514fc3ad..4eca16ac5b5 100644 --- a/cmd/gitserver/internal/serverutil.go +++ b/cmd/gitserver/internal/serverutil.go @@ -3,11 +3,8 @@ package internal import ( "fmt" "io" - "net/http" "os" - "reflect" "strings" - "sync" "time" "github.com/sourcegraph/log" @@ -107,111 +104,6 @@ func (l *limitWriter) Write(p []byte) (int, error) { return n, err } -// flushingResponseWriter is a http.ResponseWriter that flushes all writes -// to the underlying connection within a certain time period after Write is -// called (instead of buffering them indefinitely). -// -// This lets, e.g., clients with a context deadline see as much partial response -// body as possible. -type flushingResponseWriter struct { - // mu ensures we don't concurrently call Flush and Write. It also protects - // state. - mu sync.Mutex - w http.ResponseWriter - flusher http.Flusher - closed bool - doFlush bool -} - -var logUnflushableResponseWriterOnce sync.Once - -// newFlushingResponseWriter creates a new flushing response writer. Callers -// must call Close to free the resources created by the writer. -// -// If w does not support flushing, it returns nil. -func newFlushingResponseWriter(logger log.Logger, w http.ResponseWriter) *flushingResponseWriter { - // We panic if we don't implement the needed interfaces. - flusher := hackilyGetHTTPFlusher(w) - if flusher == nil { - logUnflushableResponseWriterOnce.Do(func() { - logger.Warn("unable to flush HTTP response bodies - Diff search performance and completeness will be affected", - log.String("type", reflect.TypeOf(w).String())) - }) - return nil - } - - w.Header().Set("Transfer-Encoding", "chunked") - - f := &flushingResponseWriter{w: w, flusher: flusher} - go f.periodicFlush() - return f -} - -// hackilyGetHTTPFlusher attempts to get an http.Flusher from w. It (hackily) handles the case where w is a -// nethttp.statusCodeTracker (which wraps http.ResponseWriter and does not implement http.Flusher). See -// https://github.com/opentracing-contrib/go-stdlib/pull/11#discussion_r164295773 and -// https://github.com/sourcegraph/sourcegraph/issues/9045. -// -// I (@sqs) wrote this hack instead of fixing it upstream immediately because seems to be some reluctance to merge -// a fix (because it'd make the http.ResponseWriter falsely appear to implement many interfaces that it doesn't -// actually implement, so it would break the correctness of Go type-assertion impl checks). -func hackilyGetHTTPFlusher(w http.ResponseWriter) http.Flusher { - if f, ok := w.(http.Flusher); ok { - return f - } - if reflect.TypeOf(w).String() == "*nethttp.statusCodeTracker" { - v := reflect.ValueOf(w).Elem() - if v.Kind() == reflect.Struct { - if rwv := v.FieldByName("ResponseWriter"); rwv.IsValid() { - f, ok := rwv.Interface().(http.Flusher) - if ok { - return f - } - } - } - } - return nil -} - -// Header implements http.ResponseWriter. -func (f *flushingResponseWriter) Header() http.Header { return f.w.Header() } - -// WriteHeader implements http.ResponseWriter. -func (f *flushingResponseWriter) WriteHeader(code int) { f.w.WriteHeader(code) } - -// Write implements http.ResponseWriter. -func (f *flushingResponseWriter) Write(p []byte) (int, error) { - f.mu.Lock() - n, err := f.w.Write(p) - if n > 0 { - f.doFlush = true - } - f.mu.Unlock() - return n, err -} - -func (f *flushingResponseWriter) periodicFlush() { - for { - time.Sleep(100 * time.Millisecond) - f.mu.Lock() - if f.closed { - f.mu.Unlock() - break - } - if f.doFlush { - f.flusher.Flush() - } - f.mu.Unlock() - } -} - -// Close signals to the flush goroutine to stop. -func (f *flushingResponseWriter) Close() { - f.mu.Lock() - f.closed = true - f.mu.Unlock() -} - // mapToLoggerField translates a map to log context fields. func mapToLoggerField(m map[string]any) []log.Field { LogFields := []log.Field{} diff --git a/cmd/gitserver/internal/serverutil_test.go b/cmd/gitserver/internal/serverutil_test.go deleted file mode 100644 index d156aa085e8..00000000000 --- a/cmd/gitserver/internal/serverutil_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package internal - -import ( - "net/http/httptest" - "testing" - "time" -) - -func TestFlushingResponseWriter(t *testing.T) { - flush := make(chan struct{}) - fw := &flushingResponseWriter{ - w: httptest.NewRecorder(), - flusher: flushFunc(func() { - flush <- struct{}{} - }), - } - done := make(chan struct{}) - go func() { - fw.periodicFlush() - close(done) - }() - - _, _ = fw.Write([]byte("hi")) - - select { - case <-flush: - close(flush) - case <-time.After(5 * time.Second): - t.Fatal("periodic flush did not happen") - } - - fw.Close() - - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("periodic flush goroutine did not close") - } -} - -type flushFunc func() - -func (f flushFunc) Flush() { - f() -} diff --git a/cmd/repo-updater/internal/repoupdater/BUILD.bazel b/cmd/repo-updater/internal/repoupdater/BUILD.bazel index a443fa8d3b6..e287b91669f 100644 --- a/cmd/repo-updater/internal/repoupdater/BUILD.bazel +++ b/cmd/repo-updater/internal/repoupdater/BUILD.bazel @@ -71,7 +71,6 @@ go_test( "@com_github_google_go_cmp//cmp/cmpopts", "@com_github_inconshreveable_log15//:log15", "@com_github_sourcegraph_log//logtest", - "@io_opentelemetry_go_otel_trace//:trace", "@org_golang_google_grpc//:go_default_library", ], ) diff --git a/cmd/repo-updater/internal/repoupdater/grpc_server.go b/cmd/repo-updater/internal/repoupdater/grpc_server.go index a8cbd8d9134..4c674603544 100644 --- a/cmd/repo-updater/internal/repoupdater/grpc_server.go +++ b/cmd/repo-updater/internal/repoupdater/grpc_server.go @@ -23,7 +23,7 @@ func (s *RepoUpdaterServiceServer) RepoUpdateSchedulerInfo(_ context.Context, re } func (s *RepoUpdaterServiceServer) RepoLookup(ctx context.Context, req *proto.RepoLookupRequest) (*proto.RepoLookupResponse, error) { - res, err := s.Server.repoLookup(ctx, api.RepoName(req.Repo)) + res, err := s.Server.repoLookup(ctx, api.RepoName(req.GetRepo())) if err != nil { return nil, err } diff --git a/cmd/repo-updater/internal/repoupdater/server.go b/cmd/repo-updater/internal/repoupdater/server.go index 0475b7d3403..c2d2c6adeb3 100644 --- a/cmd/repo-updater/internal/repoupdater/server.go +++ b/cmd/repo-updater/internal/repoupdater/server.go @@ -3,7 +3,6 @@ package repoupdater import ( "context" - "encoding/json" "fmt" "net/http" "time" @@ -42,60 +41,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("/healthz", trace.WithRouteName("healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) - mux.HandleFunc("/repo-update-scheduler-info", trace.WithRouteName("repo-update-scheduler-info", s.handleRepoUpdateSchedulerInfo)) - mux.HandleFunc("/repo-lookup", trace.WithRouteName("repo-lookup", s.handleRepoLookup)) - mux.HandleFunc("/enqueue-repo-update", trace.WithRouteName("enqueue-repo-update", s.handleEnqueueRepoUpdate)) - mux.HandleFunc("/enqueue-changeset-sync", trace.WithRouteName("enqueue-changeset-sync", s.handleEnqueueChangesetSync)) return mux } -func (s *Server) handleRepoUpdateSchedulerInfo(w http.ResponseWriter, r *http.Request) { - var args protocol.RepoUpdateSchedulerInfoArgs - if err := json.NewDecoder(r.Body).Decode(&args); err != nil { - s.respond(w, http.StatusBadRequest, err) - return - } - - result := s.Scheduler.ScheduleInfo(args.ID) - s.respond(w, http.StatusOK, result) -} - -func (s *Server) handleRepoLookup(w http.ResponseWriter, r *http.Request) { - var args protocol.RepoLookupArgs - if err := json.NewDecoder(r.Body).Decode(&args); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - result, err := s.repoLookup(r.Context(), args.Repo) - if err != nil { - if r.Context().Err() != nil { - http.Error(w, "request canceled", http.StatusGatewayTimeout) - return - } - s.Logger.Error("repoLookup failed", log.String("name", string(args.Repo)), log.Error(err)) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - s.respond(w, http.StatusOK, result) -} - -func (s *Server) handleEnqueueRepoUpdate(w http.ResponseWriter, r *http.Request) { - var req protocol.RepoUpdateRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - s.respond(w, http.StatusBadRequest, err) - return - } - result, status, err := s.enqueueRepoUpdate(r.Context(), &req) - if err != nil { - s.Logger.Warn("enqueueRepoUpdate failed", log.String("req", fmt.Sprint(req)), log.Error(err)) - s.respond(w, status, err) - return - } - s.respond(w, status, result) -} - func (s *Server) enqueueRepoUpdate(ctx context.Context, req *protocol.RepoUpdateRequest) (resp *protocol.RepoUpdateResponse, httpStatus int, err error) { tr, ctx := trace.New(ctx, "enqueueRepoUpdate", attribute.Stringer("req", req)) defer func() { @@ -129,30 +77,6 @@ func (s *Server) enqueueRepoUpdate(ctx context.Context, req *protocol.RepoUpdate }, http.StatusOK, nil } -func (s *Server) respond(w http.ResponseWriter, code int, v any) { - switch val := v.(type) { - case error: - if val != nil { - s.Logger.Error("response value error", log.Error(val)) - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.WriteHeader(code) - fmt.Fprintf(w, "%v", val) - } - default: - w.Header().Set("Content-Type", "application/json") - bs, err := json.Marshal(v) - if err != nil { - s.respond(w, http.StatusInternalServerError, err) - return - } - - w.WriteHeader(code) - if _, err = w.Write(bs); err != nil { - s.Logger.Error("failed to write response", log.Error(err)) - } - } -} - var mockRepoLookup func(api.RepoName) (*protocol.RepoLookupResult, error) func (s *Server) repoLookup(ctx context.Context, repoName api.RepoName) (result *protocol.RepoLookupResult, err error) { @@ -196,28 +120,3 @@ func (s *Server) repoLookup(ctx context.Context, repoName api.RepoName) (result return &protocol.RepoLookupResult{Repo: protocol.NewRepoInfo(repo)}, nil } - -func (s *Server) handleEnqueueChangesetSync(w http.ResponseWriter, r *http.Request) { - if s.ChangesetSyncRegistry == nil { - s.Logger.Warn("ChangesetSyncer is nil") - s.respond(w, http.StatusForbidden, nil) - return - } - - var req protocol.ChangesetSyncRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - s.respond(w, http.StatusBadRequest, err) - return - } - if len(req.IDs) == 0 { - s.respond(w, http.StatusBadRequest, errors.New("no ids provided")) - return - } - err := s.ChangesetSyncRegistry.EnqueueChangesetSyncs(r.Context(), req.IDs) - if err != nil { - resp := protocol.ChangesetSyncResponse{Error: err.Error()} - s.respond(w, http.StatusInternalServerError, resp) - return - } - s.respond(w, http.StatusOK, nil) -} diff --git a/cmd/repo-updater/internal/repoupdater/server_test.go b/cmd/repo-updater/internal/repoupdater/server_test.go index 9a30b01afef..67ac1cbcfd2 100644 --- a/cmd/repo-updater/internal/repoupdater/server_test.go +++ b/cmd/repo-updater/internal/repoupdater/server_test.go @@ -1,9 +1,7 @@ package repoupdater import ( - "bytes" "context" - "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -14,7 +12,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "github.com/sourcegraph/log/logtest" @@ -43,108 +40,6 @@ import ( "github.com/sourcegraph/sourcegraph/lib/errors" ) -func TestServer_handleRepoLookup(t *testing.T) { - logger := logtest.Scoped(t) - s := &Server{Logger: logger} - - h := ObservedHandler( - logger, - NewHandlerMetrics(), - trace.NewNoopTracerProvider(), - )(s.Handler()) - - repoLookup := func(t *testing.T, repo api.RepoName) (resp *protocol.RepoLookupResult, statusCode int) { - t.Helper() - rr := httptest.NewRecorder() - body, err := json.Marshal(protocol.RepoLookupArgs{Repo: repo}) - if err != nil { - t.Fatal(err) - } - req := httptest.NewRequest("GET", "/repo-lookup", bytes.NewReader(body)) - fmt.Printf("h: %v rr: %v req: %v\n", h, rr, req) - h.ServeHTTP(rr, req) - if rr.Code == http.StatusOK { - if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { - t.Fatal(err) - } - } - return resp, rr.Code - } - repoLookupResult := func(t *testing.T, repo api.RepoName) protocol.RepoLookupResult { - t.Helper() - resp, statusCode := repoLookup(t, repo) - if statusCode != http.StatusOK { - t.Fatalf("http non-200 status %d", statusCode) - } - return *resp - } - - t.Run("args", func(t *testing.T) { - called := false - mockRepoLookup = func(repoName api.RepoName) (*protocol.RepoLookupResult, error) { - called = true - if want := api.RepoName("github.com/a/b"); repoName != want { - t.Errorf("got owner %q, want %q", repoName, want) - } - return &protocol.RepoLookupResult{Repo: nil}, nil - } - defer func() { mockRepoLookup = nil }() - - repoLookupResult(t, "github.com/a/b") - if !called { - t.Error("!called") - } - }) - - t.Run("not found", func(t *testing.T) { - mockRepoLookup = func(api.RepoName) (*protocol.RepoLookupResult, error) { - return &protocol.RepoLookupResult{Repo: nil}, nil - } - defer func() { mockRepoLookup = nil }() - - if got, want := repoLookupResult(t, "github.com/a/b"), (protocol.RepoLookupResult{}); !reflect.DeepEqual(got, want) { - t.Errorf("got %+v, want %+v", got, want) - } - }) - - t.Run("unexpected error", func(t *testing.T) { - mockRepoLookup = func(api.RepoName) (*protocol.RepoLookupResult, error) { - return nil, errors.New("x") - } - defer func() { mockRepoLookup = nil }() - - result, statusCode := repoLookup(t, "github.com/a/b") - if result != nil { - t.Errorf("got result %+v, want nil", result) - } - if want := http.StatusInternalServerError; statusCode != want { - t.Errorf("got HTTP status code %d, want %d", statusCode, want) - } - }) - - t.Run("found", func(t *testing.T) { - want := protocol.RepoLookupResult{ - Repo: &protocol.RepoInfo{ - ExternalRepo: api.ExternalRepoSpec{ - ID: "a", - ServiceType: extsvc.TypeGitHub, - ServiceID: "https://github.com/", - }, - Name: "github.com/c/d", - Description: "b", - Fork: true, - }, - } - mockRepoLookup = func(api.RepoName) (*protocol.RepoLookupResult, error) { - return &want, nil - } - defer func() { mockRepoLookup = nil }() - if got := repoLookupResult(t, "github.com/c/d"); !reflect.DeepEqual(got, want) { - t.Errorf("got %+v, want %+v", got, want) - } - }) -} - func TestServer_EnqueueRepoUpdate(t *testing.T) { ctx := context.Background() diff --git a/cmd/repo-updater/shared/main.go b/cmd/repo-updater/shared/main.go index 65dc5985760..0da41d218a9 100644 --- a/cmd/repo-updater/shared/main.go +++ b/cmd/repo-updater/shared/main.go @@ -109,7 +109,7 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic sourceMetrics := repos.NewSourceMetrics() sourceMetrics.MustRegister(prometheus.DefaultRegisterer) - src := repos.NewSourcer(sourcerLogger, db, cf, repos.WithDependenciesService(dependencies.NewService(observationCtx, db)), repos.ObservedSource(sourcerLogger, sourceMetrics)) + src := repos.NewSourcer(sourcerLogger, db, cf, gitserver.NewClient("repo-updater.sourcer"), repos.WithDependenciesService(dependencies.NewService(observationCtx, db)), repos.ObservedSource(sourcerLogger, sourceMetrics)) syncer := repos.NewSyncer(observationCtx, store, src) updateScheduler := scheduler.NewUpdateScheduler(logger, db, gitserver.NewClient("repos.updatescheduler")) server := &repoupdater.Server{ diff --git a/cmd/searcher/internal/search/BUILD.bazel b/cmd/searcher/internal/search/BUILD.bazel index fc151232902..3e493f606e9 100644 --- a/cmd/searcher/internal/search/BUILD.bazel +++ b/cmd/searcher/internal/search/BUILD.bazel @@ -38,8 +38,6 @@ go_library( "//internal/search", "//internal/search/backend", "//internal/search/casetransform", - "//internal/search/searcher", - "//internal/search/streaming/http", "//internal/search/zoekt", "//internal/searcher/v1:searcher", "//internal/trace", @@ -135,6 +133,7 @@ go_test( "//cmd/searcher/protocol", "//internal/api", "//internal/comby", + "//internal/conf", "//internal/errcode", "//internal/gitserver", "//internal/grpc", @@ -142,7 +141,6 @@ go_test( "//internal/observation", "//internal/search", "//internal/search/backend", - "//internal/search/searcher", "//internal/searcher/v1:searcher", "//lib/errors", "//schema", diff --git a/cmd/searcher/internal/search/hybrid_test.go b/cmd/searcher/internal/search/hybrid_test.go index 36bbffde7c9..120d25b6775 100644 --- a/cmd/searcher/internal/search/hybrid_test.go +++ b/cmd/searcher/internal/search/hybrid_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "net/http" "net/http/httptest" "sort" "strings" @@ -135,7 +136,7 @@ Hello world example in go`, typeFile}, Service: service, }) - handler := internalgrpc.MultiplexHandlers(grpcServer, service) + handler := internalgrpc.MultiplexHandlers(grpcServer, http.HandlerFunc(http.NotFound)) ts := httptest.NewServer(handler) @@ -260,7 +261,7 @@ unchanged.md FetchTimeout: fetchTimeoutForCI(t), } - m, err := doSearch(ts.URL, &req) + m, err := doSearch(t, ts.URL, &req) if err != nil { t.Fatal(err) } diff --git a/cmd/searcher/internal/search/search.go b/cmd/searcher/internal/search/search.go index 182c8202728..ceead34aa6b 100644 --- a/cmd/searcher/internal/search/search.go +++ b/cmd/searcher/internal/search/search.go @@ -13,11 +13,6 @@ package search import ( "context" - "encoding/json" - "math" - "net" - "net/http" - "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -30,8 +25,6 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/errcode" - "github.com/sourcegraph/sourcegraph/internal/search/searcher" - streamhttp "github.com/sourcegraph/sourcegraph/internal/search/streaming/http" "github.com/sourcegraph/sourcegraph/internal/trace" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -62,81 +55,6 @@ type Service struct { MaxTotalPathsLength int } -// ServeHTTP handles HTTP based search requests -func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - - var p protocol.Request - dec := json.NewDecoder(r.Body) - if err := dec.Decode(&p); err != nil { - http.Error(w, "failed to decode form: "+err.Error(), http.StatusBadRequest) - return - } - - if !p.PatternMatchesContent && !p.PatternMatchesPath { - // BACKCOMPAT: Old frontends send neither of these fields, but we still want to - // search file content in that case. - p.PatternMatchesContent = true - } - if err := validateParams(&p); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - s.streamSearch(ctx, w, p) -} - -// isNetOpError returns true if net.OpError is contained in err. This is -// useful to ignore errors when the connection has gone away. -func isNetOpError(err error) bool { - return errors.HasType(err, (*net.OpError)(nil)) -} - -func (s *Service) streamSearch(ctx context.Context, w http.ResponseWriter, p protocol.Request) { - if p.Limit == 0 { - // No limit for streaming search since upstream limits - // will either be sent in the request, or propagated by - // a cancelled context. - p.Limit = math.MaxInt32 - } - eventWriter, err := streamhttp.NewWriter(w) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - var bufMux sync.Mutex - matchesBuf := streamhttp.NewJSONArrayBuf(32*1024, func(data []byte) error { - return eventWriter.EventBytes("matches", data) - }) - onMatches := func(match protocol.FileMatch) { - bufMux.Lock() - if err := matchesBuf.Append(match); err != nil && !isNetOpError(err) { - s.Log.Warn("failed appending match to buffer", log.Error(err)) - } - bufMux.Unlock() - } - - ctx, cancel, stream := newLimitedStream(ctx, p.Limit, onMatches) - defer cancel() - - err = s.search(ctx, &p, stream) - doneEvent := searcher.EventDone{ - LimitHit: stream.LimitHit(), - } - if err != nil { - doneEvent.Error = err.Error() - } - - // Flush remaining matches before sending a different event - if err := matchesBuf.Flush(); err != nil && !isNetOpError(err) { - s.Log.Warn("failed to flush matches", log.Error(err)) - } - if err := eventWriter.Event("done", doneEvent); err != nil && !isNetOpError(err) { - s.Log.Warn("failed to send done event", log.Error(err)) - } -} - func (s *Service) search(ctx context.Context, p *protocol.Request, sender matchSender) (err error) { metricRunning.Inc() defer metricRunning.Dec() diff --git a/cmd/searcher/internal/search/search_grpc.go b/cmd/searcher/internal/search/search_grpc.go index 56b40ed621c..1575575b7fa 100644 --- a/cmd/searcher/internal/search/search_grpc.go +++ b/cmd/searcher/internal/search/search_grpc.go @@ -2,13 +2,15 @@ package search import ( "context" + "math" "strings" "sync" - "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" - proto "github.com/sourcegraph/sourcegraph/internal/searcher/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" + proto "github.com/sourcegraph/sourcegraph/internal/searcher/v1" ) type Server struct { @@ -17,8 +19,24 @@ type Server struct { } func (s *Server) Search(req *proto.SearchRequest, stream proto.SearcherService_SearchServer) error { - var unmarshaledReq protocol.Request - unmarshaledReq.FromProto(req) + var p protocol.Request + p.FromProto(req) + + if !p.PatternMatchesContent && !p.PatternMatchesPath { + // BACKCOMPAT: Old frontends send neither of these fields, but we still want to + // search file content in that case. + p.PatternMatchesContent = true + } + if err := validateParams(&p); err != nil { + return status.Error(codes.InvalidArgument, err.Error()) + } + + if p.Limit == 0 { + // No limit for streaming search since upstream limits + // will either be sent in the request, or propagated by + // a cancelled context. + p.Limit = math.MaxInt32 + } // mu protects the stream from concurrent writes. var mu sync.Mutex @@ -33,10 +51,10 @@ func (s *Server) Search(req *proto.SearchRequest, stream proto.SearcherService_S }) } - ctx, cancel, matchStream := newLimitedStream(stream.Context(), int(req.PatternInfo.Limit), onMatches) + ctx, cancel, matchStream := newLimitedStream(stream.Context(), int(p.PatternInfo.Limit), onMatches) defer cancel() - err := s.Service.search(ctx, &unmarshaledReq, matchStream) + err := s.Service.search(ctx, &p, matchStream) if err != nil { return convertToGRPCError(ctx, err) } diff --git a/cmd/searcher/internal/search/search_test.go b/cmd/searcher/internal/search/search_test.go index 06d6dcdfb8a..3c9ea6a8586 100644 --- a/cmd/searcher/internal/search/search_test.go +++ b/cmd/searcher/internal/search/search_test.go @@ -4,10 +4,10 @@ import ( "archive/tar" "bytes" "context" - "encoding/json" "io" "net/http" "net/http/httptest" + "net/url" "os" "os/exec" "runtime" @@ -22,15 +22,19 @@ import ( "github.com/sourcegraph/log/logtest" "github.com/sourcegraph/zoekt" + "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/gitserver" + internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc" + "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" + "github.com/sourcegraph/sourcegraph/internal/search/backend" + proto "github.com/sourcegraph/sourcegraph/internal/searcher/v1" + v1 "github.com/sourcegraph/sourcegraph/internal/searcher/v1" + "github.com/sourcegraph/sourcegraph/lib/errors" "github.com/sourcegraph/sourcegraph/cmd/searcher/internal/search" "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/observation" - "github.com/sourcegraph/sourcegraph/internal/search/backend" - "github.com/sourcegraph/sourcegraph/internal/search/searcher" - "github.com/sourcegraph/sourcegraph/lib/errors" ) type fileType int @@ -71,23 +75,27 @@ func main() { arg protocol.PatternInfo contextLines int32 want autogold.Value - }{{ - arg: protocol.PatternInfo{Pattern: "foo"}, - want: autogold.Expect(""), - }, { - arg: protocol.PatternInfo{Pattern: "World", IsCaseSensitive: true}, - want: autogold.Expect("README.md:1:1:\n# Hello World\n"), - }, { - arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, - want: autogold.Expect(`README.md:3:3: + }{ + { + arg: protocol.PatternInfo{Pattern: "foo"}, + want: autogold.Expect(""), + }, + { + arg: protocol.PatternInfo{Pattern: "World", IsCaseSensitive: true}, + want: autogold.Expect("README.md:1:1:\n# Hello World\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, + want: autogold.Expect(`README.md:3:3: Hello world example in go main.go:6:6: fmt.Println("Hello world") `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, - contextLines: 1, - want: autogold.Expect(`README.md:2:3: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, + contextLines: 1, + want: autogold.Expect(`README.md:2:3: Hello world example in go main.go:5:7: @@ -95,10 +103,11 @@ func main() { fmt.Println("Hello world") } `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, - contextLines: 2, - want: autogold.Expect(`README.md:1:3: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, + contextLines: 2, + want: autogold.Expect(`README.md:1:3: # Hello World Hello world example in go @@ -108,10 +117,11 @@ func main() { fmt.Println("Hello world") } `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, - contextLines: 999, - want: autogold.Expect(`README.md:1:3: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IsCaseSensitive: true}, + contextLines: 999, + want: autogold.Expect(`README.md:1:3: # Hello World Hello world example in go @@ -124,148 +134,174 @@ func main() { fmt.Println("Hello world") } `), - }, { - arg: protocol.PatternInfo{Pattern: "world"}, - want: autogold.Expect(`README.md:1:1: + }, + { + arg: protocol.PatternInfo{Pattern: "world"}, + want: autogold.Expect(`README.md:1:1: # Hello World README.md:3:3: Hello world example in go main.go:6:6: fmt.Println("Hello world") `), - }, { - arg: protocol.PatternInfo{Pattern: "func.*main"}, - want: autogold.Expect(""), - }, { - arg: protocol.PatternInfo{Pattern: "func.*main", IsRegExp: true}, - want: autogold.Expect("main.go:5:5:\nfunc main() {\n"), - }, { - // https://github.com/sourcegraph/sourcegraph/issues/8155 - arg: protocol.PatternInfo{Pattern: "^func", IsRegExp: true}, - want: autogold.Expect("main.go:5:5:\nfunc main() {\n"), - }, { - arg: protocol.PatternInfo{Pattern: "^FuNc", IsRegExp: true}, - want: autogold.Expect("main.go:5:5:\nfunc main() {\n"), - }, { - // Ensure we handle CaseInsensitive regexp searches with - // special uppercase chars in pattern. - arg: protocol.PatternInfo{Pattern: `printL\B`, IsRegExp: true}, - want: autogold.Expect(`main.go:6:6: + }, + { + arg: protocol.PatternInfo{Pattern: "func.*main"}, + want: autogold.Expect(""), + }, + { + arg: protocol.PatternInfo{Pattern: "func.*main", IsRegExp: true}, + want: autogold.Expect("main.go:5:5:\nfunc main() {\n"), + }, + { + // https://github.com/sourcegraph/sourcegraph/issues/8155 + arg: protocol.PatternInfo{Pattern: "^func", IsRegExp: true}, + want: autogold.Expect("main.go:5:5:\nfunc main() {\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "^FuNc", IsRegExp: true}, + want: autogold.Expect("main.go:5:5:\nfunc main() {\n"), + }, + { + // Ensure we handle CaseInsensitive regexp searches with + // special uppercase chars in pattern. + arg: protocol.PatternInfo{Pattern: `printL\B`, IsRegExp: true}, + want: autogold.Expect(`main.go:6:6: fmt.Println("Hello world") `), - }, { - arg: protocol.PatternInfo{Pattern: "world", ExcludePattern: "README.md"}, - want: autogold.Expect(`main.go:6:6: + }, + { + arg: protocol.PatternInfo{Pattern: "world", ExcludePattern: "README.md"}, + want: autogold.Expect(`main.go:6:6: fmt.Println("Hello world") `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.md$`}}, - want: autogold.Expect(`README.md:1:1: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.md$`}}, + want: autogold.Expect(`README.md:1:1: # Hello World README.md:3:3: Hello world example in go `), - }, { - arg: protocol.PatternInfo{Pattern: "w", IncludePatterns: []string{`\.(md|txt)$`, `\.txt$`}}, - want: autogold.Expect("abc.txt:1:1:\nw\n"), - }, { - arg: protocol.PatternInfo{Pattern: "world", ExcludePattern: "README\\.md"}, - want: autogold.Expect(`main.go:6:6: + }, + { + arg: protocol.PatternInfo{Pattern: "w", IncludePatterns: []string{`\.(md|txt)$`, `\.txt$`}}, + want: autogold.Expect("abc.txt:1:1:\nw\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "world", ExcludePattern: "README\\.md"}, + want: autogold.Expect(`main.go:6:6: fmt.Println("Hello world") `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{"\\.md"}}, - want: autogold.Expect(`README.md:1:1: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{"\\.md"}}, + want: autogold.Expect(`README.md:1:1: # Hello World README.md:3:3: Hello world example in go `), - }, { - arg: protocol.PatternInfo{Pattern: "w", IncludePatterns: []string{"\\.(md|txt)", "README"}}, - want: autogold.Expect(`README.md:1:1: + }, + { + arg: protocol.PatternInfo{Pattern: "w", IncludePatterns: []string{"\\.(md|txt)", "README"}}, + want: autogold.Expect(`README.md:1:1: # Hello World README.md:3:3: Hello world example in go `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)$`}, PathPatternsAreCaseSensitive: true}, - want: autogold.Expect(`main.go:6:6: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)$`}, PathPatternsAreCaseSensitive: true}, + want: autogold.Expect(`main.go:6:6: fmt.Println("Hello world") `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)$`}, PathPatternsAreCaseSensitive: true}, - contextLines: 1, - want: autogold.Expect(`main.go:5:7: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)$`}, PathPatternsAreCaseSensitive: true}, + contextLines: 1, + want: autogold.Expect(`main.go:5:7: func main() { fmt.Println("Hello world") } `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)$`}, PathPatternsAreCaseSensitive: true}, - contextLines: 2, - want: autogold.Expect(`main.go:4:7: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)$`}, PathPatternsAreCaseSensitive: true}, + contextLines: 2, + want: autogold.Expect(`main.go:4:7: func main() { fmt.Println("Hello world") } `), - }, { - arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)`}, PathPatternsAreCaseSensitive: true}, - want: autogold.Expect(`main.go:6:6: + }, + { + arg: protocol.PatternInfo{Pattern: "world", IncludePatterns: []string{`\.(MD|go)`}, PathPatternsAreCaseSensitive: true}, + want: autogold.Expect(`main.go:6:6: fmt.Println("Hello world") `), - }, { - arg: protocol.PatternInfo{Pattern: "doesnotmatch"}, - want: autogold.Expect(""), - }, { - arg: protocol.PatternInfo{Pattern: "", IsRegExp: false, IncludePatterns: []string{"\\.png"}, PatternMatchesPath: true}, - want: autogold.Expect("milton.png\n"), - }, { - arg: protocol.PatternInfo{Pattern: "package main\n\nimport \"fmt\"", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect(`main.go:1:3: + }, + { + arg: protocol.PatternInfo{Pattern: "doesnotmatch"}, + want: autogold.Expect(""), + }, + { + arg: protocol.PatternInfo{Pattern: "", IsRegExp: false, IncludePatterns: []string{"\\.png"}, PatternMatchesPath: true}, + want: autogold.Expect("milton.png\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "package main\n\nimport \"fmt\"", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect(`main.go:1:3: package main import "fmt" `), - }, { - arg: protocol.PatternInfo{Pattern: "package main\n\\s*import \"fmt\"", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect(`main.go:1:3: + }, + { + arg: protocol.PatternInfo{Pattern: "package main\n\\s*import \"fmt\"", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect(`main.go:1:3: package main import "fmt" `), - }, { - arg: protocol.PatternInfo{Pattern: "package main\n", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect("main.go:1:2:\npackage main\n\n"), - }, { - arg: protocol.PatternInfo{Pattern: "package main\n\\s*", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect(`main.go:1:3: + }, + { + arg: protocol.PatternInfo{Pattern: "package main\n", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect("main.go:1:2:\npackage main\n\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "package main\n\\s*", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect(`main.go:1:3: package main import "fmt" `), - }, { - arg: protocol.PatternInfo{Pattern: "\nfunc", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect("main.go:4:5:\n\nfunc main() {\n"), - }, { - arg: protocol.PatternInfo{Pattern: "\n\\s*func", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect(`main.go:3:5: + }, + { + arg: protocol.PatternInfo{Pattern: "\nfunc", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect("main.go:4:5:\n\nfunc main() {\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "\n\\s*func", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect(`main.go:3:5: import "fmt" func main() { `), - }, { - arg: protocol.PatternInfo{Pattern: "package main\n\nimport \"fmt\"\n\nfunc main\\(\\) {", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect(`main.go:1:5: + }, + { + arg: protocol.PatternInfo{Pattern: "package main\n\nimport \"fmt\"\n\nfunc main\\(\\) {", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect(`main.go:1:5: package main import "fmt" func main() { `), - }, { - arg: protocol.PatternInfo{Pattern: "\n", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect(`README.md:1:3: + }, + { + arg: protocol.PatternInfo{Pattern: "\n", IsCaseSensitive: false, IsRegExp: true, PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect(`README.md:1:3: # Hello World Hello world example in go @@ -279,9 +315,10 @@ fmt.Println("Hello world") } `), - }, { - arg: protocol.PatternInfo{Pattern: "^$", IsRegExp: true}, - want: autogold.Expect(`README.md:2:2: + }, + { + arg: protocol.PatternInfo{Pattern: "^$", IsRegExp: true}, + want: autogold.Expect(`README.md:2:2: main.go:2:2: @@ -292,64 +329,73 @@ main.go:8:8: milton.png:1:1: `), - }, { - arg: protocol.PatternInfo{ - Pattern: "filename contains regex metachars", - IncludePatterns: []string{regexp.QuoteMeta("file++.plus")}, - IsStructuralPat: true, - IsRegExp: true, // To test for a regression, imply that IsStructuralPat takes precedence. }, - want: autogold.Expect(`file++.plus:1:1: + { + arg: protocol.PatternInfo{ + Pattern: "filename contains regex metachars", + IncludePatterns: []string{regexp.QuoteMeta("file++.plus")}, + IsStructuralPat: true, + IsRegExp: true, // To test for a regression, imply that IsStructuralPat takes precedence. + }, + want: autogold.Expect(`file++.plus:1:1: filename contains regex metachars `), - }, { - arg: protocol.PatternInfo{Pattern: "World", IsNegated: true}, - want: autogold.Expect(`abc.txt + }, + { + arg: protocol.PatternInfo{Pattern: "World", IsNegated: true}, + want: autogold.Expect(`abc.txt file++.plus milton.png nonutf8.txt symlink `), - }, { - arg: protocol.PatternInfo{Pattern: "World", IsCaseSensitive: true, IsNegated: true}, - want: autogold.Expect(`abc.txt + }, + { + arg: protocol.PatternInfo{Pattern: "World", IsCaseSensitive: true, IsNegated: true}, + want: autogold.Expect(`abc.txt file++.plus main.go milton.png nonutf8.txt symlink `), - }, { - arg: protocol.PatternInfo{Pattern: "fmt", IsNegated: true}, - want: autogold.Expect(`README.md + }, + { + arg: protocol.PatternInfo{Pattern: "fmt", IsNegated: true}, + want: autogold.Expect(`README.md abc.txt file++.plus milton.png nonutf8.txt symlink `), - }, { - arg: protocol.PatternInfo{Pattern: "go", IsNegated: true, PatternMatchesPath: true, ExcludePattern: "\\.txt"}, - want: autogold.Expect(`README.md + }, + { + arg: protocol.PatternInfo{Pattern: "go", IsNegated: true, PatternMatchesPath: true, ExcludePattern: "\\.txt"}, + want: autogold.Expect(`README.md file++.plus milton.png symlink `), - }, { - arg: protocol.PatternInfo{Pattern: "abc", PatternMatchesPath: true, PatternMatchesContent: true}, - want: autogold.Expect("abc.txt\nsymlink:1:1:\nabc.txt\n"), - }, { - arg: protocol.PatternInfo{Pattern: "abc", PatternMatchesPath: false, PatternMatchesContent: true}, - want: autogold.Expect("symlink:1:1:\nabc.txt\n"), - }, { - arg: protocol.PatternInfo{Pattern: "abc", PatternMatchesPath: true, PatternMatchesContent: false}, - want: autogold.Expect("abc.txt\n"), - }, { - arg: protocol.PatternInfo{Pattern: "utf8", PatternMatchesPath: false, PatternMatchesContent: true}, - want: autogold.Expect(`nonutf8.txt:1:1: + }, + { + arg: protocol.PatternInfo{Pattern: "abc", PatternMatchesPath: true, PatternMatchesContent: true}, + want: autogold.Expect("abc.txt\nsymlink:1:1:\nabc.txt\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "abc", PatternMatchesPath: false, PatternMatchesContent: true}, + want: autogold.Expect("symlink:1:1:\nabc.txt\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "abc", PatternMatchesPath: true, PatternMatchesContent: false}, + want: autogold.Expect("abc.txt\n"), + }, + { + arg: protocol.PatternInfo{Pattern: "utf8", PatternMatchesPath: false, PatternMatchesContent: true}, + want: autogold.Expect(`nonutf8.txt:1:1: file contains invalid utf8 � characters `), - }} + }} zoektURL := newZoekt(t, &zoekt.Repository{}, nil) s := newStore(t, files) @@ -358,12 +404,29 @@ file contains invalid utf8 � characters return hdr.Name == "ignore.me" }, nil } - ts := httptest.NewServer(&search.Service{ + service := &search.Service{ Store: s, Log: s.Log, Indexed: backend.ZoektDial(zoektURL), + } + + grpcServer := defaults.NewServer(logtest.Scoped(t)) + proto.RegisterSearcherServiceServer(grpcServer, &search.Server{ + Service: service, + }) + + handler := internalgrpc.MultiplexHandlers(grpcServer, http.HandlerFunc(http.NotFound)) + + ts := httptest.NewServer(handler) + + t.Cleanup(func() { + ts.Close() + }) + + conf.Mock(&conf.Unified{}) + t.Cleanup(func() { + conf.Mock(nil) }) - defer ts.Close() for i, test := range cases { t.Run(strconv.Itoa(i), func(t *testing.T) { @@ -379,7 +442,7 @@ file contains invalid utf8 � characters FetchTimeout: fetchTimeoutForCI(t), NumContextLines: test.contextLines, } - m, err := doSearch(ts.URL, &req) + m, err := doSearch(t, ts.URL, &req) if err != nil { t.Fatalf("%s failed: %s", test.arg.String(), err) } @@ -517,62 +580,68 @@ func TestSearch_badrequest(t *testing.T) { }, } + zoektURL := newZoekt(t, &zoekt.Repository{}, nil) store := newStore(t, nil) - ts := httptest.NewServer(&search.Service{ - Store: store, - Log: store.Log, - }) - defer ts.Close() + service := &search.Service{ + Store: store, + Log: store.Log, + Indexed: backend.ZoektDial(zoektURL), + } - for _, p := range cases { - p.PatternInfo.PatternMatchesContent = true - _, err := doSearch(ts.URL, &p) - if err == nil { - t.Fatalf("%v expected to fail", p) - } + grpcServer := defaults.NewServer(logtest.Scoped(t)) + proto.RegisterSearcherServiceServer(grpcServer, &search.Server{ + Service: service, + }) + + handler := internalgrpc.MultiplexHandlers(grpcServer, http.HandlerFunc(http.NotFound)) + + ts := httptest.NewServer(handler) + t.Cleanup(func() { + ts.Close() + }) + + for i, p := range cases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + p.PatternInfo.PatternMatchesContent = true + _, err := doSearch(t, ts.URL, &p) + if err == nil { + t.Fatalf("%v expected to fail", p) + } + }) } } -func doSearch(u string, p *protocol.Request) ([]protocol.FileMatch, error) { - reqBody, err := json.Marshal(p) +func doSearch(t *testing.T, urlString string, p *protocol.Request) ([]protocol.FileMatch, error) { + u, err := url.Parse(urlString) if err != nil { return nil, err } - resp, err := http.Post(u, "application/json", bytes.NewReader(reqBody)) + conn, err := defaults.Dial(u.Host, logtest.Scoped(t)) + if err != nil { + return nil, err + } + c := v1.NewSearcherServiceClient(conn) + + cc, err := c.Search(context.Background(), p.ToProto()) if err != nil { return nil, err } - if resp.StatusCode != 200 { - body, err := io.ReadAll(resp.Body) + var matches []protocol.FileMatch + for { + msg, err := cc.Recv() if err != nil { + if err == io.EOF { + return matches, nil + } return nil, err } - return nil, errors.Errorf("non-200 response: code=%d body=%s", resp.StatusCode, string(body)) + if m := msg.GetFileMatch(); m != nil { + var fm protocol.FileMatch + fm.FromProto(m) + matches = append(matches, fm) + } } - - var ed searcher.EventDone - var matches []protocol.FileMatch - dec := searcher.StreamDecoder{ - OnMatches: func(newMatches []*protocol.FileMatch) { - for _, match := range newMatches { - matches = append(matches, *match) - } - }, - OnDone: func(e searcher.EventDone) { - ed = e - }, - OnUnknown: func(event []byte, _ []byte) { - panic("unknown event") - }, - } - if err := dec.ReadAll(resp.Body); err != nil { - return nil, err - } - if ed.Error != "" { - return nil, errors.New(ed.Error) - } - return matches, err } func newStore(t *testing.T, files map[string]struct { diff --git a/cmd/searcher/internal/search/store_test.go b/cmd/searcher/internal/search/store_test.go index 72570a8c87b..47ece73b620 100644 --- a/cmd/searcher/internal/search/store_test.go +++ b/cmd/searcher/internal/search/store_test.go @@ -6,7 +6,6 @@ import ( "bytes" "context" "io" - "net" "os" "os/exec" "path/filepath" @@ -317,12 +316,3 @@ func emptyTar(t *testing.T) io.ReadCloser { } return io.NopCloser(bytes.NewReader(buf.Bytes())) } - -func TestIsNetOpError(t *testing.T) { - if !isNetOpError(&net.OpError{}) { - t.Fatal("should be net.OpError") - } - if isNetOpError(errors.New("hi")) { - t.Fatal("should not be net.OpError") - } -} diff --git a/cmd/searcher/shared/BUILD.bazel b/cmd/searcher/shared/BUILD.bazel index d636a8224eb..737f0cd58e3 100644 --- a/cmd/searcher/shared/BUILD.bazel +++ b/cmd/searcher/shared/BUILD.bazel @@ -13,7 +13,6 @@ go_library( "//cmd/searcher/internal/search", "//internal/actor", "//internal/api", - "//internal/conf", "//internal/debugserver", "//internal/env", "//internal/gitserver", @@ -21,12 +20,10 @@ go_library( "//internal/goroutine", "//internal/grpc", "//internal/grpc/defaults", - "//internal/instrumentation", "//internal/observation", "//internal/search", "//internal/searcher/v1:searcher", "//internal/service", - "//internal/trace", "//lib/errors", "@com_github_keegancsmith_tmpfriend//:tmpfriend", "@com_github_sourcegraph_log//:log", diff --git a/cmd/searcher/shared/shared.go b/cmd/searcher/shared/shared.go index 079adb38ae3..f5a3c1a8187 100644 --- a/cmd/searcher/shared/shared.go +++ b/cmd/searcher/shared/shared.go @@ -21,19 +21,16 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/searcher/internal/search" "github.com/sourcegraph/sourcegraph/internal/actor" "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/env" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" "github.com/sourcegraph/sourcegraph/internal/goroutine" internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/instrumentation" "github.com/sourcegraph/sourcegraph/internal/observation" sharedsearch "github.com/sourcegraph/sourcegraph/internal/search" proto "github.com/sourcegraph/sourcegraph/internal/searcher/v1" "github.com/sourcegraph/sourcegraph/internal/service" - "github.com/sourcegraph/sourcegraph/internal/trace" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -175,11 +172,6 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi } sService.Store.Start() - // Set up handler middleware - handler := actor.HTTPMiddleware(logger, sService) - handler = trace.HTTPMiddleware(logger, handler, conf.DefaultClient()) - handler = instrumentation.HTTPMiddleware("", handler) - ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -203,7 +195,7 @@ func Start(ctx context.Context, observationCtx *observation.Context, ready servi _, _ = w.Write([]byte("ok")) return } - handler.ServeHTTP(w, r) + http.NotFoundHandler().ServeHTTP(w, r) })), } diff --git a/cmd/symbols/internal/api/BUILD.bazel b/cmd/symbols/internal/api/BUILD.bazel index ee3f39a5915..ecd67da06a8 100644 --- a/cmd/symbols/internal/api/BUILD.bazel +++ b/cmd/symbols/internal/api/BUILD.bazel @@ -19,7 +19,6 @@ go_library( "//cmd/symbols/squirrel", "//cmd/symbols/types", "//internal/database", - "//internal/env", "//internal/grpc", "//internal/grpc/chunk", "//internal/grpc/defaults", @@ -58,7 +57,6 @@ go_test( "//internal/endpoint", "//internal/gitserver/gitdomain", "//internal/grpc/defaults", - "//internal/httpcli", "//internal/observation", "//internal/search", "//internal/search/result", diff --git a/cmd/symbols/internal/api/handler.go b/cmd/symbols/internal/api/handler.go index 631e4b31f49..2e443ab1fe2 100644 --- a/cmd/symbols/internal/api/handler.go +++ b/cmd/symbols/internal/api/handler.go @@ -2,7 +2,6 @@ package api import ( "context" - "encoding/json" "fmt" "net/http" @@ -15,7 +14,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/search/result" proto "github.com/sourcegraph/sourcegraph/internal/symbols/v1" internaltypes "github.com/sourcegraph/sourcegraph/internal/types" - "github.com/sourcegraph/sourcegraph/lib/errors" ) const maxNumSymbolResults = 500 @@ -62,7 +60,6 @@ func NewHandler( handleStatus func(http.ResponseWriter, *http.Request), ctagsBinary string, ) http.Handler { - searchFuncWrapper := func(ctx context.Context, args search.SymbolsParameters) (result.Symbols, error) { // Massage the arguments to ensure that First is set to a reasonable value. if args.First < 0 || args.First > maxNumSymbolResults { @@ -87,10 +84,8 @@ func NewHandler( // Initialize the legacy JSON API server mux := http.NewServeMux() - mux.HandleFunc("/search", handleSearchWith(jsonLogger, searchFuncWrapper)) mux.HandleFunc("/healthz", handleHealthCheck(jsonLogger)) - addHandlers(mux, searchFunc, readFileFunc) if handleStatus != nil { mux.HandleFunc("/status", handleStatus) } @@ -98,40 +93,6 @@ func NewHandler( return internalgrpc.MultiplexHandlers(grpcServer, mux) } -func handleSearchWith(l logger.Logger, searchFunc types.SearchFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var args search.SymbolsParameters - if err := json.NewDecoder(r.Body).Decode(&args); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - resultSymbols, err := searchFunc(r.Context(), args) - if err != nil { - // Ignore reporting errors where client disconnected - if r.Context().Err() == context.Canceled && errors.Is(err, context.Canceled) { - return - } - - argsStr := fmt.Sprintf("%+v", args) - - l.Error("symbol search failed", - logger.String("arguments", argsStr), - logger.Error(err), - ) - - if err := json.NewEncoder(w).Encode(search.SymbolsResponse{Err: err.Error()}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - if err := json.NewEncoder(w).Encode(search.SymbolsResponse{Symbols: resultSymbols}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - } -} - func handleHealthCheck(l logger.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/cmd/symbols/internal/api/handler_cgo.go b/cmd/symbols/internal/api/handler_cgo.go index bba062d93a6..e3cb360c1cd 100644 --- a/cmd/symbols/internal/api/handler_cgo.go +++ b/cmd/symbols/internal/api/handler_cgo.go @@ -4,28 +4,17 @@ package api import ( "context" - "net/http" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/sourcegraph/sourcegraph/cmd/symbols/squirrel" - "github.com/sourcegraph/sourcegraph/cmd/symbols/types" "github.com/sourcegraph/sourcegraph/internal/grpc/chunk" proto "github.com/sourcegraph/sourcegraph/internal/symbols/v1" internaltypes "github.com/sourcegraph/sourcegraph/internal/types" "github.com/sourcegraph/sourcegraph/lib/errors" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) -// addHandlers adds handlers that require cgo. -func addHandlers( - mux *http.ServeMux, - searchFunc types.SearchFunc, - readFileFunc func(context.Context, internaltypes.RepoCommitPath) ([]byte, error), -) { - mux.HandleFunc("/localCodeIntel", squirrel.LocalCodeIntelHandler(readFileFunc)) - mux.HandleFunc("/symbolInfo", squirrel.NewSymbolInfoHandler(searchFunc, readFileFunc)) -} - func convertSquirrelErrorToGrpcError(err error) *status.Status { if errors.Is(err, squirrel.UnrecognizedFileExtensionError) { return status.New(codes.InvalidArgument, err.Error()) diff --git a/cmd/symbols/internal/api/handler_nocgo.go b/cmd/symbols/internal/api/handler_nocgo.go index fc2b6d2f448..8cbe1fc06c3 100644 --- a/cmd/symbols/internal/api/handler_nocgo.go +++ b/cmd/symbols/internal/api/handler_nocgo.go @@ -7,29 +7,11 @@ import ( "encoding/json" "net/http" - "github.com/sourcegraph/sourcegraph/cmd/symbols/types" - "github.com/sourcegraph/sourcegraph/internal/env" - proto "github.com/sourcegraph/sourcegraph/internal/symbols/v1" - internaltypes "github.com/sourcegraph/sourcegraph/internal/types" "google.golang.org/grpc/status" + + proto "github.com/sourcegraph/sourcegraph/internal/symbols/v1" ) -// addHandlers adds handlers that do not require cgo, which speeds up compile times but omits local -// code intelligence features. This non-cgo variant must only be used for development. Release -// builds of Sourcegraph must be built with cgo, or else they'll miss critical features. -func addHandlers( - mux *http.ServeMux, - searchFunc types.SearchFunc, - readFileFunc func(context.Context, internaltypes.RepoCommitPath) ([]byte, error), -) { - if !env.InsecureDev { - panic("must build with cgo (non-cgo variant is only for local dev)") - } - - mux.HandleFunc("/localCodeIntel", jsonResponseHandler(internaltypes.LocalCodeIntelPayload{Symbols: []internaltypes.Symbol{}})) - mux.HandleFunc("/symbolInfo", jsonResponseHandler(internaltypes.SymbolInfo{})) -} - func jsonResponseHandler(v any) http.HandlerFunc { data, _ := json.Marshal(v) return func(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/symbols/internal/api/handler_test.go b/cmd/symbols/internal/api/handler_test.go index 1ebb55992ea..4043d0cd130 100644 --- a/cmd/symbols/internal/api/handler_test.go +++ b/cmd/symbols/internal/api/handler_test.go @@ -22,7 +22,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/diskcache" "github.com/sourcegraph/sourcegraph/internal/endpoint" internalgrpc "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/result" @@ -81,7 +80,6 @@ func TestHandler(t *testing.T) { client := symbolsclient.Client{ Endpoints: endpoint.Static(server.URL), GRPCConnectionCache: connectionCache, - HTTPClient: httpcli.InternalDoer, } x := result.Symbol{Name: "x", Path: "a.js", Line: 0, Character: 4} diff --git a/cmd/symbols/squirrel/BUILD.bazel b/cmd/symbols/squirrel/BUILD.bazel index 4f06eb9ab8f..e0ce9652d1d 100644 --- a/cmd/symbols/squirrel/BUILD.bazel +++ b/cmd/symbols/squirrel/BUILD.bazel @@ -6,7 +6,6 @@ go_library( srcs = [ "breadcrumbs.go", "hover.go", - "http_handlers.go", "lang_java.go", "lang_python.go", "lang_starlark.go", @@ -28,7 +27,6 @@ go_library( "//lib/errors", "@com_github_fatih_color//:color", "@com_github_grafana_regexp//:regexp", - "@com_github_inconshreveable_log15//:log15", "@com_github_smacker_go_tree_sitter//:go-tree-sitter", "@com_github_smacker_go_tree_sitter//cpp", "@com_github_smacker_go_tree_sitter//csharp", diff --git a/cmd/symbols/squirrel/http_handlers.go b/cmd/symbols/squirrel/http_handlers.go deleted file mode 100644 index 838d98fa153..00000000000 --- a/cmd/symbols/squirrel/http_handlers.go +++ /dev/null @@ -1,128 +0,0 @@ -package squirrel - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "strings" - - "github.com/inconshreveable/log15" //nolint:logging // TODO move all logging to sourcegraph/log - - symbolsTypes "github.com/sourcegraph/sourcegraph/cmd/symbols/types" - "github.com/sourcegraph/sourcegraph/internal/types" - "github.com/sourcegraph/sourcegraph/lib/errors" -) - -// LocalCodeIntelHandler responds to /localCodeIntel -func LocalCodeIntelHandler(readFile readFileFunc) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - // Read the args from the request body. - body, err := io.ReadAll(r.Body) - if err != nil { - log15.Error("failed to read request body", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - var args types.RepoCommitPath - if err := json.NewDecoder(bytes.NewReader(body)).Decode(&args); err != nil { - log15.Error("failed to decode request body", "err", err, "body", string(body)) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - squirrel := New(readFile, nil) - defer squirrel.Close() - - // Compute the local code intel payload. - payload, err := squirrel.LocalCodeIntel(r.Context(), args) - if payload != nil && os.Getenv("SQUIRREL_DEBUG") == "true" { - debugStringBuilder := &strings.Builder{} - fmt.Fprintln(debugStringBuilder, "👉 /localCodeIntel repo:", args.Repo, "commit:", args.Commit, "path:", args.Path) - contents, err := readFile(r.Context(), args) - if err != nil { - log15.Error("failed to read file from gitserver", "err", err) - } else { - prettyPrintLocalCodeIntelPayload(debugStringBuilder, *payload, string(contents)) - fmt.Fprintln(debugStringBuilder, "✅ /localCodeIntel repo:", args.Repo, "commit:", args.Commit, "path:", args.Path) - - fmt.Println(" ") - fmt.Println(bracket(debugStringBuilder.String())) - fmt.Println(" ") - } - } - if err != nil { - _ = json.NewEncoder(w).Encode(nil) - - // Log the error if it's not an unrecognized file extension or unsupported language error. - if !errors.Is(err, UnrecognizedFileExtensionError) && !errors.Is(err, UnsupportedLanguageError) { - log15.Error("failed to generate local code intel payload", "err", err) - } - - return - } - - // Write the response. - w.Header().Set("Content-Type", "application/json") - err = json.NewEncoder(w).Encode(payload) - if err != nil { - log15.Error("failed to write response: %s", "error", err) - http.Error(w, fmt.Sprintf("failed to generate local code intel payload: %s", err), http.StatusInternalServerError) - return - } - } -} - -// NewSymbolInfoHandler responds to /symbolInfo -func NewSymbolInfoHandler(symbolSearch symbolsTypes.SearchFunc, readFile readFileFunc) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - // Read the args from the request body. - body, err := io.ReadAll(r.Body) - if err != nil { - log15.Error("failed to read request body", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - var args types.RepoCommitPathPoint - if err := json.NewDecoder(bytes.NewReader(body)).Decode(&args); err != nil { - log15.Error("failed to decode request body", "err", err, "body", string(body)) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Find the symbol. - squirrel := New(readFile, symbolSearch) - defer squirrel.Close() - result, err := squirrel.SymbolInfo(r.Context(), args) - if os.Getenv("SQUIRREL_DEBUG") == "true" { - debugStringBuilder := &strings.Builder{} - fmt.Fprintln(debugStringBuilder, "👉 /symbolInfo repo:", args.Repo, "commit:", args.Commit, "path:", args.Path, "row:", args.Row, "column:", args.Column) - squirrel.breadcrumbs.pretty(debugStringBuilder, readFile) - if result == nil { - fmt.Fprintln(debugStringBuilder, "❌ no definition found") - } else { - fmt.Fprintln(debugStringBuilder, "✅ /symbolInfo", *result) - } - - fmt.Println(" ") - fmt.Println(bracket(debugStringBuilder.String())) - fmt.Println(" ") - } - if err != nil { - _ = json.NewEncoder(w).Encode(nil) - log15.Error("failed to get definition", "err", err) - return - } - - // Write the response. - w.Header().Set("Content-Type", "application/json") - err = json.NewEncoder(w).Encode(result) - if err != nil { - log15.Error("failed to write response: %s", "error", err) - http.Error(w, fmt.Sprintf("failed to get definition: %s", err), http.StatusInternalServerError) - return - } - } -} diff --git a/cmd/worker/internal/embeddings/repo/BUILD.bazel b/cmd/worker/internal/embeddings/repo/BUILD.bazel index 82f3d640563..56a20b09681 100644 --- a/cmd/worker/internal/embeddings/repo/BUILD.bazel +++ b/cmd/worker/internal/embeddings/repo/BUILD.bazel @@ -4,7 +4,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "repo", srcs = [ - "document_ranks.go", "handler.go", "janitor.go", "scheduler.go", @@ -19,9 +18,8 @@ go_library( "//cmd/worker/shared/init/db", "//internal/actor", "//internal/api", - "//internal/api/internalapi", "//internal/codeintel/context", - "//internal/codeintel/types", + "//internal/codeintel/ranking", "//internal/conf", "//internal/conf/conftypes", "//internal/database", @@ -33,7 +31,6 @@ go_library( "//internal/featureflag", "//internal/gitserver", "//internal/goroutine", - "//internal/httpcli", "//internal/observation", "//internal/paths", "//internal/uploadstore", diff --git a/cmd/worker/internal/embeddings/repo/document_ranks.go b/cmd/worker/internal/embeddings/repo/document_ranks.go deleted file mode 100644 index 89f5765ee0a..00000000000 --- a/cmd/worker/internal/embeddings/repo/document_ranks.go +++ /dev/null @@ -1,61 +0,0 @@ -package repo - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/url" - "strings" - - "github.com/sourcegraph/sourcegraph/internal/api/internalapi" - "github.com/sourcegraph/sourcegraph/internal/codeintel/types" - "github.com/sourcegraph/sourcegraph/internal/httpcli" - "github.com/sourcegraph/sourcegraph/lib/errors" -) - -func getDocumentRanks(ctx context.Context, repoName string) (types.RepoPathRanks, error) { - root, err := url.Parse(internalapi.Client.URL) - if err != nil { - return types.RepoPathRanks{}, err - } - u := root.ResolveReference(&url.URL{ - Path: "/.internal/ranks/" + strings.Trim(repoName, "/") + "/documents", - }) - - req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) - if err != nil { - return types.RepoPathRanks{}, err - } - - resp, err := httpcli.InternalDoer.Do(req) - if err != nil { - return types.RepoPathRanks{}, err - } - - if resp.StatusCode != http.StatusOK { - b, err := io.ReadAll(io.LimitReader(resp.Body, 1024)) - _ = resp.Body.Close() - if err != nil { - return types.RepoPathRanks{}, err - } - return types.RepoPathRanks{}, &url.Error{ - Op: "Get", - URL: u.String(), - Err: errors.Errorf("%s: %s", resp.Status, string(b)), - } - } - - b, err := io.ReadAll(resp.Body) - if err != nil { - return types.RepoPathRanks{}, err - } - - ranks := types.RepoPathRanks{} - err = json.Unmarshal(b, &ranks) - if err != nil { - return types.RepoPathRanks{}, err - } - - return ranks, nil -} diff --git a/cmd/worker/internal/embeddings/repo/handler.go b/cmd/worker/internal/embeddings/repo/handler.go index ba6996888fa..27f245a39fe 100644 --- a/cmd/worker/internal/embeddings/repo/handler.go +++ b/cmd/worker/internal/embeddings/repo/handler.go @@ -9,6 +9,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/actor" "github.com/sourcegraph/sourcegraph/internal/api" codeintelContext "github.com/sourcegraph/sourcegraph/internal/codeintel/context" + "github.com/sourcegraph/sourcegraph/internal/codeintel/ranking" "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/database" @@ -32,6 +33,7 @@ type handler struct { getQdrantInserter func() (db.VectorInserter, error) contextService embed.ContextService repoEmbeddingJobsStore bgrepo.RepoEmbeddingJobsStore + rankingService *ranking.Service } var _ workerutil.Handler[*bgrepo.RepoEmbeddingJob] = &handler{} @@ -161,7 +163,7 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo. } } - ranks, err := getDocumentRanks(ctx, string(repo.Name)) + ranks, err := h.rankingService.GetDocumentRanks(ctx, repo.Name) if err != nil { return err } diff --git a/cmd/worker/internal/embeddings/repo/worker.go b/cmd/worker/internal/embeddings/repo/worker.go index 02160d3a422..7ce359903d5 100644 --- a/cmd/worker/internal/embeddings/repo/worker.go +++ b/cmd/worker/internal/embeddings/repo/worker.go @@ -8,6 +8,7 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/codeintel" workerdb "github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/db" "github.com/sourcegraph/sourcegraph/internal/actor" + "github.com/sourcegraph/sourcegraph/internal/codeintel/ranking" "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/embeddings" repoembeddingsbg "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo" @@ -68,6 +69,7 @@ func (s *repoEmbeddingJob) Routines(_ context.Context, observationCtx *observati getQdrantInserter, services.ContextService, repoembeddingsbg.NewRepoEmbeddingJobsStore(db), + services.RankingService, ), }, nil } @@ -82,6 +84,7 @@ func newRepoEmbeddingJobWorker( getQdrantInserter func() (vdb.VectorInserter, error), contextService embed.ContextService, repoEmbeddingJobsStore repoembeddingsbg.RepoEmbeddingJobsStore, + rankingService *ranking.Service, ) *workerutil.Worker[*repoembeddingsbg.RepoEmbeddingJob] { handler := &handler{ db: db, @@ -90,6 +93,7 @@ func newRepoEmbeddingJobWorker( getQdrantInserter: getQdrantInserter, contextService: contextService, repoEmbeddingJobsStore: repoEmbeddingJobsStore, + rankingService: rankingService, } return dbworker.NewWorker[*repoembeddingsbg.RepoEmbeddingJob](ctx, workerStore, handler, workerutil.WorkerOptions{ Name: "repo_embedding_job_worker", diff --git a/dev/run-server-image.sh b/dev/run-server-image.sh index 668006caa8a..7058307962d 100755 --- a/dev/run-server-image.sh +++ b/dev/run-server-image.sh @@ -8,7 +8,6 @@ PORT=${PORT:-"7080"} URL="http://localhost:$PORT" DATA=${DATA:-"/tmp/sourcegraph-data"} SOURCEGRAPH_LICENSE_GENERATION_KEY=${SOURCEGRAPH_LICENSE_GENERATION_KEY:-""} -SG_FEATURE_FLAG_GRPC=${SG_FEATURE_FLAG_GRPC:-"true"} DB_STARTUP_TIMEOUT="10s" echo "--- Checking for existing Sourcegraph instance at $URL" @@ -48,7 +47,6 @@ docker run "$@" \ --publish "$PORT":7080 \ -e ALLOW_SINGLE_DOCKER_CODE_INSIGHTS=t \ -e SOURCEGRAPH_LICENSE_GENERATION_KEY="$SOURCEGRAPH_LICENSE_GENERATION_KEY" \ - -e SG_FEATURE_FLAG_GRPC="$SG_FEATURE_FLAG_GRPC" \ -e DB_STARTUP_TIMEOUT="$DB_STARTUP_TIMEOUT" \ -e SOURCEGRAPH_5_1_DB_MIGRATION=true \ --volume "$DATA/config:/etc/sourcegraph" \ diff --git a/doc/admin/observability/alerts.md b/doc/admin/observability/alerts.md index 4adf98b1a35..658bfee57d0 100644 --- a/doc/admin/observability/alerts.md +++ b/doc/admin/observability/alerts.md @@ -707,37 +707,6 @@ Generated query for warning alert: `max((sum by (code) (increase(searcher_servic
-## frontend: internalapi_error_responses - -

internal API error responses every 5m by route

- -**Descriptions** - -- warning frontend: 5%+ internal API error responses every 5m by route for 15m0s - -**Next steps** - -- May not be a substantial issue, check the `frontend` logs for potential causes. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#frontend-internalapi-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_frontend_internalapi_error_responses" -] -``` - -*Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2.."}[5m])) / ignoring (code) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count[5m])) * 100) >= 5)` - -
- -
- ## frontend: 99th_percentile_gitserver_duration

99th percentile successful gitserver query duration over 5m

@@ -1572,43 +1541,6 @@ Generated query for warning alert: `max((sum(src_gitserver_lsremote_queue)) >= 2
-## gitserver: frontend_internal_api_error_responses - -

frontend-internal API error responses every 5m by route

- -**Descriptions** - -- warning gitserver: 2%+ frontend-internal API error responses every 5m by route for 5m0s - -**Next steps** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs gitserver` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs gitserver` for logs indicating request failures to `frontend` or `frontend-internal`. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#gitserver-frontend-internal-api-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_gitserver_frontend_internal_api_error_responses" -] -``` - -*Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2..",job="gitserver"}[5m])) / ignoring (category) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count{job="gitserver"}[5m]))) >= 2)` - -
- -
- ## gitserver: gitserver_site_configuration_duration_since_last_successful_update_by_instance

maximum duration since last successful site configuration update (all "gitserver" instances)

@@ -2383,43 +2315,6 @@ Generated query for critical alert: `max((max(src_codeintel_upload_queued_durati
-## precise-code-intel-worker: frontend_internal_api_error_responses - -

frontend-internal API error responses every 5m by route

- -**Descriptions** - -- warning precise-code-intel-worker: 2%+ frontend-internal API error responses every 5m by route for 5m0s - -**Next steps** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs precise-code-intel-worker` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs precise-code-intel-worker` for logs indicating request failures to `frontend` or `frontend-internal`. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#precise-code-intel-worker-frontend-internal-api-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_precise-code-intel-worker_frontend_internal_api_error_responses" -] -``` - -*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2..",job="precise-code-intel-worker"}[5m])) / ignoring (category) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count{job="precise-code-intel-worker"}[5m]))) >= 2)` - -
- -
- ## precise-code-intel-worker: mean_blocked_seconds_per_conn_request

mean blocked seconds per conn request

@@ -3396,43 +3291,6 @@ Generated query for warning alert: `max((max(src_query_runner_worker_total{job=~
-## worker: frontend_internal_api_error_responses - -

frontend-internal API error responses every 5m by route

- -**Descriptions** - -- warning worker: 2%+ frontend-internal API error responses every 5m by route for 5m0s - -**Next steps** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs worker` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs worker` for logs indicating request failures to `frontend` or `frontend-internal`. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#worker-frontend-internal-api-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_worker_frontend_internal_api_error_responses" -] -``` - -*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2..",job="worker"}[5m])) / ignoring (category) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count{job="worker"}[5m]))) >= 2)` - -
- -
- ## worker: mean_blocked_seconds_per_conn_request

mean blocked seconds per conn request

@@ -4645,43 +4503,6 @@ Generated query for critical alert: `max((max(max_over_time(src_conf_client_time
-## repo-updater: frontend_internal_api_error_responses - -

frontend-internal API error responses every 5m by route

- -**Descriptions** - -- warning repo-updater: 2%+ frontend-internal API error responses every 5m by route for 5m0s - -**Next steps** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs repo-updater` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs repo-updater` for logs indicating request failures to `frontend` or `frontend-internal`. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#repo-updater-frontend-internal-api-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_repo-updater_frontend_internal_api_error_responses" -] -``` - -*Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2..",job="repo-updater"}[5m])) / ignoring (category) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count{job="repo-updater"}[5m]))) >= 2)` - -
- -
- ## repo-updater: mean_blocked_seconds_per_conn_request

mean blocked seconds per conn request

@@ -5164,43 +4985,6 @@ Generated query for critical alert: `max((sum by (app_name, db_name) (increase(s
-## searcher: frontend_internal_api_error_responses - -

frontend-internal API error responses every 5m by route

- -**Descriptions** - -- warning searcher: 2%+ frontend-internal API error responses every 5m by route for 5m0s - -**Next steps** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs searcher` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs searcher` for logs indicating request failures to `frontend` or `frontend-internal`. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#searcher-frontend-internal-api-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_searcher_frontend_internal_api_error_responses" -] -``` - -*Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2..",job="searcher"}[5m])) / ignoring (category) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count{job="searcher"}[5m]))) >= 2)` - -
- -
- ## searcher: container_cpu_usage

container cpu usage total (1m average) across all cores by instance

@@ -5586,43 +5370,6 @@ Generated query for critical alert: `max((sum by (app_name, db_name) (increase(s
-## symbols: frontend_internal_api_error_responses - -

frontend-internal API error responses every 5m by route

- -**Descriptions** - -- warning symbols: 2%+ frontend-internal API error responses every 5m by route for 5m0s - -**Next steps** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs symbols` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs symbols` for logs indicating request failures to `frontend` or `frontend-internal`. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#symbols-frontend-internal-api-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_symbols_frontend_internal_api_error_responses" -] -``` - -*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2..",job="symbols"}[5m])) / ignoring (category) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count{job="symbols"}[5m]))) >= 2)` - -
- -
- ## symbols: container_cpu_usage

container cpu usage total (1m average) across all cores by instance

@@ -8006,43 +7753,6 @@ Generated query for critical alert: `max((sum by (app_name, db_name) (increase(s
-## embeddings: frontend_internal_api_error_responses - -

frontend-internal API error responses every 5m by route

- -**Descriptions** - -- warning embeddings: 2%+ frontend-internal API error responses every 5m by route for 5m0s - -**Next steps** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs embeddings` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs embeddings` for logs indicating request failures to `frontend` or `frontend-internal`. -- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#embeddings-frontend-internal-api-error-responses). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_embeddings_frontend_internal_api_error_responses" -] -``` - -*Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* - -
-Technical details - -Generated query for warning alert: `max((sum by (category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2..",job="embeddings"}[5m])) / ignoring (category) group_left () sum(increase(src_frontend_internal_request_duration_seconds_count{job="embeddings"}[5m]))) >= 2)` - -
- -
- ## embeddings: container_cpu_usage

container cpu usage total (1m average) across all cores by instance

diff --git a/doc/admin/observability/dashboards.md b/doc/admin/observability/dashboards.md index 6a5530f3535..b031fbef0de 100644 --- a/doc/admin/observability/dashboards.md +++ b/doc/admin/observability/dashboards.md @@ -3857,25 +3857,6 @@ Query: `sum by(code) (increase(searcher_service_request_total{code!~"2.."}[5m]))
-#### frontend: internalapi_error_responses - -

Internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#frontend-internalapi-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/frontend/frontend?viewPanel=102602` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* - -
-Technical details - -Query: `sum by(category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2.."}[5m])) / ignoring(code) group_left sum(increase(src_frontend_internal_request_duration_seconds_count[5m])) * 100` - -
- -
- #### frontend: 99th_percentile_gitserver_duration

99th percentile successful gitserver query duration over 5m

@@ -5628,25 +5609,6 @@ Query: `max(src_gitserver_echo_duration_seconds)`
-#### gitserver: frontend_internal_api_error_responses - -

Frontend-internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#gitserver-frontend-internal-api-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/gitserver/gitserver?viewPanel=100081` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* - -
-Technical details - -Query: `sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="gitserver",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="gitserver"}[5m]))` - -
- -
- #### gitserver: src_gitserver_repo_count

Number of repositories on gitserver

@@ -9270,27 +9232,6 @@ Query: `sum by (op)(increase(src_codeintel_uploadstore_errors_total{job=~"^preci
-### Precise Code Intel Worker: Internal service requests - -#### precise-code-intel-worker: frontend_internal_api_error_responses - -

Frontend-internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-frontend-internal-api-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100700` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* - -
-Technical details - -Query: `sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="precise-code-intel-worker",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="precise-code-intel-worker"}[5m]))` - -
- -
- ### Precise Code Intel Worker: Database connections #### precise-code-intel-worker: max_open_conns @@ -9299,7 +9240,7 @@ Query: `sum by (category)(increase(src_frontend_internal_request_duration_second This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100800` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100700` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9318,7 +9259,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_max_open{app_name="precise-c This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100801` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100701` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9337,7 +9278,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_open{app_name="precise-code- This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100810` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100710` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9356,7 +9297,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_in_use{app_name="precise-cod This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100811` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100711` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9375,7 +9316,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_idle{app_name="precise-code- Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-mean-blocked-seconds-per-conn-request) for 2 alerts related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100820` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100720` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9394,7 +9335,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_blocked_seconds{app This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100830` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100730` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9413,7 +9354,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle{app This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100831` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100731` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9432,7 +9373,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_lifetime This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100832` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100732` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -9463,7 +9404,7 @@ value change independent of deployment events (such as an upgrade), it could ind This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100900` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100800` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9482,7 +9423,7 @@ Query: `count by(name) ((time() - container_last_seen{name=~"^precise-code-intel Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-container-cpu-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100901` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100801` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9501,7 +9442,7 @@ Query: `cadvisor_container_cpu_usage_percentage_total{name=~"^precise-code-intel Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-container-memory-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100902` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100802` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9523,7 +9464,7 @@ When extremely high, this can indicate a resource usage problem, or can cause pr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100903` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100803` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9544,7 +9485,7 @@ Query: `sum by(name) (rate(container_fs_reads_total{name=~"^precise-code-intel-w Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-provisioning-container-cpu-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101000` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100900` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9563,7 +9504,7 @@ Query: `quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{na Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-provisioning-container-memory-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101001` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100901` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9582,7 +9523,7 @@ Query: `max_over_time(cadvisor_container_memory_usage_percentage_total{name=~"^p Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-provisioning-container-cpu-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101010` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100910` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9601,7 +9542,7 @@ Query: `max_over_time(cadvisor_container_cpu_usage_percentage_total{name=~"^prec Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-provisioning-container-memory-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101011` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100911` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9623,7 +9564,7 @@ When it occurs frequently, it is an indicator of underprovisioning. Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-container-oomkill-events-total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101012` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=100912` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9646,7 +9587,7 @@ A high value here indicates a possible goroutine leak. Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-go-goroutines) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101100` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101000` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9665,7 +9606,7 @@ Query: `max by(instance) (go_goroutines{job=~".*precise-code-intel-worker"})` Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-go-gc-duration-seconds) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101101` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101001` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -9686,7 +9627,7 @@ Query: `max by(instance) (go_gc_duration_seconds{job=~".*precise-code-intel-work Refer to the [alerts reference](./alerts.md#precise-code-intel-worker-pods-available-percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101200` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/precise-code-intel-worker/precise-code-intel-worker?viewPanel=101100` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12637,27 +12578,6 @@ Query: `max(src_query_runner_worker_total{job=~"^worker.*"}) > 0 and on(job) sum
-### Worker: Internal service requests - -#### worker: frontend_internal_api_error_responses - -

Frontend-internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#worker-frontend-internal-api-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102800` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* - -
-Technical details - -Query: `sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="worker",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="worker"}[5m]))` - -
- -
- ### Worker: Database connections #### worker: max_open_conns @@ -12666,7 +12586,7 @@ Query: `sum by (category)(increase(src_frontend_internal_request_duration_second This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102900` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102800` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12685,7 +12605,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_max_open{app_name="worker"}) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102901` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102801` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12704,7 +12624,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_open{app_name="worker"})` This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102910` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102810` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12723,7 +12643,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_in_use{app_name="worker"})` This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102911` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102811` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12742,7 +12662,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_idle{app_name="worker"})` Refer to the [alerts reference](./alerts.md#worker-mean-blocked-seconds-per-conn-request) for 2 alerts related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102920` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102820` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12761,7 +12681,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_blocked_seconds{app This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102930` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102830` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12780,7 +12700,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle{app This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102931` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102831` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12799,7 +12719,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_lifetime This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102932` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102832` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -12830,7 +12750,7 @@ value change independent of deployment events (such as an upgrade), it could ind This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103000` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102900` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12849,7 +12769,7 @@ Query: `count by(name) ((time() - container_last_seen{name=~"^worker.*"}) > 60)` Refer to the [alerts reference](./alerts.md#worker-container-cpu-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103001` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102901` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12868,7 +12788,7 @@ Query: `cadvisor_container_cpu_usage_percentage_total{name=~"^worker.*"}` Refer to the [alerts reference](./alerts.md#worker-container-memory-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103002` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102902` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12890,7 +12810,7 @@ When extremely high, this can indicate a resource usage problem, or can cause pr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103003` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=102903` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12911,7 +12831,7 @@ Query: `sum by(name) (rate(container_fs_reads_total{name=~"^worker.*"}[1h]) + ra Refer to the [alerts reference](./alerts.md#worker-provisioning-container-cpu-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103100` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103000` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12930,7 +12850,7 @@ Query: `quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{na Refer to the [alerts reference](./alerts.md#worker-provisioning-container-memory-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103101` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103001` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12949,7 +12869,7 @@ Query: `max_over_time(cadvisor_container_memory_usage_percentage_total{name=~"^w Refer to the [alerts reference](./alerts.md#worker-provisioning-container-cpu-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103110` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103010` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12968,7 +12888,7 @@ Query: `max_over_time(cadvisor_container_cpu_usage_percentage_total{name=~"^work Refer to the [alerts reference](./alerts.md#worker-provisioning-container-memory-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103111` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103011` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -12990,7 +12910,7 @@ When it occurs frequently, it is an indicator of underprovisioning. Refer to the [alerts reference](./alerts.md#worker-container-oomkill-events-total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103112` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103012` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13013,7 +12933,7 @@ A high value here indicates a possible goroutine leak. Refer to the [alerts reference](./alerts.md#worker-go-goroutines) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103200` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103100` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13032,7 +12952,7 @@ Query: `max by(instance) (go_goroutines{job=~".*worker"})` Refer to the [alerts reference](./alerts.md#worker-go-gc-duration-seconds) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103201` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103101` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13053,7 +12973,7 @@ Query: `max by(instance) (go_gc_duration_seconds{job=~".*worker"})` Refer to the [alerts reference](./alerts.md#worker-pods-available-percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103300` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103200` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13074,7 +12994,7 @@ Query: `sum by(app) (up{app=~".*worker"}) / count by (app) (up{app=~".*worker"}) This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103400` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103300` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13093,7 +13013,7 @@ Query: `sum(increase(src_workerutil_dbworker_store_own_background_worker_store_t This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103401` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103301` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13112,7 +13032,7 @@ Query: `sum by (le)(rate(src_workerutil_dbworker_store_own_background_worker_st This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103402` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103302` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13131,7 +13051,7 @@ Query: `sum(increase(src_workerutil_dbworker_store_own_background_worker_store_e This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103403` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103303` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13150,7 +13070,7 @@ Query: `sum(increase(src_workerutil_dbworker_store_own_background_worker_store_e This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103410` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103310` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13169,7 +13089,7 @@ Query: `sum by (op)(increase(src_workerutil_dbworker_store_own_background_worker This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103411` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103311` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13188,7 +13108,7 @@ Query: `histogram_quantile(0.99, sum by (le,op)(rate(src_workerutil_dbworker_st This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103412` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103312` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13207,7 +13127,7 @@ Query: `sum by (op)(increase(src_workerutil_dbworker_store_own_background_worker This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103413` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103313` on your Sourcegraph instance. *Managed by the [Sourcegraph own team](https://handbook.sourcegraph.com/departments/engineering/teams/own).* @@ -13228,7 +13148,7 @@ Query: `sum by (op)(increase(src_workerutil_dbworker_store_own_background_worker This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103500` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103400` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13247,7 +13167,7 @@ Query: `sum(src_own_background_worker_processor_handlers{job=~"^worker.*"})` This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103510` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103410` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13266,7 +13186,7 @@ Query: `sum(increase(src_own_background_worker_processor_total{job=~"^worker.*"} This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103511` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103411` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13285,7 +13205,7 @@ Query: `sum by (le)(rate(src_own_background_worker_processor_duration_seconds_b This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103512` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103412` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13304,7 +13224,7 @@ Query: `sum(increase(src_own_background_worker_processor_errors_total{job=~"^wor This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103513` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103413` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13325,7 +13245,7 @@ Query: `sum(increase(src_own_background_worker_processor_errors_total{job=~"^wor This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103600` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103500` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13344,7 +13264,7 @@ Query: `sum(increase(src_own_background_worker_record_resets_total{job=~"^worker This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103601` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103501` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13363,7 +13283,7 @@ Query: `sum(increase(src_own_background_worker_record_reset_failures_total{job=~ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103602` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103502` on your Sourcegraph instance. *Managed by the [Sourcegraph Code Search team](https://handbook.sourcegraph.com/departments/engineering/teams/code-search).* @@ -13384,7 +13304,7 @@ Query: `sum(increase(src_own_background_worker_record_reset_errors_total{job=~"^ This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103700` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103600` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13403,7 +13323,7 @@ Query: `sum by (op)(increase(src_own_background_index_scheduler_total{job=~"^wor This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103701` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103601` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13422,7 +13342,7 @@ Query: `histogram_quantile(0.99, sum by (le,op)(rate(src_own_background_index_s This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103702` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103602` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13441,7 +13361,7 @@ Query: `sum by (op)(increase(src_own_background_index_scheduler_errors_total{job This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103703` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103603` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -13464,7 +13384,7 @@ The duration since the configuration client used by the "worker" service last su This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103800` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103700` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -13483,7 +13403,7 @@ Query: `src_conf_client_time_since_last_successful_update_seconds{instance=~`${i Refer to the [alerts reference](./alerts.md#worker-worker-site-configuration-duration-since-last-successful-update-by-instance) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103801` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/worker/worker?viewPanel=103701` on your Sourcegraph instance. *Managed by the [Sourcegraph Infrastructure Org team](https://handbook.sourcegraph.com/departments/engineering/infrastructure).* @@ -15809,27 +15729,6 @@ Query: `histogram_quantile(0.95, sum(rate(src_http_request_duration_seconds_buck
-### Repo Updater: Internal service requests - -#### repo-updater: frontend_internal_api_error_responses - -

Frontend-internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#repo-updater-frontend-internal-api-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101200` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* - -
-Technical details - -Query: `sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="repo-updater",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="repo-updater"}[5m]))` - -
- -
- ### Repo Updater: Database connections #### repo-updater: max_open_conns @@ -15838,7 +15737,7 @@ Query: `sum by (category)(increase(src_frontend_internal_request_duration_second This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101300` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101200` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -15857,7 +15756,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_max_open{app_name="repo-upda This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101301` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101201` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -15876,7 +15775,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_open{app_name="repo-updater" This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101310` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101210` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -15895,7 +15794,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_in_use{app_name="repo-update This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101311` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101211` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -15914,7 +15813,7 @@ Query: `sum by (app_name, db_name) (src_pgsql_conns_idle{app_name="repo-updater" Refer to the [alerts reference](./alerts.md#repo-updater-mean-blocked-seconds-per-conn-request) for 2 alerts related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101320` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101220` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -15933,7 +15832,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_blocked_seconds{app This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101330` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101230` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -15952,7 +15851,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle{app This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101331` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101231` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -15971,7 +15870,7 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_lifetime This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101332` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101232` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16002,7 +15901,7 @@ value change independent of deployment events (such as an upgrade), it could ind This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101400` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101300` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16021,7 +15920,7 @@ Query: `count by(name) ((time() - container_last_seen{name=~"^repo-updater.*"}) Refer to the [alerts reference](./alerts.md#repo-updater-container-cpu-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101401` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101301` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16040,7 +15939,7 @@ Query: `cadvisor_container_cpu_usage_percentage_total{name=~"^repo-updater.*"}` Refer to the [alerts reference](./alerts.md#repo-updater-container-memory-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101402` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101302` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16062,7 +15961,7 @@ When extremely high, this can indicate a resource usage problem, or can cause pr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101403` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101303` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16083,7 +15982,7 @@ Query: `sum by(name) (rate(container_fs_reads_total{name=~"^repo-updater.*"}[1h] Refer to the [alerts reference](./alerts.md#repo-updater-provisioning-container-cpu-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101500` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101400` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16102,7 +16001,7 @@ Query: `quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{na Refer to the [alerts reference](./alerts.md#repo-updater-provisioning-container-memory-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101501` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101401` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16121,7 +16020,7 @@ Query: `max_over_time(cadvisor_container_memory_usage_percentage_total{name=~"^r Refer to the [alerts reference](./alerts.md#repo-updater-provisioning-container-cpu-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101510` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101410` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16140,7 +16039,7 @@ Query: `max_over_time(cadvisor_container_cpu_usage_percentage_total{name=~"^repo Refer to the [alerts reference](./alerts.md#repo-updater-provisioning-container-memory-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101511` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101411` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16162,7 +16061,7 @@ When it occurs frequently, it is an indicator of underprovisioning. Refer to the [alerts reference](./alerts.md#repo-updater-container-oomkill-events-total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101512` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101412` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16185,7 +16084,7 @@ A high value here indicates a possible goroutine leak. Refer to the [alerts reference](./alerts.md#repo-updater-go-goroutines) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101600` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101500` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16204,7 +16103,7 @@ Query: `max by(instance) (go_goroutines{job=~".*repo-updater"})` Refer to the [alerts reference](./alerts.md#repo-updater-go-gc-duration-seconds) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101601` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101501` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -16225,7 +16124,7 @@ Query: `max by(instance) (go_gc_duration_seconds{job=~".*repo-updater"})` Refer to the [alerts reference](./alerts.md#repo-updater-pods-available-percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101700` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/repo-updater/repo-updater?viewPanel=101600` on your Sourcegraph instance. *Managed by the [Sourcegraph Source team](https://handbook.sourcegraph.com/departments/engineering/teams/source).* @@ -17398,27 +17297,6 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle_tim
-### Searcher: Internal service requests - -#### searcher: frontend_internal_api_error_responses - -

Frontend-internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#searcher-frontend-internal-api-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100800` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* - -
-Technical details - -Query: `sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="searcher",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="searcher"}[5m]))` - -
- -
- ### Searcher: Container monitoring (not available on server) #### searcher: container_missing @@ -17437,7 +17315,7 @@ value change independent of deployment events (such as an upgrade), it could ind This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100900` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100800` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17456,7 +17334,7 @@ Query: `count by(name) ((time() - container_last_seen{name=~"^searcher.*"}) > 60 Refer to the [alerts reference](./alerts.md#searcher-container-cpu-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100901` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100801` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17475,7 +17353,7 @@ Query: `cadvisor_container_cpu_usage_percentage_total{name=~"^searcher.*"}` Refer to the [alerts reference](./alerts.md#searcher-container-memory-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100902` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100802` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17497,7 +17375,7 @@ When extremely high, this can indicate a resource usage problem, or can cause pr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100903` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100803` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17518,7 +17396,7 @@ Query: `sum by(name) (rate(container_fs_reads_total{name=~"^searcher.*"}[1h]) + Refer to the [alerts reference](./alerts.md#searcher-provisioning-container-cpu-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101000` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100900` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17537,7 +17415,7 @@ Query: `quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{na Refer to the [alerts reference](./alerts.md#searcher-provisioning-container-memory-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101001` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100901` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17556,7 +17434,7 @@ Query: `max_over_time(cadvisor_container_memory_usage_percentage_total{name=~"^s Refer to the [alerts reference](./alerts.md#searcher-provisioning-container-cpu-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101010` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100910` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17575,7 +17453,7 @@ Query: `max_over_time(cadvisor_container_cpu_usage_percentage_total{name=~"^sear Refer to the [alerts reference](./alerts.md#searcher-provisioning-container-memory-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101011` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100911` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17597,7 +17475,7 @@ When it occurs frequently, it is an indicator of underprovisioning. Refer to the [alerts reference](./alerts.md#searcher-container-oomkill-events-total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101012` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=100912` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17620,7 +17498,7 @@ A high value here indicates a possible goroutine leak. Refer to the [alerts reference](./alerts.md#searcher-go-goroutines) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101100` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101000` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17639,7 +17517,7 @@ Query: `max by(instance) (go_goroutines{job=~".*searcher"})` Refer to the [alerts reference](./alerts.md#searcher-go-gc-duration-seconds) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101101` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101001` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -17660,7 +17538,7 @@ Query: `max by(instance) (go_gc_duration_seconds{job=~".*searcher"})` Refer to the [alerts reference](./alerts.md#searcher-pods-available-percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101200` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/searcher/searcher?viewPanel=101100` on your Sourcegraph instance. *Managed by the [Sourcegraph Search Platform team](https://handbook.sourcegraph.com/departments/engineering/teams/search/core).* @@ -19133,27 +19011,6 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle_tim
-### Symbols: Internal service requests - -#### symbols: frontend_internal_api_error_responses - -

Frontend-internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#symbols-frontend-internal-api-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=100900` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* - -
-Technical details - -Query: `sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="symbols",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="symbols"}[5m]))` - -
- -
- ### Symbols: Container monitoring (not available on server) #### symbols: container_missing @@ -19172,7 +19029,7 @@ value change independent of deployment events (such as an upgrade), it could ind This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101000` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=100900` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19191,7 +19048,7 @@ Query: `count by(name) ((time() - container_last_seen{name=~"^symbols.*"}) > 60) Refer to the [alerts reference](./alerts.md#symbols-container-cpu-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101001` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=100901` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19210,7 +19067,7 @@ Query: `cadvisor_container_cpu_usage_percentage_total{name=~"^symbols.*"}` Refer to the [alerts reference](./alerts.md#symbols-container-memory-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101002` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=100902` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19232,7 +19089,7 @@ When extremely high, this can indicate a resource usage problem, or can cause pr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101003` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=100903` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19253,7 +19110,7 @@ Query: `sum by(name) (rate(container_fs_reads_total{name=~"^symbols.*"}[1h]) + r Refer to the [alerts reference](./alerts.md#symbols-provisioning-container-cpu-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101100` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101000` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19272,7 +19129,7 @@ Query: `quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{na Refer to the [alerts reference](./alerts.md#symbols-provisioning-container-memory-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101101` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101001` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19291,7 +19148,7 @@ Query: `max_over_time(cadvisor_container_memory_usage_percentage_total{name=~"^s Refer to the [alerts reference](./alerts.md#symbols-provisioning-container-cpu-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101110` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101010` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19310,7 +19167,7 @@ Query: `max_over_time(cadvisor_container_cpu_usage_percentage_total{name=~"^symb Refer to the [alerts reference](./alerts.md#symbols-provisioning-container-memory-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101111` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101011` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19332,7 +19189,7 @@ When it occurs frequently, it is an indicator of underprovisioning. Refer to the [alerts reference](./alerts.md#symbols-container-oomkill-events-total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101112` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101012` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19355,7 +19212,7 @@ A high value here indicates a possible goroutine leak. Refer to the [alerts reference](./alerts.md#symbols-go-goroutines) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101200` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101100` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19374,7 +19231,7 @@ Query: `max by(instance) (go_goroutines{job=~".*symbols"})` Refer to the [alerts reference](./alerts.md#symbols-go-gc-duration-seconds) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101201` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101101` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -19395,7 +19252,7 @@ Query: `max by(instance) (go_gc_duration_seconds{job=~".*symbols"})` Refer to the [alerts reference](./alerts.md#symbols-pods-available-percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101300` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/symbols/symbols?viewPanel=101200` on your Sourcegraph instance. *Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/engineering/teams/code-intelligence).* @@ -31881,27 +31738,6 @@ Query: `sum by (app_name, db_name) (increase(src_pgsql_conns_closed_max_idle_tim
-### Embeddings: Internal service requests - -#### embeddings: frontend_internal_api_error_responses - -

Frontend-internal API error responses every 5m by route

- -Refer to the [alerts reference](./alerts.md#embeddings-frontend-internal-api-error-responses) for 1 alert related to this panel. - -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100200` on your Sourcegraph instance. - -*Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* - -
-Technical details - -Query: `sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="embeddings",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="embeddings"}[5m]))` - -
- -
- ### Embeddings: Container monitoring (not available on server) #### embeddings: container_missing @@ -31920,7 +31756,7 @@ value change independent of deployment events (such as an upgrade), it could ind This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100300` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100200` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -31939,7 +31775,7 @@ Query: `count by(name) ((time() - container_last_seen{name=~"^embeddings.*"}) > Refer to the [alerts reference](./alerts.md#embeddings-container-cpu-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100301` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100201` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -31958,7 +31794,7 @@ Query: `cadvisor_container_cpu_usage_percentage_total{name=~"^embeddings.*"}` Refer to the [alerts reference](./alerts.md#embeddings-container-memory-usage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100302` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100202` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -31980,7 +31816,7 @@ When extremely high, this can indicate a resource usage problem, or can cause pr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100303` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100203` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32001,7 +31837,7 @@ Query: `sum by(name) (rate(container_fs_reads_total{name=~"^embeddings.*"}[1h]) Refer to the [alerts reference](./alerts.md#embeddings-provisioning-container-cpu-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100400` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100300` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32020,7 +31856,7 @@ Query: `quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{na Refer to the [alerts reference](./alerts.md#embeddings-provisioning-container-memory-usage-long-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100401` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100301` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32039,7 +31875,7 @@ Query: `max_over_time(cadvisor_container_memory_usage_percentage_total{name=~"^e Refer to the [alerts reference](./alerts.md#embeddings-provisioning-container-cpu-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100410` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100310` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32058,7 +31894,7 @@ Query: `max_over_time(cadvisor_container_cpu_usage_percentage_total{name=~"^embe Refer to the [alerts reference](./alerts.md#embeddings-provisioning-container-memory-usage-short-term) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100411` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100311` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32080,7 +31916,7 @@ When it occurs frequently, it is an indicator of underprovisioning. Refer to the [alerts reference](./alerts.md#embeddings-container-oomkill-events-total) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100412` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100312` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32103,7 +31939,7 @@ A high value here indicates a possible goroutine leak. Refer to the [alerts reference](./alerts.md#embeddings-go-goroutines) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100500` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100400` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32122,7 +31958,7 @@ Query: `max by(instance) (go_goroutines{job=~".*embeddings"})` Refer to the [alerts reference](./alerts.md#embeddings-go-gc-duration-seconds) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100501` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100401` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32143,7 +31979,7 @@ Query: `max by(instance) (go_gc_duration_seconds{job=~".*embeddings"})` Refer to the [alerts reference](./alerts.md#embeddings-pods-available-percentage) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100600` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100500` on your Sourcegraph instance. *Managed by the [Sourcegraph Cody team](https://handbook.sourcegraph.com/departments/engineering/teams/cody).* @@ -32166,7 +32002,7 @@ A low hit rate indicates your cache is not well utilized. Consider increasing th This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100700` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100600` on your Sourcegraph instance.
@@ -32186,7 +32022,7 @@ A high volume of misses indicates that the many searches are not hitting the cac This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100701` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/embeddings/embeddings?viewPanel=100601` on your Sourcegraph instance.
diff --git a/internal/api/internalapi/BUILD.bazel b/internal/api/internalapi/BUILD.bazel index ed9832ceb12..15e5371de5a 100644 --- a/internal/api/internalapi/BUILD.bazel +++ b/internal/api/internalapi/BUILD.bazel @@ -10,15 +10,10 @@ go_library( importpath = "github.com/sourcegraph/sourcegraph/internal/api/internalapi", visibility = ["//:__subpackages__"], deps = [ - "//internal/actor", "//internal/api/internalapi/v1:internalapi", "//internal/conf/conftypes", "//internal/env", "//internal/grpc/defaults", - "//internal/httpcli", - "//lib/errors", - "@com_github_prometheus_client_golang//prometheus", - "@com_github_prometheus_client_golang//prometheus/promauto", "@com_github_sourcegraph_log//:log", "@org_golang_google_grpc//:go_default_library", ], diff --git a/internal/api/internalapi/client.go b/internal/api/internalapi/client.go index 23938a90eda..ed3322070bc 100644 --- a/internal/api/internalapi/client.go +++ b/internal/api/internalapi/client.go @@ -1,30 +1,19 @@ package internalapi import ( - "bytes" "context" - "encoding/json" "fmt" "net" - "net/http" "net/url" - "strconv" "strings" "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sourcegraph/log" - "github.com/sourcegraph/sourcegraph/internal/actor" proto "github.com/sourcegraph/sourcegraph/internal/api/internalapi/v1" "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/env" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpcli" - "github.com/sourcegraph/sourcegraph/lib/errors" ) var frontendInternal = func() *url.URL { @@ -32,10 +21,6 @@ var frontendInternal = func() *url.URL { return mustParseSourcegraphInternalURL(rawURL) }() -// NOTE: this intentionally does not use the site configuration option because we need to make the decision -// about whether or not to use gRPC to fetch the site configuration in the first place. -var enableGRPC = env.MustGetBool("SRC_GRPC_ENABLE_CONF", false, "Enable gRPC for configuration updates") - func defaultFrontendInternal() string { return "sourcegraph-frontend-internal" } @@ -61,12 +46,6 @@ var Client = &internalClient{ }), } -var requestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "src_frontend_internal_request_duration_seconds", - Help: "Time (in seconds) spent on request.", - Buckets: prometheus.DefBuckets, -}, []string{"category", "code"}) - // MockClientConfiguration mocks (*internalClient).Configuration. var MockClientConfiguration func() (conftypes.RawUnified, error) @@ -75,102 +54,17 @@ func (c *internalClient) Configuration(ctx context.Context) (conftypes.RawUnifie return MockClientConfiguration() } - if enableGRPC { - cc, err := c.getConfClient() - if err != nil { - return conftypes.RawUnified{}, err - } - resp, err := cc.GetConfig(ctx, &proto.GetConfigRequest{}) - if err != nil { - return conftypes.RawUnified{}, err - } - var raw conftypes.RawUnified - raw.FromProto(resp.RawUnified) - return raw, nil - } - - var cfg conftypes.RawUnified - err := c.postInternal(ctx, "configuration", nil, &cfg) - return cfg, err -} - -// postInternal sends an HTTP post request to the internal route. -func (c *internalClient) postInternal(ctx context.Context, route string, reqBody, respBody any) error { - return c.meteredPost(ctx, "/.internal/"+route, reqBody, respBody) -} - -func (c *internalClient) meteredPost(ctx context.Context, route string, reqBody, respBody any) error { - start := time.Now() - statusCode, err := c.post(ctx, route, reqBody, respBody) - d := time.Since(start) - - code := strconv.Itoa(statusCode) + cc, err := c.getConfClient() if err != nil { - code = "error" + return conftypes.RawUnified{}, err } - requestDuration.WithLabelValues(route, code).Observe(d.Seconds()) - return err -} - -// post sends an HTTP post request to the provided route. If reqBody is -// non-nil it will Marshal it as JSON and set that as the Request body. If -// respBody is non-nil the response body will be JSON unmarshalled to resp. -func (c *internalClient) post(ctx context.Context, route string, reqBody, respBody any) (int, error) { - var data []byte - if reqBody != nil { - var err error - data, err = json.Marshal(reqBody) - if err != nil { - return -1, err - } - } - - req, err := http.NewRequest("POST", c.URL+route, bytes.NewBuffer(data)) + resp, err := cc.GetConfig(ctx, &proto.GetConfigRequest{}) if err != nil { - return -1, err + return conftypes.RawUnified{}, err } - - req.Header.Set("Content-Type", "application/json") - - // Check if we have an actor, if not, ensure that we use our internal actor since - // this is an internal request. - a := actor.FromContext(ctx) - if !a.IsAuthenticated() && !a.IsInternal() { - ctx = actor.WithInternalActor(ctx) - } - - resp, err := httpcli.InternalDoer.Do(req.WithContext(ctx)) - if err != nil { - return -1, err - } - defer resp.Body.Close() - if err := checkAPIResponse(resp); err != nil { - return resp.StatusCode, err - } - - if respBody != nil { - return resp.StatusCode, json.NewDecoder(resp.Body).Decode(respBody) - } - return resp.StatusCode, nil -} - -func checkAPIResponse(resp *http.Response) error { - if 200 > resp.StatusCode || resp.StatusCode > 299 { - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(resp.Body) - b := buf.Bytes() - errString := string(b) - if errString != "" { - return errors.Errorf( - "internal API response error code %d: %s (%s)", - resp.StatusCode, - errString, - resp.Request.URL, - ) - } - return errors.Errorf("internal API response error code %d (%s)", resp.StatusCode, resp.Request.URL) - } - return nil + var raw conftypes.RawUnified + raw.FromProto(resp.RawUnified) + return raw, nil } // mustParseSourcegraphInternalURL parses a frontend internal URL string and panics if it is invalid. diff --git a/internal/batches/sources/BUILD.bazel b/internal/batches/sources/BUILD.bazel index 64aab47dbb9..19ba863a8ba 100644 --- a/internal/batches/sources/BUILD.bazel +++ b/internal/batches/sources/BUILD.bazel @@ -95,6 +95,7 @@ go_test( "//internal/extsvc/gerrit", "//internal/extsvc/github", "//internal/extsvc/gitlab", + "//internal/extsvc/gitolite", "//internal/extsvc/versions", "//internal/github_apps/auth", "//internal/github_apps/store", diff --git a/internal/batches/sources/mocks_test.go b/internal/batches/sources/mocks_test.go index 0c7790812b4..7bdb14fbeb9 100644 --- a/internal/batches/sources/mocks_test.go +++ b/internal/batches/sources/mocks_test.go @@ -23,6 +23,7 @@ import ( azuredevops "github.com/sourcegraph/sourcegraph/internal/extsvc/azuredevops" bitbucketcloud "github.com/sourcegraph/sourcegraph/internal/extsvc/bitbucketcloud" gerrit "github.com/sourcegraph/sourcegraph/internal/extsvc/gerrit" + gitolite "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" store1 "github.com/sourcegraph/sourcegraph/internal/github_apps/store" gitserver "github.com/sourcegraph/sourcegraph/internal/gitserver" gitdomain "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" @@ -10949,6 +10950,9 @@ type MockGitserverClient struct { // ListDirectoryChildrenFunc is an instance of a mock function object // controlling the behavior of the method ListDirectoryChildren. ListDirectoryChildrenFunc *GitserverClientListDirectoryChildrenFunc + // ListGitoliteReposFunc is an instance of a mock function object + // controlling the behavior of the method ListGitoliteRepos. + ListGitoliteReposFunc *GitserverClientListGitoliteReposFunc // ListRefsFunc is an instance of a mock function object controlling the // behavior of the method ListRefs. ListRefsFunc *GitserverClientListRefsFunc @@ -11191,6 +11195,11 @@ func NewMockGitserverClient() *MockGitserverClient { return }, }, + ListGitoliteReposFunc: &GitserverClientListGitoliteReposFunc{ + defaultHook: func(context.Context, string) (r0 []*gitolite.Repo, r1 error) { + return + }, + }, ListRefsFunc: &GitserverClientListRefsFunc{ defaultHook: func(context.Context, api.RepoName) (r0 []gitdomain.Ref, r1 error) { return @@ -11488,6 +11497,11 @@ func NewStrictMockGitserverClient() *MockGitserverClient { panic("unexpected invocation of MockGitserverClient.ListDirectoryChildren") }, }, + ListGitoliteReposFunc: &GitserverClientListGitoliteReposFunc{ + defaultHook: func(context.Context, string) ([]*gitolite.Repo, error) { + panic("unexpected invocation of MockGitserverClient.ListGitoliteRepos") + }, + }, ListRefsFunc: &GitserverClientListRefsFunc{ defaultHook: func(context.Context, api.RepoName) ([]gitdomain.Ref, error) { panic("unexpected invocation of MockGitserverClient.ListRefs") @@ -11724,6 +11738,9 @@ func NewMockGitserverClientFrom(i gitserver.Client) *MockGitserverClient { ListDirectoryChildrenFunc: &GitserverClientListDirectoryChildrenFunc{ defaultHook: i.ListDirectoryChildren, }, + ListGitoliteReposFunc: &GitserverClientListGitoliteReposFunc{ + defaultHook: i.ListGitoliteRepos, + }, ListRefsFunc: &GitserverClientListRefsFunc{ defaultHook: i.ListRefs, }, @@ -15275,6 +15292,117 @@ func (c GitserverClientListDirectoryChildrenFuncCall) Results() []interface{} { return []interface{}{c.Result0, c.Result1} } +// GitserverClientListGitoliteReposFunc describes the behavior when the +// ListGitoliteRepos method of the parent MockGitserverClient instance is +// invoked. +type GitserverClientListGitoliteReposFunc struct { + defaultHook func(context.Context, string) ([]*gitolite.Repo, error) + hooks []func(context.Context, string) ([]*gitolite.Repo, error) + history []GitserverClientListGitoliteReposFuncCall + mutex sync.Mutex +} + +// ListGitoliteRepos delegates to the next hook function in the queue and +// stores the parameter and result values of this invocation. +func (m *MockGitserverClient) ListGitoliteRepos(v0 context.Context, v1 string) ([]*gitolite.Repo, error) { + r0, r1 := m.ListGitoliteReposFunc.nextHook()(v0, v1) + m.ListGitoliteReposFunc.appendCall(GitserverClientListGitoliteReposFuncCall{v0, v1, r0, r1}) + return r0, r1 +} + +// SetDefaultHook sets function that is called when the ListGitoliteRepos +// method of the parent MockGitserverClient instance is invoked and the hook +// queue is empty. +func (f *GitserverClientListGitoliteReposFunc) SetDefaultHook(hook func(context.Context, string) ([]*gitolite.Repo, error)) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// ListGitoliteRepos method of the parent MockGitserverClient instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverClientListGitoliteReposFunc) PushHook(hook func(context.Context, string) ([]*gitolite.Repo, error)) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverClientListGitoliteReposFunc) SetDefaultReturn(r0 []*gitolite.Repo, r1 error) { + f.SetDefaultHook(func(context.Context, string) ([]*gitolite.Repo, error) { + return r0, r1 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverClientListGitoliteReposFunc) PushReturn(r0 []*gitolite.Repo, r1 error) { + f.PushHook(func(context.Context, string) ([]*gitolite.Repo, error) { + return r0, r1 + }) +} + +func (f *GitserverClientListGitoliteReposFunc) nextHook() func(context.Context, string) ([]*gitolite.Repo, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverClientListGitoliteReposFunc) appendCall(r0 GitserverClientListGitoliteReposFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of GitserverClientListGitoliteReposFuncCall +// objects describing the invocations of this function. +func (f *GitserverClientListGitoliteReposFunc) History() []GitserverClientListGitoliteReposFuncCall { + f.mutex.Lock() + history := make([]GitserverClientListGitoliteReposFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverClientListGitoliteReposFuncCall is an object that describes an +// invocation of method ListGitoliteRepos on an instance of +// MockGitserverClient. +type GitserverClientListGitoliteReposFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 context.Context + // Arg1 is the value of the 2nd argument passed to this method + // invocation. + Arg1 string + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 []*gitolite.Repo + // Result1 is the value of the 2nd result returned from this method + // invocation. + Result1 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverClientListGitoliteReposFuncCall) Args() []interface{} { + return []interface{}{c.Arg0, c.Arg1} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverClientListGitoliteReposFuncCall) Results() []interface{} { + return []interface{}{c.Result0, c.Result1} +} + // GitserverClientListRefsFunc describes the behavior when the ListRefs // method of the parent MockGitserverClient instance is invoked. type GitserverClientListRefsFunc struct { diff --git a/internal/conf/BUILD.bazel b/internal/conf/BUILD.bazel index 06d017d49d7..df905f17045 100644 --- a/internal/conf/BUILD.bazel +++ b/internal/conf/BUILD.bazel @@ -11,7 +11,6 @@ go_library( "computed.go", "conf.go", "diff.go", - "grpc.go", "helpers.go", "init.go", "log_sinks.go", @@ -60,7 +59,6 @@ go_test( "client_test.go", "computed_test.go", "diff_test.go", - "grpc_test.go", "mocks_test.go", "validate_test.go", ], diff --git a/internal/conf/grpc.go b/internal/conf/grpc.go deleted file mode 100644 index f762ac0cfe9..00000000000 --- a/internal/conf/grpc.go +++ /dev/null @@ -1,20 +0,0 @@ -package conf - -import ( - "context" - "os" - "strconv" -) - -const envGRPCEnabled = "SG_FEATURE_FLAG_GRPC" - -func IsGRPCEnabled(ctx context.Context) bool { - if val, err := strconv.ParseBool(os.Getenv(envGRPCEnabled)); err == nil { - return val - } - if c := Get(); c.ExperimentalFeatures != nil && c.ExperimentalFeatures.EnableGRPC != nil { - return *c.ExperimentalFeatures.EnableGRPC - } - - return true -} diff --git a/internal/conf/grpc_test.go b/internal/conf/grpc_test.go deleted file mode 100644 index ca7811524cb..00000000000 --- a/internal/conf/grpc_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package conf - -import ( - "context" - "testing" -) - -func TestIsGRPCEnabled(t *testing.T) { - tests := []struct { - name string - envValue string - expected bool - }{ - { - name: "enabled", - - envValue: "true", - expected: true, - }, - { - name: "disabled", - - envValue: "false", - expected: false, - }, - { - name: "empty env var - default true", - - envValue: "", - expected: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Setenv(envGRPCEnabled, test.envValue) - actual := IsGRPCEnabled(context.Background()) - - if actual != test.expected { - t.Errorf("expected %v but got %v", test.expected, actual) - } - }) - } - -} diff --git a/internal/embeddings/client.go b/internal/embeddings/client.go index 28e985b72b9..7843ed40a11 100644 --- a/internal/embeddings/client.go +++ b/internal/embeddings/client.go @@ -9,9 +9,10 @@ import ( "strings" "github.com/sourcegraph/conc/pool" - "github.com/sourcegraph/sourcegraph/lib/errors" "go.opentelemetry.io/otel/attribute" + "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/endpoint" diff --git a/internal/extsvc/versions/BUILD.bazel b/internal/extsvc/versions/BUILD.bazel index 8f26d4212ac..566a575d4f2 100644 --- a/internal/extsvc/versions/BUILD.bazel +++ b/internal/extsvc/versions/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "//internal/database", "//internal/env", "//internal/extsvc", + "//internal/gitserver", "//internal/goroutine", "//internal/httpcli", "//internal/observation", diff --git a/internal/extsvc/versions/sync.go b/internal/extsvc/versions/sync.go index 680f0205d71..6489ad23458 100644 --- a/internal/extsvc/versions/sync.go +++ b/internal/extsvc/versions/sync.go @@ -12,6 +12,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/env" "github.com/sourcegraph/sourcegraph/internal/extsvc" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/goroutine" "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/observation" @@ -50,7 +51,7 @@ func (j *syncingJob) Routines(_ context.Context, observationCtx *observation.Con sourcerCF := httpcli.NewExternalClientFactory( httpcli.NewLoggingMiddleware(sourcerLogger), ) - sourcer := repos.NewSourcer(sourcerLogger, db, sourcerCF) + sourcer := repos.NewSourcer(sourcerLogger, db, sourcerCF, gitserver.NewClient("extsvc.version-syncer")) store := db.ExternalServices() handler := goroutine.HandlerFunc(func(ctx context.Context) error { diff --git a/internal/gitserver/BUILD.bazel b/internal/gitserver/BUILD.bazel index d1b239698e3..46e5ce9b359 100644 --- a/internal/gitserver/BUILD.bazel +++ b/internal/gitserver/BUILD.bazel @@ -8,11 +8,9 @@ go_library( "client.go", "commands.go", "git_command.go", - "gitolite.go", "mock.go", "mocks_temp.go", "observability.go", - "proxy.go", "retry.go", "stream_client.go", "stream_hunks.go", @@ -34,9 +32,7 @@ go_library( "//internal/grpc/defaults", "//internal/grpc/streamio", "//internal/honey", - "//internal/httpcli", "//internal/lazyregexp", - "//internal/limiter", "//internal/metrics", "//internal/observation", "//internal/perforce", @@ -54,8 +50,8 @@ go_library( "@io_opentelemetry_go_otel//attribute", "@org_golang_google_grpc//:go_default_library", "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//metadata", "@org_golang_google_grpc//status", - "@org_golang_google_protobuf//encoding/protojson", "@org_golang_x_exp//slices", "@org_golang_x_sync//errgroup", "@org_golang_x_sync//semaphore", @@ -89,7 +85,6 @@ go_test( "//internal/gitserver/protocol", "//internal/gitserver/v1:gitserver", "//internal/grpc", - "//internal/httpcli", "//internal/types", "//lib/errors", "//schema", @@ -98,6 +93,5 @@ go_test( "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@org_golang_google_grpc//:go_default_library", - "@org_golang_google_protobuf//encoding/protojson", ], ) diff --git a/internal/gitserver/client.go b/internal/gitserver/client.go index 3c0871f8c0b..81da986d82c 100644 --- a/internal/gitserver/client.go +++ b/internal/gitserver/client.go @@ -3,13 +3,9 @@ package gitserver import ( "bytes" "context" - "encoding/gob" - "encoding/json" "fmt" "io" "io/fs" - "net/http" - "net/url" "os" "path/filepath" "strings" @@ -23,7 +19,6 @@ import ( "golang.org/x/sync/semaphore" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/encoding/protojson" "github.com/sourcegraph/conc/pool" "github.com/sourcegraph/go-diff/diff" @@ -33,14 +28,12 @@ import ( "github.com/sourcegraph/sourcegraph/internal/actor" "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/authz" - "github.com/sourcegraph/sourcegraph/internal/conf" + "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" "github.com/sourcegraph/sourcegraph/internal/gitserver/protocol" proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" "github.com/sourcegraph/sourcegraph/internal/grpc/streamio" - "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/lazyregexp" - "github.com/sourcegraph/sourcegraph/internal/limiter" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/perforce" "github.com/sourcegraph/sourcegraph/lib/errors" @@ -48,13 +41,6 @@ import ( const git = "git" -var ( - clientFactory = httpcli.NewInternalClientFactory("gitserver") - defaultDoer, _ = clientFactory.Doer() - // defaultLimiter limits concurrent HTTP requests per running process to gitserver. - defaultLimiter = limiter.New(500) -) - var ClientMocks, emptyClientMocks struct { GetObject func(repo api.RepoName, objectName string) (*gitdomain.GitObject, error) Archive func(ctx context.Context, repo api.RepoName, opt ArchiveOptions) (_ io.ReadCloser, err error) @@ -92,10 +78,8 @@ type ClientSource interface { func NewClient(scope string) Client { logger := sglog.Scoped("GitserverClient") return &clientImplementor{ - logger: logger, - scope: scope, - httpClient: defaultDoer, - HTTPLimiter: defaultLimiter, + logger: logger, + scope: scope, // Use the binary name for userAgent. This should effectively identify // which service is making the request (excluding requests proxied via the // frontend internal API) @@ -111,10 +95,8 @@ func NewTestClient(t testing.TB) TestClient { logger := logtest.Scoped(t) return &clientImplementor{ - logger: logger, - httpClient: http.DefaultClient, - HTTPLimiter: limiter.New(500), - scope: fmt.Sprintf("gitserver.test.%s", t.Name()), + logger: logger, + scope: fmt.Sprintf("gitserver.test.%s", t.Name()), // Use the binary name for userAgent. This should effectively identify // which service is making the request (excluding requests proxied via the // frontend internal API) @@ -128,7 +110,6 @@ func NewTestClient(t testing.TB) TestClient { type TestClient interface { Client WithChecker(authz.SubRepoPermissionChecker) TestClient - WithDoer(httpcli.Doer) TestClient WithClientSource(ClientSource) TestClient } @@ -137,11 +118,6 @@ func (c *clientImplementor) WithChecker(checker authz.SubRepoPermissionChecker) return c } -func (c *clientImplementor) WithDoer(doer httpcli.Doer) TestClient { - c.httpClient = doer - return c -} - func (c *clientImplementor) WithClientSource(cs ClientSource) TestClient { c.clientSource = cs return c @@ -220,9 +196,6 @@ func NewMockClientWithExecReader(checker authz.SubRepoPermissionChecker, execRea // clientImplementor is a gitserver client. type clientImplementor struct { - // Limits concurrency of outstanding HTTP posts - HTTPLimiter limiter.Limiter - // userAgent is a string identifying who the client is. It will be logged in // the telemetry in gitserver. userAgent string @@ -230,9 +203,6 @@ type clientImplementor struct { // the current scope of the client. scope string - // HTTP client to use - httpClient httpcli.Doer - // logger is used for all logging and logger creation logger sglog.Logger @@ -251,8 +221,6 @@ func (c *clientImplementor) Scoped(scope string) Client { return &clientImplementor{ logger: c.logger, scope: appendScope(c.scope, scope), - httpClient: c.httpClient, - HTTPLimiter: c.HTTPLimiter, userAgent: c.userAgent, operations: c.operations, clientSource: c.clientSource, @@ -527,6 +495,10 @@ type Client interface { // PerforceGetChangelist gets the perforce changelist details for the given changelist ID. PerforceGetChangelist(ctx context.Context, conn protocol.PerforceConnectionDetails, changelist string) (*perforce.Changelist, error) + + // ListGitoliteRepos returns a list of Gitolite repositories. Gitserver owns the SSH keys + // so is the only service able to talk to gitolite. + ListGitoliteRepos(ctx context.Context, host string) ([]*gitolite.Repo, error) } func (c *clientImplementor) SystemsInfo(ctx context.Context) (_ []protocol.SystemInfo, err error) { @@ -585,39 +557,15 @@ func (c *clientImplementor) SystemInfo(ctx context.Context, addr string) (_ prot } func (c *clientImplementor) getDiskInfo(ctx context.Context, addr AddressWithClient) (*proto.DiskInfoResponse, error) { - if conf.IsGRPCEnabled(ctx) { - client, err := addr.GRPCClient() - if err != nil { - return nil, err - } - resp, err := client.DiskInfo(ctx, &proto.DiskInfoRequest{}) - if err != nil { - return nil, err - } - return resp, nil - } - - uri := fmt.Sprintf("http://%s/disk-info", addr.Address()) - rs, err := c.do(ctx, "", uri, nil) + client, err := addr.GRPCClient() if err != nil { return nil, err } - defer rs.Body.Close() - if rs.StatusCode != http.StatusOK { - return nil, errors.Newf("http status %d: %s", rs.StatusCode, readResponseBody(io.LimitReader(rs.Body, 200))) - } - - body, err := io.ReadAll(rs.Body) + resp, err := client.DiskInfo(ctx, &proto.DiskInfoRequest{}) if err != nil { - return nil, errors.Wrap(err, "reading read disk info response body") + return nil, err } - - var resp proto.DiskInfoResponse - if err := protojson.Unmarshal(body, &resp); err != nil { - return nil, errors.Wrap(err, "parsing disk info response body") - } - - return &resp, nil + return resp, nil } func (c *clientImplementor) Addrs() []string { @@ -721,32 +669,6 @@ func (a *archiveReader) Close() error { return a.base.Close() } -// archiveURL returns a URL from which an archive of the given Git repository can -// be downloaded from. -func (c *clientImplementor) archiveURL(ctx context.Context, repo api.RepoName, opt ArchiveOptions) *url.URL { - q := url.Values{ - "repo": {string(repo)}, - "treeish": {opt.Treeish}, - "format": {string(opt.Format)}, - } - - for _, pathspec := range opt.Pathspecs { - q.Add("path", string(pathspec)) - } - - addrForRepo := c.AddrForRepo(ctx, repo) - return &url.URL{ - Scheme: "http", - Host: addrForRepo, - Path: "/archive", - RawQuery: q.Encode(), - } -} - -type badRequestError struct{ error } - -func (e badRequestError) BadRequest() bool { return true } - func (c *RemoteGitCommand) sendExec(ctx context.Context) (_ io.ReadCloser, err error) { ctx, cancel := context.WithCancel(ctx) ctx, _, endObservation := c.execOp.With(ctx, &err, observation.Args{ @@ -774,69 +696,36 @@ func (c *RemoteGitCommand) sendExec(ctx context.Context) (_ io.ReadCloser, err e return nil, err } - if conf.IsGRPCEnabled(ctx) { - client, err := c.execer.ClientForRepo(ctx, repoName) - if err != nil { - return nil, err - } - - req := &proto.ExecRequest{ - Repo: string(repoName), - Args: stringsToByteSlices(c.args[1:]), - Stdin: c.stdin, - NoTimeout: c.noTimeout, - - // 🚨Warning🚨: There is no guarantee that EnsureRevision is a valid utf-8 string. - EnsureRevision: []byte(c.EnsureRevision()), - } - - stream, err := client.Exec(ctx, req) - if err != nil { - return nil, err - } - r := streamio.NewReader(func() ([]byte, error) { - msg, err := stream.Recv() - if status.Code(err) == codes.Canceled { - return nil, context.Canceled - } else if err != nil { - return nil, err - } - return msg.GetData(), nil - }) - - return &readCloseWrapper{r: r, closeFn: done}, nil - - } else { - req := &protocol.ExecRequest{ - Repo: repoName, - EnsureRevision: c.EnsureRevision(), - Args: c.args[1:], - Stdin: c.stdin, - NoTimeout: c.noTimeout, - } - resp, err := c.execer.httpPost(ctx, repoName, "exec", req) - if err != nil { - return nil, err - } - - switch resp.StatusCode { - case http.StatusOK: - return &cmdReader{rc: &readCloseWrapper{r: resp.Body, closeFn: done}, trailer: resp.Trailer}, nil - - case http.StatusNotFound: - var payload protocol.NotFoundPayload - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - resp.Body.Close() - return nil, err - } - resp.Body.Close() - return nil, &gitdomain.RepoNotExistError{Repo: repoName, CloneInProgress: payload.CloneInProgress, CloneProgress: payload.CloneProgress} - - default: - resp.Body.Close() - return nil, errors.Errorf("unexpected status code: %d", resp.StatusCode) - } + client, err := c.execer.ClientForRepo(ctx, repoName) + if err != nil { + return nil, err } + + req := &proto.ExecRequest{ + Repo: string(repoName), + Args: stringsToByteSlices(c.args[1:]), + Stdin: c.stdin, + NoTimeout: c.noTimeout, + + // 🚨Warning🚨: There is no guarantee that EnsureRevision is a valid utf-8 string. + EnsureRevision: []byte(c.EnsureRevision()), + } + + stream, err := client.Exec(ctx, req) + if err != nil { + return nil, err + } + r := streamio.NewReader(func() ([]byte, error) { + msg, err := stream.Recv() + if status.Code(err) == codes.Canceled { + return nil, context.Canceled + } else if err != nil { + return nil, err + } + return msg.GetData(), nil + }) + + return &readCloseWrapper{r: r, closeFn: done}, nil } type readCloseWrapper struct { @@ -922,7 +811,7 @@ func isRevisionNotFound(err string) bool { return strings.Contains(loweredErr, "not a valid object") } -func (c *clientImplementor) Search(ctx context.Context, args *protocol.SearchRequest, onMatches func([]protocol.CommitMatch)) (limitHit bool, err error) { +func (c *clientImplementor) Search(ctx context.Context, args *protocol.SearchRequest, onMatches func([]protocol.CommitMatch)) (_ bool, err error) { ctx, _, endObservation := c.operations.search.With(ctx, &err, observation.Args{ MetricLabelValues: []string{c.scope}, Attrs: []attribute.KeyValue{ @@ -936,76 +825,32 @@ func (c *clientImplementor) Search(ctx context.Context, args *protocol.SearchReq repoName := protocol.NormalizeRepo(args.Repo) - if conf.IsGRPCEnabled(ctx) { - client, err := c.ClientForRepo(ctx, repoName) - if err != nil { - return false, err - } - - cs, err := client.Search(ctx, args.ToProto()) - if err != nil { - return false, convertGitserverError(err) - } - - limitHit := false - for { - msg, err := cs.Recv() - if err != nil { - return limitHit, convertGitserverError(err) - } - - switch m := msg.Message.(type) { - case *proto.SearchResponse_LimitHit: - limitHit = limitHit || m.LimitHit - case *proto.SearchResponse_Match: - onMatches([]protocol.CommitMatch{protocol.CommitMatchFromProto(m.Match)}) - default: - return false, errors.Newf("unknown message type %T", m) - } - } - } - - addrForRepo := c.AddrForRepo(ctx, repoName) - - protocol.RegisterGob() - var buf bytes.Buffer - enc := gob.NewEncoder(&buf) - if err := enc.Encode(args); err != nil { - return false, err - } - - uri := "http://" + addrForRepo + "/search" - resp, err := c.do(ctx, repoName, uri, buf.Bytes()) + client, err := c.ClientForRepo(ctx, repoName) if err != nil { return false, err } - defer resp.Body.Close() - var ( - decodeErr error - eventDone protocol.SearchEventDone - ) - dec := StreamSearchDecoder{ - OnMatches: func(e protocol.SearchEventMatches) { - onMatches(e) - }, - OnDone: func(e protocol.SearchEventDone) { - eventDone = e - }, - OnUnknown: func(event, _ []byte) { - decodeErr = errors.Errorf("unknown event %s", event) - }, + cs, err := client.Search(ctx, args.ToProto()) + if err != nil { + return false, convertGitserverError(err) } - if err := dec.ReadAll(resp.Body); err != nil { - return false, err - } + limitHit := false + for { + msg, err := cs.Recv() + if err != nil { + return limitHit, convertGitserverError(err) + } - if decodeErr != nil { - return false, decodeErr + switch m := msg.Message.(type) { + case *proto.SearchResponse_LimitHit: + limitHit = limitHit || m.LimitHit + case *proto.SearchResponse_Match: + onMatches([]protocol.CommitMatch{protocol.CommitMatchFromProto(m.Match)}) + default: + return false, errors.Newf("unknown message type %T", m) + } } - - return eventDone.LimitHit, eventDone.Err() } func convertGitserverError(err error) error { @@ -1086,43 +931,19 @@ func (c *clientImplementor) BatchLog(ctx context.Context, opts BatchLogOptions, var response protocol.BatchLogResponse - if conf.IsGRPCEnabled(ctx) { - client, err := grpcClient.client, grpcClient.dialErr - if err != nil { - return err - } - - resp, err := client.BatchLog(ctx, request.ToProto()) - if err != nil { - return err - } - - response.FromProto(resp) - logger.AddEvent("read response", attribute.Int("numResults", len(response.Results))) - } else { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(request); err != nil { - return err - } - - uri := "http://" + addr + "/batch-log" - resp, err := c.do(ctx, api.RepoName(strings.Join(repoNames, ",")), uri, buf.Bytes()) - if err != nil { - return err - } - defer resp.Body.Close() - logger.AddEvent("POST", attribute.Int("resp.StatusCode", resp.StatusCode)) - - if resp.StatusCode != http.StatusOK { - return errors.Newf("http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))) - } - - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return err - } - logger.AddEvent("read response", attribute.Int("numResults", len(response.Results))) + client, err := grpcClient.client, grpcClient.dialErr + if err != nil { + return err } + resp, err := client.BatchLog(ctx, request.ToProto()) + if err != nil { + return err + } + + response.FromProto(resp) + logger.AddEvent("read response", attribute.Int("numResults", len(response.Results))) + for _, result := range response.Results { var err error if result.CommandError != "" { @@ -1253,40 +1074,20 @@ func (c *clientImplementor) RequestRepoUpdate(ctx context.Context, repo api.Repo Since: since, } - if conf.IsGRPCEnabled(ctx) { - client, err := c.ClientForRepo(ctx, repo) - if err != nil { - return nil, err - } - - resp, err := client.RepoUpdate(ctx, req.ToProto()) - if err != nil { - return nil, err - } - - var info protocol.RepoUpdateResponse - info.FromProto(resp) - - return &info, nil - - } else { - resp, err := c.httpPost(ctx, repo, "repo-update", req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "RepoUpdate", - Err: errors.Errorf("RepoUpdate: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - var info protocol.RepoUpdateResponse - err = json.NewDecoder(resp.Body).Decode(&info) - return &info, err + client, err := c.ClientForRepo(ctx, repo) + if err != nil { + return nil, err } + + resp, err := client.RepoUpdate(ctx, req.ToProto()) + if err != nil { + return nil, err + } + + var info protocol.RepoUpdateResponse + info.FromProto(resp) + + return &info, nil } // RequestRepoClone requests that the gitserver does an asynchronous clone of the repository. @@ -1299,47 +1100,23 @@ func (c *clientImplementor) RequestRepoClone(ctx context.Context, repo api.RepoN }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - client, err := c.ClientForRepo(ctx, repo) - if err != nil { - return nil, err - } - - req := proto.RepoCloneRequest{ - Repo: string(repo), - } - - resp, err := client.RepoClone(ctx, &req) - if err != nil { - return nil, err - } - - var info protocol.RepoCloneResponse - info.FromProto(resp) - return &info, nil - - } else { - - req := &protocol.RepoCloneRequest{ - Repo: repo, - } - resp, err := c.httpPost(ctx, repo, "repo-clone", req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "RepoInfo", - Err: errors.Errorf("RepoInfo: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - var info *protocol.RepoCloneResponse - err = json.NewDecoder(resp.Body).Decode(&info) - return info, err + client, err := c.ClientForRepo(ctx, repo) + if err != nil { + return nil, err } + + req := proto.RepoCloneRequest{ + Repo: string(repo), + } + + resp, err := client.RepoClone(ctx, &req) + if err != nil { + return nil, err + } + + var info protocol.RepoCloneResponse + info.FromProto(resp) + return &info, nil } // MockIsRepoCloneable mocks (*Client).IsRepoCloneable for tests. @@ -1360,40 +1137,22 @@ func (c *clientImplementor) IsRepoCloneable(ctx context.Context, repo api.RepoNa var resp protocol.IsRepoCloneableResponse - if conf.IsGRPCEnabled(ctx) { - client, err := c.ClientForRepo(ctx, repo) - if err != nil { - return err - } - - req := &proto.IsRepoCloneableRequest{ - Repo: string(repo), - } - - r, err := client.IsRepoCloneable(ctx, req) - if err != nil { - return err - } - - resp.FromProto(r) - } else { - req := &protocol.IsRepoCloneableRequest{ - Repo: repo, - } - r, err := c.httpPost(ctx, repo, "is-repo-cloneable", req) - if err != nil { - return err - } - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - return errors.Errorf("gitserver error (status code %d): %s", r.StatusCode, readResponseBody(r.Body)) - } - - if err := json.NewDecoder(r.Body).Decode(&resp); err != nil { - return err - } + client, err := c.ClientForRepo(ctx, repo) + if err != nil { + return err } + req := &proto.IsRepoCloneableRequest{ + Repo: string(repo), + } + + r, err := client.IsRepoCloneable(ctx, req) + if err != nil { + return err + } + + resp.FromProto(r) + if resp.Cloneable { return nil } @@ -1444,122 +1203,52 @@ func (c *clientImplementor) RepoCloneProgress(ctx context.Context, repos ...api. numPossibleShards := len(c.Addrs()) - if conf.IsGRPCEnabled(ctx) { - shards := make(map[proto.GitserverServiceClient]*proto.RepoCloneProgressRequest, (len(repos)/numPossibleShards)*2) // 2x because it may not be a perfect division - for _, r := range repos { - client, err := c.ClientForRepo(ctx, r) - if err != nil { - return nil, err - } - - shard := shards[client] - if shard == nil { - shard = new(proto.RepoCloneProgressRequest) - shards[client] = shard - } - - shard.Repos = append(shard.Repos, string(r)) - } - - p := pool.NewWithResults[*proto.RepoCloneProgressResponse]().WithContext(ctx) - - for client, req := range shards { - client := client - req := req - p.Go(func(ctx context.Context) (*proto.RepoCloneProgressResponse, error) { - return client.RepoCloneProgress(ctx, req) - - }) - } - - res, err := p.Wait() + shards := make(map[proto.GitserverServiceClient]*proto.RepoCloneProgressRequest, (len(repos)/numPossibleShards)*2) // 2x because it may not be a perfect division + for _, r := range repos { + client, err := c.ClientForRepo(ctx, r) if err != nil { return nil, err } - result := &protocol.RepoCloneProgressResponse{ - Results: make(map[api.RepoName]*protocol.RepoCloneProgress), - } - for _, r := range res { - - for repo, info := range r.Results { - var rp protocol.RepoCloneProgress - rp.FromProto(info) - result.Results[api.RepoName(repo)] = &rp - } - + shard := shards[client] + if shard == nil { + shard = new(proto.RepoCloneProgressRequest) + shards[client] = shard } - return result, nil - - } else { - - shards := make(map[string]*protocol.RepoCloneProgressRequest, (len(repos)/numPossibleShards)*2) // 2x because it may not be a perfect division - - for _, r := range repos { - addr := c.AddrForRepo(ctx, r) - shard := shards[addr] - - if shard == nil { - shard = new(protocol.RepoCloneProgressRequest) - shards[addr] = shard - } - - shard.Repos = append(shard.Repos, r) - } - - type op struct { - req *protocol.RepoCloneProgressRequest - res *protocol.RepoCloneProgressResponse - err error - } - - ch := make(chan op, len(shards)) - for _, req := range shards { - go func(o op) { - var resp *http.Response - resp, o.err = c.httpPost(ctx, o.req.Repos[0], "repo-clone-progress", o.req) - if o.err != nil { - ch <- o - return - } - - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - o.err = &url.Error{ - URL: resp.Request.URL.String(), - Op: "RepoCloneProgress", - Err: errors.Errorf("RepoCloneProgress: http status %d", resp.StatusCode), - } - ch <- o - return // we never get an error status code AND result - } - - o.res = new(protocol.RepoCloneProgressResponse) - o.err = json.NewDecoder(resp.Body).Decode(o.res) - ch <- o - }(op{req: req}) - } - - var err error - res := protocol.RepoCloneProgressResponse{ - Results: make(map[api.RepoName]*protocol.RepoCloneProgress), - } - - for i := 0; i < cap(ch); i++ { - o := <-ch - - if o.err != nil { - err = errors.Append(err, o.err) - continue - } - - for repo, info := range o.res.Results { - res.Results[repo] = info - } - } - return &res, err + shard.Repos = append(shard.Repos, string(r)) } + + p := pool.NewWithResults[*proto.RepoCloneProgressResponse]().WithContext(ctx) + + for client, req := range shards { + client := client + req := req + p.Go(func(ctx context.Context) (*proto.RepoCloneProgressResponse, error) { + return client.RepoCloneProgress(ctx, req) + + }) + } + + res, err := p.Wait() + if err != nil { + return nil, err + } + + result := &protocol.RepoCloneProgressResponse{ + Results: make(map[api.RepoName]*protocol.RepoCloneProgress), + } + for _, r := range res { + + for repo, info := range r.Results { + var rp protocol.RepoCloneProgress + rp.FromProto(info) + result.Results[api.RepoName(repo)] = &rp + } + + } + + return result, nil } func (c *clientImplementor) Remove(ctx context.Context, repo api.RepoName) (err error) { @@ -1575,44 +1264,14 @@ func (c *clientImplementor) Remove(ctx context.Context, repo api.RepoName) (err // the old name in order to land on the correct gitserver instance undeletedName := api.UndeletedRepoName(repo) - if conf.IsGRPCEnabled(ctx) { - client, err := c.ClientForRepo(ctx, undeletedName) - if err != nil { - return err - } - _, err = client.RepoDelete(ctx, &proto.RepoDeleteRequest{ - Repo: string(repo), - }) + client, err := c.ClientForRepo(ctx, undeletedName) + if err != nil { return err } - - addr := c.AddrForRepo(ctx, undeletedName) - return c.removeFrom(ctx, undeletedName, addr) -} - -func (c *clientImplementor) removeFrom(ctx context.Context, repo api.RepoName, from string) error { - b, err := json.Marshal(&protocol.RepoDeleteRequest{ - Repo: repo, + _, err = client.RepoDelete(ctx, &proto.RepoDeleteRequest{ + Repo: string(repo), }) - if err != nil { - return err - } - - uri := "http://" + from + "/delete" - resp, err := c.do(ctx, repo, uri, b) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return &url.Error{ - URL: resp.Request.URL.String(), - Op: "RepoRemove", - Err: errors.Errorf("RepoRemove: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - return nil + return err } func (c *clientImplementor) IsPerforcePathCloneable(ctx context.Context, conn protocol.PerforceConnectionDetails, depotPath string) (err error) { @@ -1621,53 +1280,23 @@ func (c *clientImplementor) IsPerforcePathCloneable(ctx context.Context, conn pr }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // depotPath is not actually a repo name, but it will spread the load of isPerforcePathCloneable - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(depotPath)) - if err != nil { - return err - } - _, err = client.IsPerforcePathCloneable(ctx, &proto.IsPerforcePathCloneableRequest{ - ConnectionDetails: conn.ToProto(), - DepotPath: depotPath, - }) - if err != nil { - // Unwrap proto errors for nicer error messages. - if s, ok := status.FromError(err); ok { - return errors.New(s.Message()) - } - return err - } - - return nil + // depotPath is not actually a repo name, but it will spread the load of isPerforcePathCloneable + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(depotPath)) + if err != nil { + return err } - - addr := c.AddrForRepo(ctx, api.RepoName(depotPath)) - b, err := json.Marshal(&protocol.IsPerforcePathCloneableRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, - DepotPath: depotPath, + _, err = client.IsPerforcePathCloneable(ctx, &proto.IsPerforcePathCloneableRequest{ + ConnectionDetails: conn.ToProto(), + DepotPath: depotPath, }) if err != nil { - return err - } - - uri := "http://" + addr + "/is-perforce-path-cloneable" - resp, err := c.do(ctx, "is-perforce-path-cloneable", uri, b) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return &url.Error{ - URL: resp.Request.URL.String(), - Op: "IsPerforcePathCloneable", - Err: errors.Errorf("IsPerforcePathCloneable: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), + // Unwrap proto errors for nicer error messages. + if s, ok := status.FromError(err); ok { + return errors.New(s.Message()) } + return err } return nil @@ -1679,51 +1308,22 @@ func (c *clientImplementor) CheckPerforceCredentials(ctx context.Context, conn p }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) - if err != nil { - return err - } - _, err = client.CheckPerforceCredentials(ctx, &proto.CheckPerforceCredentialsRequest{ - ConnectionDetails: conn.ToProto(), - }) - if err != nil { - // Unwrap proto errors for nicer error messages. - if s, ok := status.FromError(err); ok { - return errors.New(s.Message()) - } - return err - } - - return nil + // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) + if err != nil { + return err } - - addr := c.AddrForRepo(ctx, api.RepoName(conn.P4Port)) - b, err := json.Marshal(&protocol.CheckPerforceCredentialsRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, + _, err = client.CheckPerforceCredentials(ctx, &proto.CheckPerforceCredentialsRequest{ + ConnectionDetails: conn.ToProto(), }) if err != nil { - return err - } - - uri := "http://" + addr + "/check-perforce-credentials" - resp, err := c.do(ctx, "check-perforce-credentials", uri, b) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return &url.Error{ - URL: resp.Request.URL.String(), - Op: "CheckPerforceCredentials", - Err: errors.Errorf("CheckPerforceCredentials: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), + // Unwrap proto errors for nicer error messages. + if s, ok := status.FromError(err); ok { + return errors.New(s.Message()) } + return err } return nil @@ -1735,66 +1335,24 @@ func (c *clientImplementor) PerforceUsers(ctx context.Context, conn protocol.Per }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) - if err != nil { - return nil, err - } - resp, err := client.PerforceUsers(ctx, &proto.PerforceUsersRequest{ - ConnectionDetails: conn.ToProto(), - }) - if err != nil { - return nil, err - } - - users := make([]*perforce.User, len(resp.GetUsers())) - for i, u := range resp.GetUsers() { - users[i] = perforce.UserFromProto(u) - } - return users, nil + // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) + if err != nil { + return nil, err } - - addr := c.AddrForRepo(ctx, api.RepoName(conn.P4Port)) - b, err := json.Marshal(&protocol.PerforceUsersRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, + resp, err := client.PerforceUsers(ctx, &proto.PerforceUsersRequest{ + ConnectionDetails: conn.ToProto(), }) if err != nil { return nil, err } - uri := "http://" + addr + "/perforce-users" - resp, err := c.do(ctx, "perforce-users", uri, b) - if err != nil { - return nil, err + users := make([]*perforce.User, len(resp.GetUsers())) + for i, u := range resp.GetUsers() { + users[i] = perforce.UserFromProto(u) } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "PerforceUsers", - Err: errors.Errorf("PerforceUsers: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - var payload protocol.PerforceUsersResponse - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, err - } - - users := make([]*perforce.User, len(payload.Users)) - for i, u := range payload.Users { - users[i] = &perforce.User{ - Username: u.Username, - Email: u.Email, - } - } - return users, nil } @@ -1807,72 +1365,25 @@ func (c *clientImplementor) PerforceProtectsForUser(ctx context.Context, conn pr }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) - if err != nil { - return nil, err - } - resp, err := client.PerforceProtectsForUser(ctx, &proto.PerforceProtectsForUserRequest{ - ConnectionDetails: conn.ToProto(), - Username: username, - }) - if err != nil { - return nil, err - } - - protects := make([]*perforce.Protect, len(resp.GetProtects())) - for i, p := range resp.GetProtects() { - protects[i] = perforce.ProtectFromProto(p) - } - return protects, nil + // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) + if err != nil { + return nil, err } - - addr := c.AddrForRepo(ctx, api.RepoName(conn.P4Port)) - b, err := json.Marshal(&protocol.PerforceProtectsForUserRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, - Username: username, + resp, err := client.PerforceProtectsForUser(ctx, &proto.PerforceProtectsForUserRequest{ + ConnectionDetails: conn.ToProto(), + Username: username, }) if err != nil { return nil, err } - uri := "http://" + addr + "/perforce-protects-for-user" - resp, err := c.do(ctx, "perforce-protects-for-user", uri, b) - if err != nil { - return nil, err + protects := make([]*perforce.Protect, len(resp.GetProtects())) + for i, p := range resp.GetProtects() { + protects[i] = perforce.ProtectFromProto(p) } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "PerforceProtectsForUser", - Err: errors.Errorf("PerforceProtectsForUser: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - var payload protocol.PerforceProtectsForUserResponse - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, err - } - - protects := make([]*perforce.Protect, len(payload.Protects)) - for i, p := range payload.Protects { - protects[i] = &perforce.Protect{ - Level: p.Level, - EntityType: p.EntityType, - EntityName: p.EntityName, - Match: p.Match, - IsExclusion: p.IsExclusion, - Host: p.Host, - } - } - return protects, nil } @@ -1885,72 +1396,25 @@ func (c *clientImplementor) PerforceProtectsForDepot(ctx context.Context, conn p }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) - if err != nil { - return nil, err - } - resp, err := client.PerforceProtectsForDepot(ctx, &proto.PerforceProtectsForDepotRequest{ - ConnectionDetails: conn.ToProto(), - Depot: depot, - }) - if err != nil { - return nil, err - } - - protects := make([]*perforce.Protect, len(resp.GetProtects())) - for i, p := range resp.GetProtects() { - protects[i] = perforce.ProtectFromProto(p) - } - return protects, nil + // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) + if err != nil { + return nil, err } - - addr := c.AddrForRepo(ctx, api.RepoName(conn.P4Port)) - b, err := json.Marshal(&protocol.PerforceProtectsForDepotRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, - Depot: depot, + resp, err := client.PerforceProtectsForDepot(ctx, &proto.PerforceProtectsForDepotRequest{ + ConnectionDetails: conn.ToProto(), + Depot: depot, }) if err != nil { return nil, err } - uri := "http://" + addr + "/perforce-protects-for-depot" - resp, err := c.do(ctx, "perforce-protects-for-depot", uri, b) - if err != nil { - return nil, err + protects := make([]*perforce.Protect, len(resp.GetProtects())) + for i, p := range resp.GetProtects() { + protects[i] = perforce.ProtectFromProto(p) } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "PerforceProtectsForDepot", - Err: errors.Errorf("PerforceProtectsForDepot: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - var payload protocol.PerforceProtectsForDepotResponse - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, err - } - - protects := make([]*perforce.Protect, len(payload.Protects)) - for i, p := range payload.Protects { - protects[i] = &perforce.Protect{ - Level: p.Level, - EntityType: p.EntityType, - EntityName: p.EntityName, - Match: p.Match, - IsExclusion: p.IsExclusion, - Host: p.Host, - } - } - return protects, nil } @@ -1963,57 +1427,22 @@ func (c *clientImplementor) PerforceGroupMembers(ctx context.Context, conn proto }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) - if err != nil { - return nil, err - } - resp, err := client.PerforceGroupMembers(ctx, &proto.PerforceGroupMembersRequest{ - ConnectionDetails: conn.ToProto(), - Group: group, - }) - if err != nil { - return nil, err - } - - return resp.GetUsernames(), nil + // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) + if err != nil { + return nil, err } - - addr := c.AddrForRepo(ctx, api.RepoName(conn.P4Port)) - b, err := json.Marshal(&protocol.PerforceGroupMembersRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, - Group: group, + resp, err := client.PerforceGroupMembers(ctx, &proto.PerforceGroupMembersRequest{ + ConnectionDetails: conn.ToProto(), + Group: group, }) if err != nil { return nil, err } - uri := "http://" + addr + "/perforce-group-members" - resp, err := c.do(ctx, "perforce-group-members", uri, b) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "PerforceGroupMembers", - Err: errors.Errorf("PerforceGroupMembers: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - var payload protocol.PerforceGroupMembersResponse - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, err - } - - return payload.Usernames, nil + return resp.GetUsernames(), nil } func (c *clientImplementor) IsPerforceSuperUser(ctx context.Context, conn protocol.PerforceConnectionDetails) (err error) { @@ -2022,46 +1451,17 @@ func (c *clientImplementor) IsPerforceSuperUser(ctx context.Context, conn protoc }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) - if err != nil { - return err - } - _, err = client.IsPerforceSuperUser(ctx, &proto.IsPerforceSuperUserRequest{ - ConnectionDetails: conn.ToProto(), - }) + // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) + if err != nil { return err } - - addr := c.AddrForRepo(ctx, api.RepoName(conn.P4Port)) - b, err := json.Marshal(&protocol.IsPerforceSuperUserRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, + _, err = client.IsPerforceSuperUser(ctx, &proto.IsPerforceSuperUserRequest{ + ConnectionDetails: conn.ToProto(), }) - if err != nil { - return err - } - - uri := "http://" + addr + "/is-perforce-super-user" - resp, err := c.do(ctx, "is-perforce-super-user", uri, b) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return &url.Error{ - URL: resp.Request.URL.String(), - Op: "IsPerforceSuperUser", - Err: errors.Errorf("IsPerforceSuperUser: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - return nil + return err } func (c *clientImplementor) PerforceGetChangelist(ctx context.Context, conn protocol.PerforceConnectionDetails, changelist string) (_ *perforce.Changelist, err error) { @@ -2073,120 +1473,22 @@ func (c *clientImplementor) PerforceGetChangelist(ctx context.Context, conn prot }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials - // a bit over the different gitserver instances. It's really just used as a consistent hashing - // key here. - client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) - if err != nil { - return nil, err - } - resp, err := client.PerforceGetChangelist(ctx, &proto.PerforceGetChangelistRequest{ - ConnectionDetails: conn.ToProto(), - ChangelistId: changelist, - }) - if err != nil { - return nil, err - } - - return perforce.ChangelistFromProto(resp.GetChangelist()), nil + // p4port is not actually a repo name, but it will spread the load of CheckPerforceCredentials + // a bit over the different gitserver instances. It's really just used as a consistent hashing + // key here. + client, err := c.ClientForRepo(ctx, api.RepoName(conn.P4Port)) + if err != nil { + return nil, err } - - addr := c.AddrForRepo(ctx, api.RepoName(conn.P4Port)) - b, err := json.Marshal(&protocol.PerforceGetChangelistRequest{ - P4Port: conn.P4Port, - P4User: conn.P4User, - P4Passwd: conn.P4Passwd, - ChangelistID: changelist, + resp, err := client.PerforceGetChangelist(ctx, &proto.PerforceGetChangelistRequest{ + ConnectionDetails: conn.ToProto(), + ChangelistId: changelist, }) if err != nil { return nil, err } - uri := "http://" + addr + "/perforce-get-changelist" - resp, err := c.do(ctx, "perforce-get-changelist", uri, b) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "PerforceGetChangelist", - Err: errors.Errorf("PerforceGetChangelist: http status %d: %s", resp.StatusCode, readResponseBody(io.LimitReader(resp.Body, 200))), - } - } - - var payload protocol.PerforceGetChangelistResponse - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - return nil, err - } - - cl := &perforce.Changelist{ - ID: payload.Changelist.ID, - CreationDate: payload.Changelist.CreationDate, - State: perforce.ChangelistState(payload.Changelist.State), - Author: payload.Changelist.Author, - Title: payload.Changelist.Title, - Message: payload.Changelist.Message, - } - - return cl, nil -} - -// httpPost will apply the MD5 hashing scheme on the repo name to determine the gitserver instance -// to which the HTTP POST request is sent. -func (c *clientImplementor) httpPost(ctx context.Context, repo api.RepoName, op string, payload any) (resp *http.Response, err error) { - b, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - addrForRepo := c.AddrForRepo(ctx, repo) - uri := "http://" + addrForRepo + "/" + op - return c.do(ctx, repo, uri, b) -} - -// do performs a request to a gitserver instance based on the address in the uri -// argument. -// -// repoForTracing parameter is optional. If it is provided, then "repo" attribute is added -// to trace span. -func (c *clientImplementor) do(ctx context.Context, repoForTracing api.RepoName, uri string, payload []byte) (resp *http.Response, err error) { - method := http.MethodPost - parsedURL, err := url.ParseRequestURI(uri) - if err != nil { - return nil, errors.Wrap(err, "do") - } - - ctx, trLogger, endObservation := c.operations.do.With(ctx, &err, observation.Args{ - MetricLabelValues: []string{c.scope}, - Attrs: []attribute.KeyValue{ - repoForTracing.Attr(), - attribute.String("method", method), - attribute.String("path", parsedURL.Path), - }, - }) - defer endObservation(1, observation.Args{}) - - req, err := http.NewRequestWithContext(ctx, method, uri, bytes.NewReader(payload)) - if err != nil { - return nil, err - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", c.userAgent) - - // Set header so that the server knows the request is from us. - req.Header.Set("X-Requested-With", "Sourcegraph") - - c.HTTPLimiter.Acquire() - defer c.HTTPLimiter.Release() - - trLogger.AddEvent("Acquired HTTP limiter") - - return c.httpClient.Do(req) + return perforce.ChangelistFromProto(resp.GetChangelist()), nil } func (c *clientImplementor) CreateCommitFromPatch(ctx context.Context, req protocol.CreateCommitFromPatchRequest) (_ *protocol.CreateCommitFromPatchResponse, err error) { @@ -2198,59 +1500,15 @@ func (c *clientImplementor) CreateCommitFromPatch(ctx context.Context, req proto }) defer endObservation(1, observation.Args{}) - if conf.IsGRPCEnabled(ctx) { - client, err := c.ClientForRepo(ctx, req.Repo) - if err != nil { - return nil, err - } - - cc, err := client.CreateCommitFromPatchBinary(ctx) - if err != nil { - st, ok := status.FromError(err) - if ok { - for _, detail := range st.Details() { - switch dt := detail.(type) { - case *proto.CreateCommitFromPatchError: - var e protocol.CreateCommitFromPatchError - e.FromProto(dt) - return nil, &e - } - } - } - return nil, err - } - - // Send the metadata event first. - if err := cc.Send(&proto.CreateCommitFromPatchBinaryRequest{Payload: &proto.CreateCommitFromPatchBinaryRequest_Metadata_{ - Metadata: req.ToMetadataProto(), - }}); err != nil { - return nil, errors.Wrap(err, "sending metadata") - } - - // Then create a writer that sends data in chunks that won't exceed the maximum - // message size of gRPC of the patch in separate events. - w := streamio.NewWriter(func(p []byte) error { - req := &proto.CreateCommitFromPatchBinaryRequest{ - Payload: &proto.CreateCommitFromPatchBinaryRequest_Patch_{ - Patch: &proto.CreateCommitFromPatchBinaryRequest_Patch{ - Data: p, - }, - }, - } - return cc.Send(req) - }) - - if _, err := w.Write(req.Patch); err != nil { - return nil, errors.Wrap(err, "writing chunk of patch") - } - - resp, err := cc.CloseAndRecv() - if err != nil { - st, ok := status.FromError(err) - if !ok { - return nil, err - } + client, err := c.ClientForRepo(ctx, req.Repo) + if err != nil { + return nil, err + } + cc, err := client.CreateCommitFromPatchBinary(ctx) + if err != nil { + st, ok := status.FromError(err) + if ok { for _, detail := range st.Details() { switch dt := detail.(type) { case *proto.CreateCommitFromPatchError: @@ -2259,39 +1517,56 @@ func (c *clientImplementor) CreateCommitFromPatch(ctx context.Context, req proto return nil, &e } } + } + return nil, err + } + // Send the metadata event first. + if err := cc.Send(&proto.CreateCommitFromPatchBinaryRequest{Payload: &proto.CreateCommitFromPatchBinaryRequest_Metadata_{ + Metadata: req.ToMetadataProto(), + }}); err != nil { + return nil, errors.Wrap(err, "sending metadata") + } + + // Then create a writer that sends data in chunks that won't exceed the maximum + // message size of gRPC of the patch in separate events. + w := streamio.NewWriter(func(p []byte) error { + req := &proto.CreateCommitFromPatchBinaryRequest{ + Payload: &proto.CreateCommitFromPatchBinaryRequest_Patch_{ + Patch: &proto.CreateCommitFromPatchBinaryRequest_Patch{ + Data: p, + }, + }, + } + return cc.Send(req) + }) + + if _, err := w.Write(req.Patch); err != nil { + return nil, errors.Wrap(err, "writing chunk of patch") + } + + resp, err := cc.CloseAndRecv() + if err != nil { + st, ok := status.FromError(err) + if !ok { return nil, err } - var res protocol.CreateCommitFromPatchResponse - res.FromProto(resp, nil) + for _, detail := range st.Details() { + switch dt := detail.(type) { + case *proto.CreateCommitFromPatchError: + var e protocol.CreateCommitFromPatchError + e.FromProto(dt) + return nil, &e + } + } - return &res, nil - } - - resp, err := c.httpPost(ctx, req.Repo, "create-commit-from-patch-binary", req) - if err != nil { return nil, err } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "failed to read response body") - } var res protocol.CreateCommitFromPatchResponse - if err := json.Unmarshal(body, &res); err != nil { - c.logger.Warn("decoding gitserver create-commit-from-patch response", sglog.Error(err)) - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "CreateCommitFromPatch", - Err: errors.Errorf("CreateCommitFromPatch: http status %d, %s", resp.StatusCode, string(body)), - } - } + res.FromProto(resp, nil) - if res.Error != nil { - return &res, res.Error - } return &res, nil } @@ -2313,50 +1588,21 @@ func (c *clientImplementor) GetObject(ctx context.Context, repo api.RepoName, ob Repo: repo, ObjectName: objectName, } - if conf.IsGRPCEnabled(ctx) { - client, err := c.ClientForRepo(ctx, req.Repo) - if err != nil { - return nil, err - } - - grpcResp, err := client.GetObject(ctx, req.ToProto()) - if err != nil { - - return nil, err - } - - var res protocol.GetObjectResponse - res.FromProto(grpcResp) - - return &res.Object, nil - - } else { - resp, err := c.httpPost(ctx, req.Repo, "commands/get-object", req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - c.logger.Warn("reading gitserver get-object response", sglog.Error(err)) - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "GetObject", - Err: errors.Errorf("GetObject: http status %d, %s", resp.StatusCode, readResponseBody(resp.Body)), - } - } - var res protocol.GetObjectResponse - if err = json.NewDecoder(resp.Body).Decode(&res); err != nil { - c.logger.Warn("decoding gitserver get-object response", sglog.Error(err)) - return nil, &url.Error{ - URL: resp.Request.URL.String(), - Op: "GetObject", - Err: errors.Errorf("GetObject: http status %d, failed to decode response body: %v", resp.StatusCode, err), - } - } - - return &res.Object, nil + client, err := c.ClientForRepo(ctx, req.Repo) + if err != nil { + return nil, err } + + grpcResp, err := client.GetObject(ctx, req.ToProto()) + if err != nil { + + return nil, err + } + + var res protocol.GetObjectResponse + res.FromProto(grpcResp) + + return &res.Object, nil } var ambiguousArgPattern = lazyregexp.New(`ambiguous argument '([^']+)'`) @@ -2442,3 +1688,29 @@ func stringsToByteSlices(in []string) [][]byte { } return res } + +func (c *clientImplementor) ListGitoliteRepos(ctx context.Context, gitoliteHost string) (list []*gitolite.Repo, err error) { + client, err := c.ClientForRepo(ctx, api.RepoName(gitoliteHost)) + if err != nil { + return nil, err + } + + req := &proto.ListGitoliteRequest{ + GitoliteHost: gitoliteHost, + } + + grpcResp, err := client.ListGitolite(ctx, req) + if err != nil { + return nil, err + } + + list = make([]*gitolite.Repo, len(grpcResp.Repos)) + for i, r := range grpcResp.GetRepos() { + list[i] = &gitolite.Repo{ + Name: r.GetName(), + URL: r.GetUrl(), + } + } + + return list, nil +} diff --git a/internal/gitserver/client_test.go b/internal/gitserver/client_test.go index dc0f0069ff2..168fb23fba9 100644 --- a/internal/gitserver/client_test.go +++ b/internal/gitserver/client_test.go @@ -1,13 +1,9 @@ package gitserver_test import ( - "bytes" "context" - "encoding/json" "fmt" - "io" "math/rand" - "net/http" "os/exec" "path/filepath" "reflect" @@ -22,17 +18,12 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "google.golang.org/grpc" - "google.golang.org/protobuf/encoding/protojson" "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/protocol" proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" - "github.com/sourcegraph/sourcegraph/internal/httpcli" - "github.com/sourcegraph/sourcegraph/lib/errors" - "github.com/sourcegraph/sourcegraph/schema" ) func TestClient_Archive_ProtoRoundTrip(t *testing.T) { @@ -291,97 +282,7 @@ func TestClient_ListGitolite_ProtoRoundTrip(t *testing.T) { } } -func TestClient_Remove(t *testing.T) { - test := func(t *testing.T, called *bool) { - repo := api.RepoName("github.com/sourcegraph/sourcegraph") - addrs := []string{"172.16.8.1:8080", "172.16.8.2:8080"} - - expected := "http://172.16.8.1:8080" - - source := gitserver.NewTestClientSource(t, addrs, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockRepoDelete := func(ctx context.Context, in *proto.RepoDeleteRequest, opts ...grpc.CallOption) (*proto.RepoDeleteResponse, error) { - *called = true - return nil, nil - } - - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.RepoDeleteFunc.SetDefaultHook(mockRepoDelete) - return cli - } - }) - cli := gitserver.NewTestClient(t). - WithDoer(httpcli.DoerFunc(func(r *http.Request) (*http.Response, error) { - switch r.URL.String() { - // Ensure that the request was received by the "expected" gitserver instance - where - // expected is the gitserver instance according to the Rendezvous hashing scheme. - // For anything else apart from this we return an error. - case expected + "/delete": - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewBufferString("{}")), - }, nil - default: - return nil, errors.Newf("unexpected URL: %q", r.URL.String()) - } - })). - WithClientSource(source) - - err := cli.Remove(context.Background(), repo) - if err != nil { - t.Fatalf("expected URL %q, but got err %q", expected, err) - } - } - - t.Run("GRPC", func(t *testing.T) { - called := false - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(true), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - test(t, &called) - if !called { - t.Fatal("grpc client not called") - } - }) - t.Run("HTTP", func(t *testing.T) { - called := false - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - test(t, &called) - if called { - t.Fatal("grpc client called") - } - }) - -} - -func TestClient_BatchLogGRPC(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(true), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - +func TestClient_BatchLog(t *testing.T) { addrs := []string{"172.16.8.1:8080"} called := false @@ -472,105 +373,6 @@ func TestClient_BatchLogGRPC(t *testing.T) { } } -func TestClient_BatchLog(t *testing.T) { - addrs := []string{"172.16.8.1:8080", "172.16.8.2:8080", "172.16.8.3:8080"} - source := gitserver.NewTestClientSource(t, addrs, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(conn *grpc.ClientConn) proto.GitserverServiceClient { - mockBatchLog := func(ctx context.Context, in *proto.BatchLogRequest, opts ...grpc.CallOption) (*proto.BatchLogResponse, error) { - var out []*proto.BatchLogResult - - for _, repoCommit := range in.GetRepoCommits() { - out = append(out, &proto.BatchLogResult{ - RepoCommit: repoCommit, - CommandOutput: fmt.Sprintf("out<%s: %s@%s>", fmt.Sprintf("http://%s/batch-log", conn.Target()), repoCommit.GetRepo(), repoCommit.GetCommit()), - CommandError: nil, - }) - } - - return &proto.BatchLogResponse{ - Results: out, - }, nil - } - - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.BatchLogFunc.SetDefaultHook(mockBatchLog) - return cli - } - }) - - cli := gitserver.NewTestClient(t). - WithDoer(httpcli.DoerFunc(func(r *http.Request) (*http.Response, error) { - var req protocol.BatchLogRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return nil, err - } - - var results []protocol.BatchLogResult - for _, repoCommit := range req.RepoCommits { - results = append(results, protocol.BatchLogResult{ - RepoCommit: repoCommit, - CommandOutput: fmt.Sprintf("out<%s: %s@%s>", r.URL.String(), repoCommit.Repo, repoCommit.CommitID), - CommandError: "", - }) - } - - encoded, _ := json.Marshal(protocol.BatchLogResponse{Results: results}) - body := io.NopCloser(strings.NewReader(strings.TrimSpace(string(encoded)))) - return &http.Response{StatusCode: 200, Body: body}, nil - })). - WithClientSource(source) - - opts := gitserver.BatchLogOptions{ - RepoCommits: []api.RepoCommit{ - {Repo: api.RepoName("github.com/test/foo"), CommitID: api.CommitID("deadbeef01")}, - {Repo: api.RepoName("github.com/test/bar"), CommitID: api.CommitID("deadbeef02")}, - {Repo: api.RepoName("github.com/test/baz"), CommitID: api.CommitID("deadbeef03")}, - {Repo: api.RepoName("github.com/test/bonk"), CommitID: api.CommitID("deadbeef04")}, - {Repo: api.RepoName("github.com/test/quux"), CommitID: api.CommitID("deadbeef05")}, - {Repo: api.RepoName("github.com/test/honk"), CommitID: api.CommitID("deadbeef06")}, - {Repo: api.RepoName("github.com/test/xyzzy"), CommitID: api.CommitID("deadbeef07")}, - {Repo: api.RepoName("github.com/test/lorem"), CommitID: api.CommitID("deadbeef08")}, - {Repo: api.RepoName("github.com/test/ipsum"), CommitID: api.CommitID("deadbeef09")}, - {Repo: api.RepoName("github.com/test/fnord"), CommitID: api.CommitID("deadbeef10")}, - }, - Format: "--format=test", - } - - results := map[api.RepoCommit]gitserver.RawBatchLogResult{} - var mu sync.Mutex - - if err := cli.BatchLog(context.Background(), opts, func(repoCommit api.RepoCommit, gitLogResult gitserver.RawBatchLogResult) error { - mu.Lock() - defer mu.Unlock() - - results[repoCommit] = gitLogResult - return nil - }); err != nil { - t.Fatalf("unexpected error performing batch log: %s", err) - } - - expectedResults := map[api.RepoCommit]gitserver.RawBatchLogResult{ - // Shard 1 - {Repo: "github.com/test/baz", CommitID: "deadbeef03"}: {Stdout: "out"}, - {Repo: "github.com/test/quux", CommitID: "deadbeef05"}: {Stdout: "out"}, - {Repo: "github.com/test/honk", CommitID: "deadbeef06"}: {Stdout: "out"}, - - // Shard 2 - {Repo: "github.com/test/bar", CommitID: "deadbeef02"}: {Stdout: "out"}, - {Repo: "github.com/test/xyzzy", CommitID: "deadbeef07"}: {Stdout: "out"}, - - // Shard 3 - {Repo: "github.com/test/foo", CommitID: "deadbeef01"}: {Stdout: "out"}, - {Repo: "github.com/test/bonk", CommitID: "deadbeef04"}: {Stdout: "out"}, - {Repo: "github.com/test/lorem", CommitID: "deadbeef08"}: {Stdout: "out"}, - {Repo: "github.com/test/ipsum", CommitID: "deadbeef09"}: {Stdout: "out"}, - {Repo: "github.com/test/fnord", CommitID: "deadbeef10"}: {Stdout: "out"}, - } - if diff := cmp.Diff(expectedResults, results); diff != "" { - t.Errorf("unexpected results (-want +got):\n%s", diff) - } -} - func TestLocalGitCommand(t *testing.T) { // creating a repo with 1 committed file root := gitserver.CreateRepoDir(t) @@ -667,97 +469,31 @@ func TestClient_IsRepoCloneableGRPC(t *testing.T) { }) } - t.Run("GRPC", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(true), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) + for _, tc := range testCases { - for _, tc := range testCases { - - called := false - source := gitserver.NewTestClientSource(t, []string{gitserverAddr}, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockIsRepoCloneable := func(ctx context.Context, in *proto.IsRepoCloneableRequest, opts ...grpc.CallOption) (*proto.IsRepoCloneableResponse, error) { - called = true - if api.RepoName(in.Repo) != tc.repo { - t.Errorf("got %q, want %q", in.Repo, tc.repo) - } - return tc.mockResponse.ToProto(), nil + called := false + source := gitserver.NewTestClientSource(t, []string{gitserverAddr}, func(o *gitserver.TestClientSourceOptions) { + o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { + mockIsRepoCloneable := func(ctx context.Context, in *proto.IsRepoCloneableRequest, opts ...grpc.CallOption) (*proto.IsRepoCloneableResponse, error) { + called = true + if api.RepoName(in.Repo) != tc.repo { + t.Errorf("got %q, want %q", in.Repo, tc.repo) } - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.IsRepoCloneableFunc.SetDefaultHook(mockIsRepoCloneable) - return cli + return tc.mockResponse.ToProto(), nil } - }) - - client := gitserver.NewTestClient(t).WithClientSource(source) - - runTests(t, client, tc) - if !called { - t.Fatal("IsRepoCloneable: grpc client not called") + cli := gitserver.NewStrictMockGitserverServiceClient() + cli.IsRepoCloneableFunc.SetDefaultHook(mockIsRepoCloneable) + return cli } - } - }) - - t.Run("HTTP", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, - }, }) - t.Cleanup(func() { - conf.Mock(nil) - }) - expected := fmt.Sprintf("http://%s", gitserverAddr) - for _, tc := range testCases { - called := false - source := gitserver.NewTestClientSource(t, []string{gitserverAddr}, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockIsRepoCloneable := func(ctx context.Context, in *proto.IsRepoCloneableRequest, opts ...grpc.CallOption) (*proto.IsRepoCloneableResponse, error) { - called = true - if api.RepoName(in.Repo) != tc.repo { - t.Errorf("got %q, want %q", in.Repo, tc.repo) - } - return tc.mockResponse.ToProto(), nil - } - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.IsRepoCloneableFunc.SetDefaultHook(mockIsRepoCloneable) - return cli - } - }) + client := gitserver.NewTestClient(t).WithClientSource(source) - client := gitserver.NewTestClient(t). - WithDoer(httpcli.DoerFunc(func(r *http.Request) (*http.Response, error) { - switch r.URL.String() { - case expected + "/is-repo-cloneable": - encoded, _ := json.Marshal(tc.mockResponse) - body := io.NopCloser(strings.NewReader(strings.TrimSpace(string(encoded)))) - return &http.Response{ - StatusCode: 200, - Body: body, - }, nil - default: - return nil, errors.Newf("unexpected URL: %q", r.URL.String()) - } - })). - WithClientSource(source) - - runTests(t, client, tc) - if called { - t.Fatal("IsRepoCloneable: http client should be called") - } + runTests(t, client, tc) + if !called { + t.Fatal("IsRepoCloneable: grpc client not called") } - }) + } } func TestClient_SystemsInfo(t *testing.T) { @@ -799,110 +535,40 @@ func TestClient_SystemsInfo(t *testing.T) { } } - t.Run("GRPC", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(true), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - - var called atomic.Bool - source := gitserver.NewTestClientSource(t, []string{gitserverAddr1, gitserverAddr2}, func(o *gitserver.TestClientSourceOptions) { - responseByAddress := make(map[string]*proto.DiskInfoResponse, len(expectedResponses)) - for _, response := range expectedResponses { - responseByAddress[response.Address] = &proto.DiskInfoResponse{ - FreeSpace: response.FreeSpace, - TotalSpace: response.TotalSpace, - PercentUsed: response.PercentUsed, - } + var called atomic.Bool + source := gitserver.NewTestClientSource(t, []string{gitserverAddr1, gitserverAddr2}, func(o *gitserver.TestClientSourceOptions) { + responseByAddress := make(map[string]*proto.DiskInfoResponse, len(expectedResponses)) + for _, response := range expectedResponses { + responseByAddress[response.Address] = &proto.DiskInfoResponse{ + FreeSpace: response.FreeSpace, + TotalSpace: response.TotalSpace, + PercentUsed: response.PercentUsed, } - - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockDiskInfo := func(ctx context.Context, in *proto.DiskInfoRequest, opts ...grpc.CallOption) (*proto.DiskInfoResponse, error) { - address := cc.Target() - response, ok := responseByAddress[address] - if !ok { - t.Fatalf("received unexpected address %q", address) - } - - called.Store(true) - return response, nil - } - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.DiskInfoFunc.SetDefaultHook(mockDiskInfo) - return cli - } - }) - - client := gitserver.NewTestClient(t).WithClientSource(source) - - runTest(t, client) - if !called.Load() { - t.Fatal("DiskInfo: grpc client not called") } - }) - t.Run("HTTP", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - - called := false - source := gitserver.NewTestClientSource(t, []string{gitserverAddr1, gitserverAddr2}, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockDiskInfo := func(ctx context.Context, in *proto.DiskInfoRequest, opts ...grpc.CallOption) (*proto.DiskInfoResponse, error) { - called = true - return nil, nil - } - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.DiskInfoFunc.SetDefaultHook(mockDiskInfo) - return cli - } - }) - - client := gitserver.NewTestClient(t). - WithDoer(httpcli.DoerFunc(func(r *http.Request) (*http.Response, error) { - responseByAddress := make(map[string]*proto.DiskInfoResponse, len(expectedResponses)) - for _, response := range expectedResponses { - responseByAddress[fmt.Sprintf("http://%s/disk-info", response.Address)] = &proto.DiskInfoResponse{ - FreeSpace: response.FreeSpace, - TotalSpace: response.TotalSpace, - PercentUsed: response.PercentUsed, - } - } - - address := r.URL.String() + o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { + mockDiskInfo := func(ctx context.Context, in *proto.DiskInfoRequest, opts ...grpc.CallOption) (*proto.DiskInfoResponse, error) { + address := cc.Target() response, ok := responseByAddress[address] if !ok { - return nil, errors.Newf("unexpected URL: %q", address) + t.Fatalf("received unexpected address %q", address) } - encoded, _ := protojson.Marshal(response) - body := io.NopCloser(strings.NewReader(string(encoded))) - return &http.Response{ - StatusCode: 200, - Body: body, - }, nil - })). - WithClientSource(source) - - runTest(t, client) - if called { - t.Fatal("DiskInfo: http client should be called") + called.Store(true) + return response, nil + } + cli := gitserver.NewStrictMockGitserverServiceClient() + cli.DiskInfoFunc.SetDefaultHook(mockDiskInfo) + return cli } }) + + client := gitserver.NewTestClient(t).WithClientSource(source) + + runTest(t, client) + if !called.Load() { + t.Fatal("DiskInfo: grpc client not called") + } } func TestClient_SystemInfo(t *testing.T) { @@ -921,86 +587,25 @@ func TestClient_SystemInfo(t *testing.T) { require.Equal(t, mockResponse.TotalSpace, info.TotalSpace) } - t.Run("GRPC", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(true), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - - called := false - source := gitserver.NewTestClientSource(t, []string{gitserverAddr}, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockDiskInfo := func(ctx context.Context, in *proto.DiskInfoRequest, opts ...grpc.CallOption) (*proto.DiskInfoResponse, error) { - called = true - return mockResponse, nil - } - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.DiskInfoFunc.SetDefaultHook(mockDiskInfo) - return cli + called := false + source := gitserver.NewTestClientSource(t, []string{gitserverAddr}, func(o *gitserver.TestClientSourceOptions) { + o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { + mockDiskInfo := func(ctx context.Context, in *proto.DiskInfoRequest, opts ...grpc.CallOption) (*proto.DiskInfoResponse, error) { + called = true + return mockResponse, nil } - }) - - client := gitserver.NewTestClient(t).WithClientSource(source) - - runTest(t, client, gitserverAddr) - if !called { - t.Fatal("DiskInfo: grpc client not called") + cli := gitserver.NewStrictMockGitserverServiceClient() + cli.DiskInfoFunc.SetDefaultHook(mockDiskInfo) + return cli } }) - t.Run("HTTP", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, - }, - }) - t.Cleanup(func() { - conf.Mock(nil) - }) - expected := fmt.Sprintf("http://%s", gitserverAddr) + client := gitserver.NewTestClient(t).WithClientSource(source) - called := false - source := gitserver.NewTestClientSource(t, []string{gitserverAddr}, func(o *gitserver.TestClientSourceOptions) { - o.ClientFunc = func(cc *grpc.ClientConn) proto.GitserverServiceClient { - mockDiskInfo := func(ctx context.Context, in *proto.DiskInfoRequest, opts ...grpc.CallOption) (*proto.DiskInfoResponse, error) { - called = true - return mockResponse, nil - } - cli := gitserver.NewStrictMockGitserverServiceClient() - cli.DiskInfoFunc.SetDefaultHook(mockDiskInfo) - return cli - } - }) - - client := gitserver.NewTestClient(t). - WithDoer(httpcli.DoerFunc(func(r *http.Request) (*http.Response, error) { - switch r.URL.String() { - case expected + "/disk-info": - encoded, _ := json.Marshal(mockResponse) - body := io.NopCloser(strings.NewReader(strings.TrimSpace(string(encoded)))) - return &http.Response{ - StatusCode: 200, - Body: body, - }, nil - default: - return nil, errors.Newf("unexpected URL: %q", r.URL.String()) - } - })). - WithClientSource(source) - - runTest(t, client, gitserverAddr) - if called { - t.Fatal("DiskInfo: http client should be called") - } - }) + runTest(t, client, gitserverAddr) + if !called { + t.Fatal("DiskInfo: grpc client not called") + } } type fuzzTime time.Time @@ -1014,7 +619,3 @@ func (fuzzTime) Generate(rand *rand.Rand, _ int) reflect.Value { } var _ quick.Generator = fuzzTime{} - -func boolPointer(b bool) *bool { - return &b -} diff --git a/internal/gitserver/commands.go b/internal/gitserver/commands.go index a4f09e85fc8..1dd8cbc8fca 100644 --- a/internal/gitserver/commands.go +++ b/internal/gitserver/commands.go @@ -5,11 +5,9 @@ import ( "bytes" "context" "encoding/hex" - "encoding/json" "fmt" "io" "io/fs" - "net/http" "net/mail" "os" stdlibpath "path" @@ -32,7 +30,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/authz" "github.com/sourcegraph/sourcegraph/internal/byteutils" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/fileutil" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" "github.com/sourcegraph/sourcegraph/internal/gitserver/protocol" @@ -2579,120 +2576,81 @@ func (c *clientImplementor) ArchiveReader( return nil, err } - if conf.IsGRPCEnabled(ctx) { - client, err := c.clientSource.ClientForRepo(ctx, c.userAgent, repo) - if err != nil { - return nil, err - } + client, err := c.clientSource.ClientForRepo(ctx, c.userAgent, repo) + if err != nil { + return nil, err + } - req := options.ToProto(string(repo)) // HACK: ArchiveOptions doesn't have a repository here, so we have to add it ourselves. + req := options.ToProto(string(repo)) // HACK: ArchiveOptions doesn't have a repository here, so we have to add it ourselves. - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := context.WithCancel(ctx) - stream, err := client.Archive(ctx, req) - if err != nil { - cancel() - return nil, err - } + stream, err := client.Archive(ctx, req) + if err != nil { + cancel() + return nil, err + } - // first message from the gRPC stream needs to be read to check for errors before continuing - // to read the rest of the stream. If the first message is an error, we cancel the stream - // and return the error. + // first message from the gRPC stream needs to be read to check for errors before continuing + // to read the rest of the stream. If the first message is an error, we cancel the stream + // and return the error. + // + // This is necessary to provide parity between the REST and gRPC implementations of + // ArchiveReader. Users of cli.ArchiveReader may assume error handling occurs immediately, + // as is the case with the HTTP implementation where errors are returned as soon as the + // function returns. gRPC is asynchronous, so we have to start consuming messages from + // the stream to see any errors from the server. Reading the first message ensures we + // handle any errors synchronously, similar to the HTTP implementation. + + firstMessage, firstError := stream.Recv() + if firstError != nil { + // Hack: The ArchiveReader.Read() implementation handles surfacing the + // any "revision not found" errors returned from the invoked git binary. // - // This is necessary to provide parity between the REST and gRPC implementations of - // ArchiveReader. Users of cli.ArchiveReader may assume error handling occurs immediately, - // as is the case with the HTTP implementation where errors are returned as soon as the - // function returns. gRPC is asynchronous, so we have to start consuming messages from - // the stream to see any errors from the server. Reading the first message ensures we - // handle any errors synchronously, similar to the HTTP implementation. + // In order to maintainparity with the HTTP API, we return this error in the ArchiveReader.Read() method + // instead of returning it immediately. - firstMessage, firstError := stream.Recv() - if firstError != nil { - // Hack: The ArchiveReader.Read() implementation handles surfacing the - // any "revision not found" errors returned from the invoked git binary. - // - // In order to maintainparity with the HTTP API, we return this error in the ArchiveReader.Read() method - // instead of returning it immediately. + // We return early only if this isn't a revision not found error. - // We return early only if this isn't a revision not found error. + err := convertGRPCErrorToGitDomainError(firstError) - err := convertGRPCErrorToGitDomainError(firstError) - - var cse *CommandStatusError - if !errors.As(err, &cse) || !isRevisionNotFound(cse.Stderr) { - cancel() - return nil, convertGRPCErrorToGitDomainError(err) - } - } - - firstMessageRead := false - - // Create a reader to read from the gRPC stream. - r := streamio.NewReader(func() ([]byte, error) { - // Check if we've read the first message yet. If not, read it and return. - if !firstMessageRead { - firstMessageRead = true - - if firstError != nil { - return nil, firstError - } - - return firstMessage.GetData(), nil - } - - // Receive the next message from the stream. - msg, err := stream.Recv() - if err != nil { - return nil, convertGRPCErrorToGitDomainError(err) - } - - // Return the data from the received message. - return msg.GetData(), nil - }) - - return &archiveReader{ - base: &readCloseWrapper{r: r, closeFn: cancel}, - repo: repo, - spec: options.Treeish, - }, nil - - } else { - // Fall back to http request - u := c.archiveURL(ctx, repo, options) - resp, err := c.do(ctx, repo, u.String(), nil) - if err != nil { - return nil, err - } - - switch resp.StatusCode { - case http.StatusOK: - return &archiveReader{ - base: &cmdReader{ - rc: resp.Body, - trailer: resp.Trailer, - }, - repo: repo, - spec: options.Treeish, - }, nil - case http.StatusNotFound: - var payload protocol.NotFoundPayload - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { - resp.Body.Close() - return nil, err - } - resp.Body.Close() - return nil, &badRequestError{ - error: &gitdomain.RepoNotExistError{ - Repo: repo, - CloneInProgress: payload.CloneInProgress, - CloneProgress: payload.CloneProgress, - }, - } - default: - resp.Body.Close() - return nil, errors.Errorf("unexpected status code: %d", resp.StatusCode) + var cse *CommandStatusError + if !errors.As(err, &cse) || !isRevisionNotFound(cse.Stderr) { + cancel() + return nil, convertGRPCErrorToGitDomainError(err) } } + + firstMessageRead := false + + // Create a reader to read from the gRPC stream. + r := streamio.NewReader(func() ([]byte, error) { + // Check if we've read the first message yet. If not, read it and return. + if !firstMessageRead { + firstMessageRead = true + + if firstError != nil { + return nil, firstError + } + + return firstMessage.GetData(), nil + } + + // Receive the next message from the stream. + msg, err := stream.Recv() + if err != nil { + return nil, convertGRPCErrorToGitDomainError(err) + } + + // Return the data from the received message. + return msg.GetData(), nil + }) + + return &archiveReader{ + base: &readCloseWrapper{r: r, closeFn: cancel}, + repo: repo, + spec: options.Treeish, + }, nil } func addNameOnly(opt CommitsOptions, checker authz.SubRepoPermissionChecker) CommitsOptions { diff --git a/internal/gitserver/git_command.go b/internal/gitserver/git_command.go index 79cc868da7d..76996bb5bb5 100644 --- a/internal/gitserver/git_command.go +++ b/internal/gitserver/git_command.go @@ -5,11 +5,9 @@ import ( "context" "fmt" "io" - "net/http" "os" "os/exec" "path/filepath" - "strconv" "strings" "syscall" @@ -176,8 +174,6 @@ type RemoteGitCommand struct { } type execer interface { - httpPost(ctx context.Context, repo api.RepoName, op string, payload any) (resp *http.Response, err error) - AddrForRepo(ctx context.Context, repo api.RepoName) string ClientForRepo(ctx context.Context, repo api.RepoName) (proto.GitserverServiceClient, error) } @@ -251,44 +247,3 @@ func (c *RemoteGitCommand) String() string { return fmt.Sprintf("%q", c.args) } func (c *RemoteGitCommand) StdoutReader(ctx context.Context) (io.ReadCloser, error) { return c.sendExec(ctx) } - -type cmdReader struct { - rc io.ReadCloser - trailer http.Header -} - -func (c *cmdReader) Read(p []byte) (int, error) { - n, err := c.rc.Read(p) - if err == io.EOF { - statusCode, err := strconv.Atoi(c.trailer.Get("X-Exec-Exit-Status")) - if err != nil { - return n, errors.Wrap(err, "failed to parse exit status code") - } - - errorMessage := c.trailer.Get("X-Exec-Error") - - // did the command exit cleanly? - if statusCode == 0 && errorMessage == "" { - // yes - propagate io.EOF - - return n, io.EOF - } - - // no - report it - - stderr := c.trailer.Get("X-Exec-Stderr") - err = &CommandStatusError{ - Stderr: stderr, - StatusCode: int32(statusCode), - Message: errorMessage, - } - - return n, err - } - - return n, err -} - -func (c *cmdReader) Close() error { - return c.rc.Close() -} diff --git a/internal/gitserver/gitolite.go b/internal/gitserver/gitolite.go deleted file mode 100644 index 19c88876404..00000000000 --- a/internal/gitserver/gitolite.go +++ /dev/null @@ -1,85 +0,0 @@ -package gitserver - -import ( - "context" - "encoding/json" - "net/http" - "net/url" - "os" - "path/filepath" - - "github.com/sourcegraph/sourcegraph/internal/conf" - "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" - proto "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" - "github.com/sourcegraph/sourcegraph/internal/httpcli" -) - -type GitoliteLister struct { - addrs func() []string - httpClient httpcli.Doer - grpcClient ClientSource - userAgent string -} - -func NewGitoliteLister(cli httpcli.Doer) *GitoliteLister { - return &GitoliteLister{ - httpClient: cli, - addrs: func() []string { - return conns.get().Addresses - }, - grpcClient: conns, - userAgent: filepath.Base(os.Args[0]), - } -} - -func (c *GitoliteLister) ListRepos(ctx context.Context, gitoliteHost string) (list []*gitolite.Repo, err error) { - addrs := c.addrs() - if len(addrs) == 0 { - panic("unexpected state: no gitserver addresses") - } - if conf.IsGRPCEnabled(ctx) { - - client, err := c.grpcClient.ClientForRepo(ctx, c.userAgent, "") - if err != nil { - return nil, err - } - req := &proto.ListGitoliteRequest{ - GitoliteHost: gitoliteHost, - } - grpcResp, err := client.ListGitolite(ctx, req) - if err != nil { - return nil, err - } - - list = make([]*gitolite.Repo, len(grpcResp.Repos)) - - for i, r := range grpcResp.GetRepos() { - list[i] = &gitolite.Repo{ - Name: r.GetName(), - URL: r.GetUrl(), - } - } - return list, nil - - } else { - // The gitserver calls the shared Gitolite server in response to this request, so - // we need to only call a single gitserver (or else we'd get duplicate results). - addr := addrForKey(gitoliteHost, addrs) - - req, err := http.NewRequest("GET", "http://"+addr+"/list-gitolite?gitolite="+url.QueryEscape(gitoliteHost), nil) - if err != nil { - return nil, err - } - // Set header so that the handler knows the request is from us - req.Header.Set("X-Requested-With", "Sourcegraph") - - resp, err := c.httpClient.Do(req.WithContext(ctx)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - err = json.NewDecoder(resp.Body).Decode(&list) - return list, err - } -} diff --git a/internal/gitserver/mock.go b/internal/gitserver/mock.go index bca8605a690..2a2717ba936 100644 --- a/internal/gitserver/mock.go +++ b/internal/gitserver/mock.go @@ -12,6 +12,7 @@ import ( v1 "github.com/sourcegraph/sourcegraph/internal/gitserver/v1" grpc "google.golang.org/grpc" + metadata "google.golang.org/grpc/metadata" ) // MockGitserverServiceClient is a mock implementation of the @@ -3054,3 +3055,877 @@ func (c GitserverServiceClientSearchFuncCall) Args() []interface{} { func (c GitserverServiceClientSearchFuncCall) Results() []interface{} { return []interface{}{c.Result0, c.Result1} } + +// MockGitserverService_ExecServer is a mock implementation of the +// GitserverService_ExecServer interface (from the package +// github.com/sourcegraph/sourcegraph/internal/gitserver/v1) used for unit +// testing. +type MockGitserverService_ExecServer struct { + // ContextFunc is an instance of a mock function object controlling the + // behavior of the method Context. + ContextFunc *GitserverService_ExecServerContextFunc + // RecvMsgFunc is an instance of a mock function object controlling the + // behavior of the method RecvMsg. + RecvMsgFunc *GitserverService_ExecServerRecvMsgFunc + // SendFunc is an instance of a mock function object controlling the + // behavior of the method Send. + SendFunc *GitserverService_ExecServerSendFunc + // SendHeaderFunc is an instance of a mock function object controlling + // the behavior of the method SendHeader. + SendHeaderFunc *GitserverService_ExecServerSendHeaderFunc + // SendMsgFunc is an instance of a mock function object controlling the + // behavior of the method SendMsg. + SendMsgFunc *GitserverService_ExecServerSendMsgFunc + // SetHeaderFunc is an instance of a mock function object controlling + // the behavior of the method SetHeader. + SetHeaderFunc *GitserverService_ExecServerSetHeaderFunc + // SetTrailerFunc is an instance of a mock function object controlling + // the behavior of the method SetTrailer. + SetTrailerFunc *GitserverService_ExecServerSetTrailerFunc +} + +// NewMockGitserverService_ExecServer creates a new mock of the +// GitserverService_ExecServer interface. All methods return zero values for +// all results, unless overwritten. +func NewMockGitserverService_ExecServer() *MockGitserverService_ExecServer { + return &MockGitserverService_ExecServer{ + ContextFunc: &GitserverService_ExecServerContextFunc{ + defaultHook: func() (r0 context.Context) { + return + }, + }, + RecvMsgFunc: &GitserverService_ExecServerRecvMsgFunc{ + defaultHook: func(interface{}) (r0 error) { + return + }, + }, + SendFunc: &GitserverService_ExecServerSendFunc{ + defaultHook: func(*v1.ExecResponse) (r0 error) { + return + }, + }, + SendHeaderFunc: &GitserverService_ExecServerSendHeaderFunc{ + defaultHook: func(metadata.MD) (r0 error) { + return + }, + }, + SendMsgFunc: &GitserverService_ExecServerSendMsgFunc{ + defaultHook: func(interface{}) (r0 error) { + return + }, + }, + SetHeaderFunc: &GitserverService_ExecServerSetHeaderFunc{ + defaultHook: func(metadata.MD) (r0 error) { + return + }, + }, + SetTrailerFunc: &GitserverService_ExecServerSetTrailerFunc{ + defaultHook: func(metadata.MD) { + return + }, + }, + } +} + +// NewStrictMockGitserverService_ExecServer creates a new mock of the +// GitserverService_ExecServer interface. All methods panic on invocation, +// unless overwritten. +func NewStrictMockGitserverService_ExecServer() *MockGitserverService_ExecServer { + return &MockGitserverService_ExecServer{ + ContextFunc: &GitserverService_ExecServerContextFunc{ + defaultHook: func() context.Context { + panic("unexpected invocation of MockGitserverService_ExecServer.Context") + }, + }, + RecvMsgFunc: &GitserverService_ExecServerRecvMsgFunc{ + defaultHook: func(interface{}) error { + panic("unexpected invocation of MockGitserverService_ExecServer.RecvMsg") + }, + }, + SendFunc: &GitserverService_ExecServerSendFunc{ + defaultHook: func(*v1.ExecResponse) error { + panic("unexpected invocation of MockGitserverService_ExecServer.Send") + }, + }, + SendHeaderFunc: &GitserverService_ExecServerSendHeaderFunc{ + defaultHook: func(metadata.MD) error { + panic("unexpected invocation of MockGitserverService_ExecServer.SendHeader") + }, + }, + SendMsgFunc: &GitserverService_ExecServerSendMsgFunc{ + defaultHook: func(interface{}) error { + panic("unexpected invocation of MockGitserverService_ExecServer.SendMsg") + }, + }, + SetHeaderFunc: &GitserverService_ExecServerSetHeaderFunc{ + defaultHook: func(metadata.MD) error { + panic("unexpected invocation of MockGitserverService_ExecServer.SetHeader") + }, + }, + SetTrailerFunc: &GitserverService_ExecServerSetTrailerFunc{ + defaultHook: func(metadata.MD) { + panic("unexpected invocation of MockGitserverService_ExecServer.SetTrailer") + }, + }, + } +} + +// NewMockGitserverService_ExecServerFrom creates a new mock of the +// MockGitserverService_ExecServer interface. All methods delegate to the +// given implementation, unless overwritten. +func NewMockGitserverService_ExecServerFrom(i v1.GitserverService_ExecServer) *MockGitserverService_ExecServer { + return &MockGitserverService_ExecServer{ + ContextFunc: &GitserverService_ExecServerContextFunc{ + defaultHook: i.Context, + }, + RecvMsgFunc: &GitserverService_ExecServerRecvMsgFunc{ + defaultHook: i.RecvMsg, + }, + SendFunc: &GitserverService_ExecServerSendFunc{ + defaultHook: i.Send, + }, + SendHeaderFunc: &GitserverService_ExecServerSendHeaderFunc{ + defaultHook: i.SendHeader, + }, + SendMsgFunc: &GitserverService_ExecServerSendMsgFunc{ + defaultHook: i.SendMsg, + }, + SetHeaderFunc: &GitserverService_ExecServerSetHeaderFunc{ + defaultHook: i.SetHeader, + }, + SetTrailerFunc: &GitserverService_ExecServerSetTrailerFunc{ + defaultHook: i.SetTrailer, + }, + } +} + +// GitserverService_ExecServerContextFunc describes the behavior when the +// Context method of the parent MockGitserverService_ExecServer instance is +// invoked. +type GitserverService_ExecServerContextFunc struct { + defaultHook func() context.Context + hooks []func() context.Context + history []GitserverService_ExecServerContextFuncCall + mutex sync.Mutex +} + +// Context delegates to the next hook function in the queue and stores the +// parameter and result values of this invocation. +func (m *MockGitserverService_ExecServer) Context() context.Context { + r0 := m.ContextFunc.nextHook()() + m.ContextFunc.appendCall(GitserverService_ExecServerContextFuncCall{r0}) + return r0 +} + +// SetDefaultHook sets function that is called when the Context method of +// the parent MockGitserverService_ExecServer instance is invoked and the +// hook queue is empty. +func (f *GitserverService_ExecServerContextFunc) SetDefaultHook(hook func() context.Context) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// Context method of the parent MockGitserverService_ExecServer instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverService_ExecServerContextFunc) PushHook(hook func() context.Context) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverService_ExecServerContextFunc) SetDefaultReturn(r0 context.Context) { + f.SetDefaultHook(func() context.Context { + return r0 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverService_ExecServerContextFunc) PushReturn(r0 context.Context) { + f.PushHook(func() context.Context { + return r0 + }) +} + +func (f *GitserverService_ExecServerContextFunc) nextHook() func() context.Context { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverService_ExecServerContextFunc) appendCall(r0 GitserverService_ExecServerContextFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of GitserverService_ExecServerContextFuncCall +// objects describing the invocations of this function. +func (f *GitserverService_ExecServerContextFunc) History() []GitserverService_ExecServerContextFuncCall { + f.mutex.Lock() + history := make([]GitserverService_ExecServerContextFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverService_ExecServerContextFuncCall is an object that describes an +// invocation of method Context on an instance of +// MockGitserverService_ExecServer. +type GitserverService_ExecServerContextFuncCall struct { + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 context.Context +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverService_ExecServerContextFuncCall) Args() []interface{} { + return []interface{}{} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverService_ExecServerContextFuncCall) Results() []interface{} { + return []interface{}{c.Result0} +} + +// GitserverService_ExecServerRecvMsgFunc describes the behavior when the +// RecvMsg method of the parent MockGitserverService_ExecServer instance is +// invoked. +type GitserverService_ExecServerRecvMsgFunc struct { + defaultHook func(interface{}) error + hooks []func(interface{}) error + history []GitserverService_ExecServerRecvMsgFuncCall + mutex sync.Mutex +} + +// RecvMsg delegates to the next hook function in the queue and stores the +// parameter and result values of this invocation. +func (m *MockGitserverService_ExecServer) RecvMsg(v0 interface{}) error { + r0 := m.RecvMsgFunc.nextHook()(v0) + m.RecvMsgFunc.appendCall(GitserverService_ExecServerRecvMsgFuncCall{v0, r0}) + return r0 +} + +// SetDefaultHook sets function that is called when the RecvMsg method of +// the parent MockGitserverService_ExecServer instance is invoked and the +// hook queue is empty. +func (f *GitserverService_ExecServerRecvMsgFunc) SetDefaultHook(hook func(interface{}) error) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// RecvMsg method of the parent MockGitserverService_ExecServer instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverService_ExecServerRecvMsgFunc) PushHook(hook func(interface{}) error) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverService_ExecServerRecvMsgFunc) SetDefaultReturn(r0 error) { + f.SetDefaultHook(func(interface{}) error { + return r0 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverService_ExecServerRecvMsgFunc) PushReturn(r0 error) { + f.PushHook(func(interface{}) error { + return r0 + }) +} + +func (f *GitserverService_ExecServerRecvMsgFunc) nextHook() func(interface{}) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverService_ExecServerRecvMsgFunc) appendCall(r0 GitserverService_ExecServerRecvMsgFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of GitserverService_ExecServerRecvMsgFuncCall +// objects describing the invocations of this function. +func (f *GitserverService_ExecServerRecvMsgFunc) History() []GitserverService_ExecServerRecvMsgFuncCall { + f.mutex.Lock() + history := make([]GitserverService_ExecServerRecvMsgFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverService_ExecServerRecvMsgFuncCall is an object that describes an +// invocation of method RecvMsg on an instance of +// MockGitserverService_ExecServer. +type GitserverService_ExecServerRecvMsgFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 interface{} + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverService_ExecServerRecvMsgFuncCall) Args() []interface{} { + return []interface{}{c.Arg0} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverService_ExecServerRecvMsgFuncCall) Results() []interface{} { + return []interface{}{c.Result0} +} + +// GitserverService_ExecServerSendFunc describes the behavior when the Send +// method of the parent MockGitserverService_ExecServer instance is invoked. +type GitserverService_ExecServerSendFunc struct { + defaultHook func(*v1.ExecResponse) error + hooks []func(*v1.ExecResponse) error + history []GitserverService_ExecServerSendFuncCall + mutex sync.Mutex +} + +// Send delegates to the next hook function in the queue and stores the +// parameter and result values of this invocation. +func (m *MockGitserverService_ExecServer) Send(v0 *v1.ExecResponse) error { + r0 := m.SendFunc.nextHook()(v0) + m.SendFunc.appendCall(GitserverService_ExecServerSendFuncCall{v0, r0}) + return r0 +} + +// SetDefaultHook sets function that is called when the Send method of the +// parent MockGitserverService_ExecServer instance is invoked and the hook +// queue is empty. +func (f *GitserverService_ExecServerSendFunc) SetDefaultHook(hook func(*v1.ExecResponse) error) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// Send method of the parent MockGitserverService_ExecServer instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverService_ExecServerSendFunc) PushHook(hook func(*v1.ExecResponse) error) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverService_ExecServerSendFunc) SetDefaultReturn(r0 error) { + f.SetDefaultHook(func(*v1.ExecResponse) error { + return r0 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverService_ExecServerSendFunc) PushReturn(r0 error) { + f.PushHook(func(*v1.ExecResponse) error { + return r0 + }) +} + +func (f *GitserverService_ExecServerSendFunc) nextHook() func(*v1.ExecResponse) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverService_ExecServerSendFunc) appendCall(r0 GitserverService_ExecServerSendFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of GitserverService_ExecServerSendFuncCall +// objects describing the invocations of this function. +func (f *GitserverService_ExecServerSendFunc) History() []GitserverService_ExecServerSendFuncCall { + f.mutex.Lock() + history := make([]GitserverService_ExecServerSendFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverService_ExecServerSendFuncCall is an object that describes an +// invocation of method Send on an instance of +// MockGitserverService_ExecServer. +type GitserverService_ExecServerSendFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 *v1.ExecResponse + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverService_ExecServerSendFuncCall) Args() []interface{} { + return []interface{}{c.Arg0} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverService_ExecServerSendFuncCall) Results() []interface{} { + return []interface{}{c.Result0} +} + +// GitserverService_ExecServerSendHeaderFunc describes the behavior when the +// SendHeader method of the parent MockGitserverService_ExecServer instance +// is invoked. +type GitserverService_ExecServerSendHeaderFunc struct { + defaultHook func(metadata.MD) error + hooks []func(metadata.MD) error + history []GitserverService_ExecServerSendHeaderFuncCall + mutex sync.Mutex +} + +// SendHeader delegates to the next hook function in the queue and stores +// the parameter and result values of this invocation. +func (m *MockGitserverService_ExecServer) SendHeader(v0 metadata.MD) error { + r0 := m.SendHeaderFunc.nextHook()(v0) + m.SendHeaderFunc.appendCall(GitserverService_ExecServerSendHeaderFuncCall{v0, r0}) + return r0 +} + +// SetDefaultHook sets function that is called when the SendHeader method of +// the parent MockGitserverService_ExecServer instance is invoked and the +// hook queue is empty. +func (f *GitserverService_ExecServerSendHeaderFunc) SetDefaultHook(hook func(metadata.MD) error) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// SendHeader method of the parent MockGitserverService_ExecServer instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverService_ExecServerSendHeaderFunc) PushHook(hook func(metadata.MD) error) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverService_ExecServerSendHeaderFunc) SetDefaultReturn(r0 error) { + f.SetDefaultHook(func(metadata.MD) error { + return r0 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverService_ExecServerSendHeaderFunc) PushReturn(r0 error) { + f.PushHook(func(metadata.MD) error { + return r0 + }) +} + +func (f *GitserverService_ExecServerSendHeaderFunc) nextHook() func(metadata.MD) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverService_ExecServerSendHeaderFunc) appendCall(r0 GitserverService_ExecServerSendHeaderFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of +// GitserverService_ExecServerSendHeaderFuncCall objects describing the +// invocations of this function. +func (f *GitserverService_ExecServerSendHeaderFunc) History() []GitserverService_ExecServerSendHeaderFuncCall { + f.mutex.Lock() + history := make([]GitserverService_ExecServerSendHeaderFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverService_ExecServerSendHeaderFuncCall is an object that describes +// an invocation of method SendHeader on an instance of +// MockGitserverService_ExecServer. +type GitserverService_ExecServerSendHeaderFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 metadata.MD + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverService_ExecServerSendHeaderFuncCall) Args() []interface{} { + return []interface{}{c.Arg0} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverService_ExecServerSendHeaderFuncCall) Results() []interface{} { + return []interface{}{c.Result0} +} + +// GitserverService_ExecServerSendMsgFunc describes the behavior when the +// SendMsg method of the parent MockGitserverService_ExecServer instance is +// invoked. +type GitserverService_ExecServerSendMsgFunc struct { + defaultHook func(interface{}) error + hooks []func(interface{}) error + history []GitserverService_ExecServerSendMsgFuncCall + mutex sync.Mutex +} + +// SendMsg delegates to the next hook function in the queue and stores the +// parameter and result values of this invocation. +func (m *MockGitserverService_ExecServer) SendMsg(v0 interface{}) error { + r0 := m.SendMsgFunc.nextHook()(v0) + m.SendMsgFunc.appendCall(GitserverService_ExecServerSendMsgFuncCall{v0, r0}) + return r0 +} + +// SetDefaultHook sets function that is called when the SendMsg method of +// the parent MockGitserverService_ExecServer instance is invoked and the +// hook queue is empty. +func (f *GitserverService_ExecServerSendMsgFunc) SetDefaultHook(hook func(interface{}) error) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// SendMsg method of the parent MockGitserverService_ExecServer instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverService_ExecServerSendMsgFunc) PushHook(hook func(interface{}) error) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverService_ExecServerSendMsgFunc) SetDefaultReturn(r0 error) { + f.SetDefaultHook(func(interface{}) error { + return r0 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverService_ExecServerSendMsgFunc) PushReturn(r0 error) { + f.PushHook(func(interface{}) error { + return r0 + }) +} + +func (f *GitserverService_ExecServerSendMsgFunc) nextHook() func(interface{}) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverService_ExecServerSendMsgFunc) appendCall(r0 GitserverService_ExecServerSendMsgFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of GitserverService_ExecServerSendMsgFuncCall +// objects describing the invocations of this function. +func (f *GitserverService_ExecServerSendMsgFunc) History() []GitserverService_ExecServerSendMsgFuncCall { + f.mutex.Lock() + history := make([]GitserverService_ExecServerSendMsgFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverService_ExecServerSendMsgFuncCall is an object that describes an +// invocation of method SendMsg on an instance of +// MockGitserverService_ExecServer. +type GitserverService_ExecServerSendMsgFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 interface{} + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverService_ExecServerSendMsgFuncCall) Args() []interface{} { + return []interface{}{c.Arg0} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverService_ExecServerSendMsgFuncCall) Results() []interface{} { + return []interface{}{c.Result0} +} + +// GitserverService_ExecServerSetHeaderFunc describes the behavior when the +// SetHeader method of the parent MockGitserverService_ExecServer instance +// is invoked. +type GitserverService_ExecServerSetHeaderFunc struct { + defaultHook func(metadata.MD) error + hooks []func(metadata.MD) error + history []GitserverService_ExecServerSetHeaderFuncCall + mutex sync.Mutex +} + +// SetHeader delegates to the next hook function in the queue and stores the +// parameter and result values of this invocation. +func (m *MockGitserverService_ExecServer) SetHeader(v0 metadata.MD) error { + r0 := m.SetHeaderFunc.nextHook()(v0) + m.SetHeaderFunc.appendCall(GitserverService_ExecServerSetHeaderFuncCall{v0, r0}) + return r0 +} + +// SetDefaultHook sets function that is called when the SetHeader method of +// the parent MockGitserverService_ExecServer instance is invoked and the +// hook queue is empty. +func (f *GitserverService_ExecServerSetHeaderFunc) SetDefaultHook(hook func(metadata.MD) error) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// SetHeader method of the parent MockGitserverService_ExecServer instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverService_ExecServerSetHeaderFunc) PushHook(hook func(metadata.MD) error) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverService_ExecServerSetHeaderFunc) SetDefaultReturn(r0 error) { + f.SetDefaultHook(func(metadata.MD) error { + return r0 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverService_ExecServerSetHeaderFunc) PushReturn(r0 error) { + f.PushHook(func(metadata.MD) error { + return r0 + }) +} + +func (f *GitserverService_ExecServerSetHeaderFunc) nextHook() func(metadata.MD) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverService_ExecServerSetHeaderFunc) appendCall(r0 GitserverService_ExecServerSetHeaderFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of +// GitserverService_ExecServerSetHeaderFuncCall objects describing the +// invocations of this function. +func (f *GitserverService_ExecServerSetHeaderFunc) History() []GitserverService_ExecServerSetHeaderFuncCall { + f.mutex.Lock() + history := make([]GitserverService_ExecServerSetHeaderFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverService_ExecServerSetHeaderFuncCall is an object that describes +// an invocation of method SetHeader on an instance of +// MockGitserverService_ExecServer. +type GitserverService_ExecServerSetHeaderFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 metadata.MD + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverService_ExecServerSetHeaderFuncCall) Args() []interface{} { + return []interface{}{c.Arg0} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverService_ExecServerSetHeaderFuncCall) Results() []interface{} { + return []interface{}{c.Result0} +} + +// GitserverService_ExecServerSetTrailerFunc describes the behavior when the +// SetTrailer method of the parent MockGitserverService_ExecServer instance +// is invoked. +type GitserverService_ExecServerSetTrailerFunc struct { + defaultHook func(metadata.MD) + hooks []func(metadata.MD) + history []GitserverService_ExecServerSetTrailerFuncCall + mutex sync.Mutex +} + +// SetTrailer delegates to the next hook function in the queue and stores +// the parameter and result values of this invocation. +func (m *MockGitserverService_ExecServer) SetTrailer(v0 metadata.MD) { + m.SetTrailerFunc.nextHook()(v0) + m.SetTrailerFunc.appendCall(GitserverService_ExecServerSetTrailerFuncCall{v0}) + return +} + +// SetDefaultHook sets function that is called when the SetTrailer method of +// the parent MockGitserverService_ExecServer instance is invoked and the +// hook queue is empty. +func (f *GitserverService_ExecServerSetTrailerFunc) SetDefaultHook(hook func(metadata.MD)) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// SetTrailer method of the parent MockGitserverService_ExecServer instance +// invokes the hook at the front of the queue and discards it. After the +// queue is empty, the default hook function is invoked for any future +// action. +func (f *GitserverService_ExecServerSetTrailerFunc) PushHook(hook func(metadata.MD)) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *GitserverService_ExecServerSetTrailerFunc) SetDefaultReturn() { + f.SetDefaultHook(func(metadata.MD) { + return + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *GitserverService_ExecServerSetTrailerFunc) PushReturn() { + f.PushHook(func(metadata.MD) { + return + }) +} + +func (f *GitserverService_ExecServerSetTrailerFunc) nextHook() func(metadata.MD) { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *GitserverService_ExecServerSetTrailerFunc) appendCall(r0 GitserverService_ExecServerSetTrailerFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of +// GitserverService_ExecServerSetTrailerFuncCall objects describing the +// invocations of this function. +func (f *GitserverService_ExecServerSetTrailerFunc) History() []GitserverService_ExecServerSetTrailerFuncCall { + f.mutex.Lock() + history := make([]GitserverService_ExecServerSetTrailerFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// GitserverService_ExecServerSetTrailerFuncCall is an object that describes +// an invocation of method SetTrailer on an instance of +// MockGitserverService_ExecServer. +type GitserverService_ExecServerSetTrailerFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 metadata.MD +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c GitserverService_ExecServerSetTrailerFuncCall) Args() []interface{} { + return []interface{}{c.Arg0} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c GitserverService_ExecServerSetTrailerFuncCall) Results() []interface{} { + return []interface{}{} +} diff --git a/internal/gitserver/mocks_temp.go b/internal/gitserver/mocks_temp.go index af3b99fd250..c7202fc4863 100644 --- a/internal/gitserver/mocks_temp.go +++ b/internal/gitserver/mocks_temp.go @@ -15,6 +15,7 @@ import ( diff "github.com/sourcegraph/go-diff/diff" api "github.com/sourcegraph/sourcegraph/internal/api" + gitolite "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" gitdomain "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" protocol "github.com/sourcegraph/sourcegraph/internal/gitserver/protocol" perforce "github.com/sourcegraph/sourcegraph/internal/perforce" @@ -117,6 +118,9 @@ type MockClient struct { // ListDirectoryChildrenFunc is an instance of a mock function object // controlling the behavior of the method ListDirectoryChildren. ListDirectoryChildrenFunc *ClientListDirectoryChildrenFunc + // ListGitoliteReposFunc is an instance of a mock function object + // controlling the behavior of the method ListGitoliteRepos. + ListGitoliteReposFunc *ClientListGitoliteReposFunc // ListRefsFunc is an instance of a mock function object controlling the // behavior of the method ListRefs. ListRefsFunc *ClientListRefsFunc @@ -359,6 +363,11 @@ func NewMockClient() *MockClient { return }, }, + ListGitoliteReposFunc: &ClientListGitoliteReposFunc{ + defaultHook: func(context.Context, string) (r0 []*gitolite.Repo, r1 error) { + return + }, + }, ListRefsFunc: &ClientListRefsFunc{ defaultHook: func(context.Context, api.RepoName) (r0 []gitdomain.Ref, r1 error) { return @@ -656,6 +665,11 @@ func NewStrictMockClient() *MockClient { panic("unexpected invocation of MockClient.ListDirectoryChildren") }, }, + ListGitoliteReposFunc: &ClientListGitoliteReposFunc{ + defaultHook: func(context.Context, string) ([]*gitolite.Repo, error) { + panic("unexpected invocation of MockClient.ListGitoliteRepos") + }, + }, ListRefsFunc: &ClientListRefsFunc{ defaultHook: func(context.Context, api.RepoName) ([]gitdomain.Ref, error) { panic("unexpected invocation of MockClient.ListRefs") @@ -891,6 +905,9 @@ func NewMockClientFrom(i Client) *MockClient { ListDirectoryChildrenFunc: &ClientListDirectoryChildrenFunc{ defaultHook: i.ListDirectoryChildren, }, + ListGitoliteReposFunc: &ClientListGitoliteReposFunc{ + defaultHook: i.ListGitoliteRepos, + }, ListRefsFunc: &ClientListRefsFunc{ defaultHook: i.ListRefs, }, @@ -4387,6 +4404,114 @@ func (c ClientListDirectoryChildrenFuncCall) Results() []interface{} { return []interface{}{c.Result0, c.Result1} } +// ClientListGitoliteReposFunc describes the behavior when the +// ListGitoliteRepos method of the parent MockClient instance is invoked. +type ClientListGitoliteReposFunc struct { + defaultHook func(context.Context, string) ([]*gitolite.Repo, error) + hooks []func(context.Context, string) ([]*gitolite.Repo, error) + history []ClientListGitoliteReposFuncCall + mutex sync.Mutex +} + +// ListGitoliteRepos delegates to the next hook function in the queue and +// stores the parameter and result values of this invocation. +func (m *MockClient) ListGitoliteRepos(v0 context.Context, v1 string) ([]*gitolite.Repo, error) { + r0, r1 := m.ListGitoliteReposFunc.nextHook()(v0, v1) + m.ListGitoliteReposFunc.appendCall(ClientListGitoliteReposFuncCall{v0, v1, r0, r1}) + return r0, r1 +} + +// SetDefaultHook sets function that is called when the ListGitoliteRepos +// method of the parent MockClient instance is invoked and the hook queue is +// empty. +func (f *ClientListGitoliteReposFunc) SetDefaultHook(hook func(context.Context, string) ([]*gitolite.Repo, error)) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// ListGitoliteRepos method of the parent MockClient instance invokes the +// hook at the front of the queue and discards it. After the queue is empty, +// the default hook function is invoked for any future action. +func (f *ClientListGitoliteReposFunc) PushHook(hook func(context.Context, string) ([]*gitolite.Repo, error)) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *ClientListGitoliteReposFunc) SetDefaultReturn(r0 []*gitolite.Repo, r1 error) { + f.SetDefaultHook(func(context.Context, string) ([]*gitolite.Repo, error) { + return r0, r1 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *ClientListGitoliteReposFunc) PushReturn(r0 []*gitolite.Repo, r1 error) { + f.PushHook(func(context.Context, string) ([]*gitolite.Repo, error) { + return r0, r1 + }) +} + +func (f *ClientListGitoliteReposFunc) nextHook() func(context.Context, string) ([]*gitolite.Repo, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *ClientListGitoliteReposFunc) appendCall(r0 ClientListGitoliteReposFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of ClientListGitoliteReposFuncCall objects +// describing the invocations of this function. +func (f *ClientListGitoliteReposFunc) History() []ClientListGitoliteReposFuncCall { + f.mutex.Lock() + history := make([]ClientListGitoliteReposFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// ClientListGitoliteReposFuncCall is an object that describes an invocation +// of method ListGitoliteRepos on an instance of MockClient. +type ClientListGitoliteReposFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 context.Context + // Arg1 is the value of the 2nd argument passed to this method + // invocation. + Arg1 string + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 []*gitolite.Repo + // Result1 is the value of the 2nd result returned from this method + // invocation. + Result1 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c ClientListGitoliteReposFuncCall) Args() []interface{} { + return []interface{}{c.Arg0, c.Arg1} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c ClientListGitoliteReposFuncCall) Results() []interface{} { + return []interface{}{c.Result0, c.Result1} +} + // ClientListRefsFunc describes the behavior when the ListRefs method of the // parent MockClient instance is invoked. type ClientListRefsFunc struct { diff --git a/internal/gitserver/proxy.go b/internal/gitserver/proxy.go deleted file mode 100644 index 7f0971884f9..00000000000 --- a/internal/gitserver/proxy.go +++ /dev/null @@ -1,56 +0,0 @@ -package gitserver - -import ( - "net/http" - "net/http/httputil" - - "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/limiter" - "github.com/sourcegraph/sourcegraph/internal/trace" - "go.opentelemetry.io/otel/attribute" -) - -// DefaultReverseProxy is the default ReverseProxy. It uses the same transport and HTTP -// limiter as the default client. -var DefaultReverseProxy = NewReverseProxy(defaultClient.Transport, defaultLimiter) - -var defaultClient, _ = clientFactory.Client() - -// NewReverseProxy returns a new gitserver.ReverseProxy instantiated with the given -// transport and HTTP limiter. -func NewReverseProxy(transport http.RoundTripper, httpLimiter limiter.Limiter) *ReverseProxy { - return &ReverseProxy{ - Transport: transport, - HTTPLimiter: httpLimiter, - } -} - -// ReverseProxy is a gitserver reverse proxy. -type ReverseProxy struct { - Transport http.RoundTripper - - // Limits concurrency of outstanding HTTP posts - HTTPLimiter limiter.Limiter -} - -// ServeHTTP creates a one-shot proxy with the given director and proxies the given request -// to gitserver. The director must rewrite the request to the correct gitserver address, which -// should be obtained via a gitserver client's AddrForRepo method. -func (p *ReverseProxy) ServeHTTP(repo api.RepoName, method, op string, director func(req *http.Request), res http.ResponseWriter, req *http.Request) { - tr, _ := trace.New(req.Context(), "ReverseProxy.ServeHTTP", - repo.Attr(), - attribute.String("method", method), - attribute.String("op", op)) - defer tr.End() - - p.HTTPLimiter.Acquire() - defer p.HTTPLimiter.Release() - tr.AddEvent("Acquired HTTP limiter") - - proxy := &httputil.ReverseProxy{ - Director: director, - Transport: p.Transport, - } - - proxy.ServeHTTP(res, req) -} diff --git a/internal/gitserver/search/search_test.go b/internal/gitserver/search/search_test.go index afcd19adeaf..5c317fd627d 100644 --- a/internal/gitserver/search/search_test.go +++ b/internal/gitserver/search/search_test.go @@ -306,7 +306,7 @@ func TestCommitScanner(t *testing.T) { cmd := exec.Command("git", (&CommitSearcher{IncludeModifiedFiles: includeModifiedFiles}).gitArgs()...) cmd.Dir = dir cmd.Stdout = &buf - cmd.Run() + require.NoError(t, cmd.Run()) return buf.Bytes() } diff --git a/internal/repos/BUILD.bazel b/internal/repos/BUILD.bazel index 33eb6ac428f..6f13dc39089 100644 --- a/internal/repos/BUILD.bazel +++ b/internal/repos/BUILD.bazel @@ -189,6 +189,7 @@ go_test( "//lib/errors", "//lib/pointers", "//schema", + "@com_github_derision_test_go_mockgen//testutil/require", "@com_github_google_go_cmp//cmp", "@com_github_google_go_cmp//cmp/cmpopts", "@com_github_google_uuid//:uuid", diff --git a/internal/repos/gitolite.go b/internal/repos/gitolite.go index 2e0693ffbe8..26e56230698 100644 --- a/internal/repos/gitolite.go +++ b/internal/repos/gitolite.go @@ -8,7 +8,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/conf/reposource" "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" "github.com/sourcegraph/sourcegraph/internal/gitserver" - "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/jsonc" "github.com/sourcegraph/sourcegraph/internal/types" "github.com/sourcegraph/sourcegraph/lib/errors" @@ -21,15 +20,11 @@ type GitoliteSource struct { svc *types.ExternalService conn *schema.GitoliteConnection excluder repoExcluder - - // gitoliteLister allows us to list Gitlolite repos. In practice, we ask - // gitserver to talk to gitolite because it holds the ssh keys required for - // authentication. - lister *gitserver.GitoliteLister + gc gitserver.Client } // NewGitoliteSource returns a new GitoliteSource from the given external service. -func NewGitoliteSource(ctx context.Context, svc *types.ExternalService, cf *httpcli.Factory) (*GitoliteSource, error) { +func NewGitoliteSource(ctx context.Context, svc *types.ExternalService, gc gitserver.Client) (*GitoliteSource, error) { rawConfig, err := svc.Config.Decrypt(ctx) if err != nil { return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err) @@ -39,17 +34,6 @@ func NewGitoliteSource(ctx context.Context, svc *types.ExternalService, cf *http return nil, errors.Wrapf(err, "external service id=%d config error", svc.ID) } - gitoliteDoer, err := cf.Doer( - httpcli.NewMaxIdleConnsPerHostOpt(500), - // The provided httpcli.Factory is one used for external services - however, - // GitoliteSource asks gitserver to communicate to gitolite instead, so we - // have to ensure that the actor transport used for internal clients is provided. - httpcli.ActorTransportOpt, - ) - if err != nil { - return nil, err - } - var ex repoExcluder for _, r := range c.Exclude { ex.AddRule(). @@ -60,12 +44,10 @@ func NewGitoliteSource(ctx context.Context, svc *types.ExternalService, cf *http return nil, err } - lister := gitserver.NewGitoliteLister(gitoliteDoer) - return &GitoliteSource{ svc: svc, conn: &c, - lister: lister, + gc: gc.Scoped("repos.gitolite"), excluder: ex, }, nil } @@ -80,7 +62,7 @@ func (s *GitoliteSource) CheckConnection(ctx context.Context) error { // ListRepos returns all Gitolite repositories accessible to all connections configured // in Sourcegraph via the external services configuration. func (s *GitoliteSource) ListRepos(ctx context.Context, results chan SourceResult) { - all, err := s.lister.ListRepos(ctx, s.conn.Host) + all, err := s.gc.ListGitoliteRepos(ctx, s.conn.Host) if err != nil { results <- SourceResult{Source: s, Err: err} return diff --git a/internal/repos/gitolite_test.go b/internal/repos/gitolite_test.go index c2d1fd5c2ec..e7eeb01bfad 100644 --- a/internal/repos/gitolite_test.go +++ b/internal/repos/gitolite_test.go @@ -4,21 +4,33 @@ import ( "context" "testing" + mockrequire "github.com/derision-test/go-mockgen/testutil/require" + "github.com/sourcegraph/sourcegraph/internal/extsvc" - "github.com/sourcegraph/sourcegraph/internal/httpcli" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/types/typestest" "github.com/sourcegraph/sourcegraph/schema" ) func TestGitoliteSource(t *testing.T) { - cf, save := newClientFactoryWithOpt(t, "basic", httpcli.ExternalTransportOpt) - defer save(t) - + gc := gitserver.NewMockClient() + gc.ScopedFunc.SetDefaultReturn(gc) svc := typestest.MakeExternalService(t, extsvc.VariantGitolite, &schema.GitoliteConnection{}) ctx := context.Background() - _, err := NewGitoliteSource(ctx, svc, cf) + s, err := NewGitoliteSource(ctx, svc, gc) if err != nil { t.Fatal(err) } + + res := make(chan SourceResult) + go func() { + s.ListRepos(ctx, res) + close(res) + }() + + for range res { + } + + mockrequire.Called(t, gc.ListGitoliteReposFunc) } diff --git a/internal/repos/sources.go b/internal/repos/sources.go index 5a9bdf9a6a4..bb3c1237718 100644 --- a/internal/repos/sources.go +++ b/internal/repos/sources.go @@ -11,6 +11,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/extsvc" "github.com/sourcegraph/sourcegraph/internal/extsvc/auth" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/types" "github.com/sourcegraph/sourcegraph/lib/errors" @@ -25,9 +26,9 @@ type Sourcer func(context.Context, *types.ExternalService) (Source, error) // http.Clients needed to contact the respective upstream code host APIs. // // The provided decorator functions will be applied to the Source. -func NewSourcer(logger log.Logger, db database.DB, cf *httpcli.Factory, decs ...func(Source) Source) Sourcer { +func NewSourcer(logger log.Logger, db database.DB, cf *httpcli.Factory, gc gitserver.Client, decs ...func(Source) Source) Sourcer { return func(ctx context.Context, svc *types.ExternalService) (Source, error) { - src, err := NewSource(ctx, logger.Scoped("source"), db, svc, cf) + src, err := NewSource(ctx, logger.Scoped("source"), db, svc, cf, gc) if err != nil { return nil, err } @@ -41,7 +42,10 @@ func NewSourcer(logger log.Logger, db database.DB, cf *httpcli.Factory, decs ... } // NewSource returns a repository yielding Source from the given ExternalService configuration. -func NewSource(ctx context.Context, logger log.Logger, db database.DB, svc *types.ExternalService, cf *httpcli.Factory) (Source, error) { +func NewSource(ctx context.Context, logger log.Logger, db database.DB, svc *types.ExternalService, cf *httpcli.Factory, gc gitserver.Client) (Source, error) { + if gc == nil { + gc = gitserver.NewClient("repos.sourcer") + } switch strings.ToUpper(svc.Kind) { case extsvc.KindGitHub: return NewGitHubSource(ctx, logger.Scoped("GithubSource"), db, svc, cf) @@ -56,7 +60,7 @@ func NewSource(ctx context.Context, logger log.Logger, db database.DB, svc *type case extsvc.KindBitbucketCloud: return NewBitbucketCloudSource(ctx, logger.Scoped("BitbucketCloudSource"), svc, cf) case extsvc.KindGitolite: - return NewGitoliteSource(ctx, svc, cf) + return NewGitoliteSource(ctx, svc, gc) case extsvc.KindPhabricator: return NewPhabricatorSource(ctx, logger.Scoped("PhabricatorSource"), svc, cf) case extsvc.KindAWSCodeCommit: diff --git a/internal/repos/sources_test.go b/internal/repos/sources_test.go index 3aff794dda2..1ec2f92b025 100644 --- a/internal/repos/sources_test.go +++ b/internal/repos/sources_test.go @@ -17,7 +17,9 @@ import ( "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/database/dbmocks" "github.com/sourcegraph/sourcegraph/internal/extsvc" + "github.com/sourcegraph/sourcegraph/internal/extsvc/gitolite" "github.com/sourcegraph/sourcegraph/internal/extsvc/phabricator" + "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/httptestutil" "github.com/sourcegraph/sourcegraph/internal/ratelimit" @@ -121,7 +123,7 @@ func TestSources_ListRepos_YieldExistingRepos(t *testing.T) { cf, save := NewClientFactory(t, name) defer save(t) - repos := listRepos(t, cf, tc.svc) + repos := listRepos(t, cf, nil, tc.svc) var haveNames []string for _, r := range repos { @@ -140,10 +142,6 @@ func TestSources_ListRepos_Excluded(t *testing.T) { conf.Mock(&conf.Unified{ ServiceConnectionConfig: conftypes.ServiceConnections{ GitServers: []string{"127.0.0.1:3178"}, - }, SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, }, }) defer conf.Mock(nil) @@ -151,9 +149,18 @@ func TestSources_ListRepos_Excluded(t *testing.T) { rcache.SetupForTest(t) ratelimit.SetupForTest(t) + gitoliteGs := gitserver.NewMockClient() + gitoliteGs.ScopedFunc.SetDefaultReturn(gitoliteGs) + gitoliteGs.ListGitoliteReposFunc.SetDefaultReturn([]*gitolite.Repo{ + {Name: "baz", URL: "gitolite.mycorp.com/baz.git"}, + {Name: "foo", URL: "gitolite.mycorp.com/foo.git"}, + {Name: "testing", URL: "gitolite.mycorp.com/testing.git"}, + }, nil) + tests := []struct { svc *types.ExternalService wantNames []string + gc gitserver.Client }{ { svc: typestest.MakeExternalService(t, extsvc.VariantGitHub, &schema.GitHubConnection{ @@ -267,6 +274,7 @@ func TestSources_ListRepos_Excluded(t *testing.T) { "gitolite.mycorp.com/foo", "gitolite.mycorp.com/testing", }, + gc: gitoliteGs, }, } @@ -277,7 +285,7 @@ func TestSources_ListRepos_Excluded(t *testing.T) { cf, save := NewClientFactory(t, name) defer save(t) - repos := listRepos(t, cf, tc.svc) + repos := listRepos(t, cf, tc.gc, tc.svc) var haveNames []string for _, r := range repos { @@ -297,10 +305,6 @@ func TestSources_ListRepos_RepositoryPathPattern(t *testing.T) { conf.Mock(&conf.Unified{ ServiceConnectionConfig: conftypes.ServiceConnections{ GitServers: []string{"127.0.0.1:3178"}, - }, SiteConfiguration: schema.SiteConfiguration{ - ExperimentalFeatures: &schema.ExperimentalFeatures{ - EnableGRPC: boolPointer(false), - }, }, }) defer conf.Mock(nil) @@ -308,10 +312,21 @@ func TestSources_ListRepos_RepositoryPathPattern(t *testing.T) { ratelimit.SetupForTest(t) rcache.SetupForTest(t) + gitoliteGs := gitserver.NewMockClient() + gitoliteGs.ScopedFunc.SetDefaultReturn(gitoliteGs) + gitoliteGs.ListGitoliteReposFunc.SetDefaultReturn([]*gitolite.Repo{ + {Name: "bar", URL: "gitolite.mycorp.com/bar.git"}, + {Name: "baz", URL: "gitolite.mycorp.com/baz.git"}, + {Name: "foo", URL: "gitolite.mycorp.com/foo.git"}, + {Name: "gitolite-admin", URL: "gitolite.mycorp.com/gitolite-admin.git"}, + {Name: "testing", URL: "gitolite.mycorp.com/testing.git"}, + }, nil) + tests := []struct { svc *types.ExternalService wantNames []string wantURIs []string + gc gitserver.Client }{ { svc: typestest.MakeExternalService(t, extsvc.VariantGitHub, &schema.GitHubConnection{ @@ -394,6 +409,7 @@ func TestSources_ListRepos_RepositoryPathPattern(t *testing.T) { "gitolite.mycorp.com/gitolite-admin", "gitolite.mycorp.com/testing", }, + gc: gitoliteGs, }, } @@ -404,7 +420,7 @@ func TestSources_ListRepos_RepositoryPathPattern(t *testing.T) { cf, save := NewClientFactory(t, name) defer save(t) - repos := listRepos(t, cf, tc.svc) + repos := listRepos(t, cf, tc.gc, tc.svc) var haveURIs, haveNames []string for _, r := range repos { @@ -431,7 +447,7 @@ func TestSources_Phabricator(t *testing.T) { Token: os.Getenv("PHABRICATOR_TOKEN"), }) - repos := listRepos(t, cf, svc) + repos := listRepos(t, cf, nil, svc) if len(repos) == 0 { t.Fatalf("no repos yielded") @@ -491,7 +507,7 @@ func TestSources_ListRepos_GitLab_NameTransformations(t *testing.T) { }, }) - repos := listRepos(t, cf, svc) + repos := listRepos(t, cf, nil, svc) haveNames := types.Repos(repos).Names() sort.Strings(haveNames) @@ -520,7 +536,7 @@ func TestSources_ListRepos_BitbucketServer_Archived(t *testing.T) { Repos: []string{"sour/vegeta", "PUBLIC/archived-repo"}, }) - repos := listRepos(t, cf, svc) + repos := listRepos(t, cf, nil, svc) wantArchived := map[string]bool{ "vegeta": false, @@ -537,13 +553,13 @@ func TestSources_ListRepos_BitbucketServer_Archived(t *testing.T) { } } -func listRepos(t *testing.T, cf *httpcli.Factory, svc *types.ExternalService) []*types.Repo { +func listRepos(t *testing.T, cf *httpcli.Factory, gc gitserver.Client, svc *types.ExternalService) []*types.Repo { t.Helper() ctx := context.Background() logger := logtest.NoOp(t) - sourcer := NewSourcer(logger, dbmocks.NewMockDB(), cf) + sourcer := NewSourcer(logger, dbmocks.NewMockDB(), cf, gc) src, err := sourcer(ctx, svc) if err != nil { diff --git a/internal/repos/sources_test_utils.go b/internal/repos/sources_test_utils.go index f5257de2f28..ae7e81d228e 100644 --- a/internal/repos/sources_test_utils.go +++ b/internal/repos/sources_test_utils.go @@ -1,7 +1,6 @@ package repos import ( - "net/http" "path/filepath" "strings" "testing" @@ -40,22 +39,10 @@ func Update(name string) bool { func TestClientFactorySetup(t testing.TB, name string, mws ...httpcli.Middleware) (httpcli.Middleware, *recorder.Recorder) { cassete := filepath.Join("testdata", "sources", strings.ReplaceAll(name, " ", "-")) rec := NewRecorder(t, cassete, Update(name)) - mws = append(mws, GitserverRedirectMiddleware) mw := httpcli.NewMiddleware(mws...) return mw, rec } -func GitserverRedirectMiddleware(cli httpcli.Doer) httpcli.Doer { - return httpcli.DoerFunc(func(req *http.Request) (*http.Response, error) { - if req.URL.Hostname() == "gitserver" { - // Start local git server first - req.URL.Host = "127.0.0.1:3178" - req.URL.Scheme = "http" - } - return cli.Do(req) - }) -} - func NewRecorder(t testing.TB, file string, record bool) *recorder.Recorder { rec, err := httptestutil.NewRecorder(file, record, func(i *cassette.Interaction) error { // The ratelimit.Monitor type resets its internal timestamp if it's diff --git a/internal/repos/testdata/sources/GITOLITE/excluded-repos-are-never-yielded.yaml b/internal/repos/testdata/sources/GITOLITE/excluded-repos-are-never-yielded.yaml deleted file mode 100644 index 80f05257a9e..00000000000 --- a/internal/repos/testdata/sources/GITOLITE/excluded-repos-are-never-yielded.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -version: 1 -interactions: -- request: - body: "" - form: {} - headers: {} - url: http://127.0.0.1:3178/list-gitolite?gitolite=ssh%3A%2F%2Fgit%40127.0.0.1%3A2222 - method: GET - response: - body: | - [{"Name":"bar","URL":"ssh://git@127.0.0.1:2222:bar"},{"Name":"baz","URL":"ssh://git@127.0.0.1:2222:baz"},{"Name":"foo","URL":"ssh://git@127.0.0.1:2222:foo"},{"Name":"gitolite-admin","URL":"ssh://git@127.0.0.1:2222:gitolite-admin"},{"Name":"testing","URL":"ssh://git@127.0.0.1:2222:testing"}] - headers: - Content-Length: - - "292" - Content-Type: - - text/plain; charset=utf-8 - Date: - - Tue, 07 May 2019 16:03:13 GMT - status: 200 OK - code: 200 - duration: "" diff --git a/internal/repos/testdata/sources/GITOLITE/repositoryPathPattern-determines-the-repo-name.yaml b/internal/repos/testdata/sources/GITOLITE/repositoryPathPattern-determines-the-repo-name.yaml deleted file mode 100644 index 32d33f00310..00000000000 --- a/internal/repos/testdata/sources/GITOLITE/repositoryPathPattern-determines-the-repo-name.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -version: 1 -interactions: -- request: - body: "" - form: {} - headers: {} - url: http://127.0.0.1:3178/list-gitolite?gitolite=ssh%3A%2F%2Fgit%40127.0.0.1%3A2222 - method: GET - response: - body: | - [{"Name":"bar","URL":"ssh://git@127.0.0.1:2222:bar"},{"Name":"baz","URL":"ssh://git@127.0.0.1:2222:baz"},{"Name":"foo","URL":"ssh://git@127.0.0.1:2222:foo"},{"Name":"gitolite-admin","URL":"ssh://git@127.0.0.1:2222:gitolite-admin"},{"Name":"testing","URL":"ssh://git@127.0.0.1:2222:testing"}] - headers: - Content-Length: - - "292" - Content-Type: - - text/plain; charset=utf-8 - Date: - - Tue, 07 May 2019 16:45:47 GMT - status: 200 OK - code: 200 - duration: "" diff --git a/internal/repos/testdata/sources/GITOLITE/yielded-repos-are-always-enabled.yaml b/internal/repos/testdata/sources/GITOLITE/yielded-repos-are-always-enabled.yaml deleted file mode 100644 index e563dde0bdb..00000000000 --- a/internal/repos/testdata/sources/GITOLITE/yielded-repos-are-always-enabled.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -version: 1 -interactions: -- request: - body: "" - form: {} - headers: {} - url: http://127.0.0.1:3178/list-gitolite?gitolite=ssh%3A%2F%2Fgit%40127.0.0.1%3A2222 - method: GET - response: - body: | - [{"Name":"bar","URL":"ssh://git@127.0.0.1:2222:bar"},{"Name":"baz","URL":"ssh://git@127.0.0.1:2222:baz"},{"Name":"foo","URL":"ssh://git@127.0.0.1:2222:foo"},{"Name":"gitolite-admin","URL":"ssh://git@127.0.0.1:2222:gitolite-admin"},{"Name":"testing","URL":"ssh://git@127.0.0.1:2222:testing"}] - headers: - Content-Length: - - "292" - Content-Type: - - text/plain; charset=utf-8 - Date: - - Fri, 03 May 2019 14:24:37 GMT - status: 200 OK - code: 200 - duration: "" diff --git a/internal/repoupdater/BUILD.bazel b/internal/repoupdater/BUILD.bazel index 315367bfadd..99ee84e602a 100644 --- a/internal/repoupdater/BUILD.bazel +++ b/internal/repoupdater/BUILD.bazel @@ -1,4 +1,3 @@ -load("//dev:go_defs.bzl", "go_test") load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( @@ -12,9 +11,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/api", - "//internal/conf", "//internal/grpc/defaults", - "//internal/httpcli", "//internal/repoupdater/protocol", "//internal/repoupdater/v1:repoupdater", "//internal/trace", @@ -26,10 +23,3 @@ go_library( "@org_golang_google_grpc//status", ], ) - -go_test( - name = "repoupdater_test", - timeout = "short", - srcs = ["client_test.go"], - embed = [":repoupdater"], -) diff --git a/internal/repoupdater/client.go b/internal/repoupdater/client.go index a60229f7a18..42cbe1c6339 100644 --- a/internal/repoupdater/client.go +++ b/internal/repoupdater/client.go @@ -1,12 +1,8 @@ package repoupdater import ( - "bytes" "context" - "encoding/json" "fmt" - "io" - "net/http" "net/url" "os" "sync" @@ -17,9 +13,7 @@ import ( "google.golang.org/grpc/status" "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/repoupdater/protocol" proto "github.com/sourcegraph/sourcegraph/internal/repoupdater/v1" "github.com/sourcegraph/sourcegraph/internal/trace" @@ -27,8 +21,6 @@ import ( ) var ( - defaultDoer, _ = httpcli.NewInternalClientFactory("repoupdater").Doer() - // DefaultClient is the default Client. Unless overwritten, it is // connected to the server specified by the REPO_UPDATER_URL // environment variable. @@ -45,12 +37,6 @@ func repoUpdaterURLDefault() string { // Client is a repoupdater client. type Client struct { - // URL to repoupdater server. - URL string - - // HTTP client to use - HTTPClient httpcli.Doer - // grpcClient is a function that lazily creates a grpc client. // Any implementation should not recreate the client more than once. grpcClient func() (proto.RepoUpdaterServiceClient, error) @@ -59,8 +45,6 @@ type Client struct { // NewClient will initiate a new repoupdater Client with the given serverURL. func NewClient(serverURL string) *Client { return &Client{ - URL: serverURL, - HTTPClient: defaultDoer, grpcClient: sync.OnceValues(func() (proto.RepoUpdaterServiceClient, error) { u, err := url.Parse(serverURL) if err != nil { @@ -84,31 +68,16 @@ func (c *Client) RepoUpdateSchedulerInfo( ctx context.Context, args protocol.RepoUpdateSchedulerInfoArgs, ) (result *protocol.RepoUpdateSchedulerInfoResult, err error) { - if conf.IsGRPCEnabled(ctx) { - client, err := c.grpcClient() - if err != nil { - return nil, err - } - req := &proto.RepoUpdateSchedulerInfoRequest{Id: int32(args.ID)} - resp, err := client.RepoUpdateSchedulerInfo(ctx, req) - if err != nil { - return nil, err - } - return protocol.RepoUpdateSchedulerInfoResultFromProto(resp), nil - } - - resp, err := c.httpPost(ctx, "repo-update-scheduler-info", args) + client, err := c.grpcClient() if err != nil { return nil, err } - - if resp.StatusCode != http.StatusOK { - stack := fmt.Sprintf("RepoScheduleInfo: %+v", args) - return nil, errors.Wrap(errors.Errorf("http status %d", resp.StatusCode), stack) + req := &proto.RepoUpdateSchedulerInfoRequest{Id: int32(args.ID)} + resp, err := client.RepoUpdateSchedulerInfo(ctx, req) + if err != nil { + return nil, err } - defer resp.Body.Close() - err = json.NewDecoder(resp.Body).Decode(&result) - return result, err + return protocol.RepoUpdateSchedulerInfoResultFromProto(resp), nil } // MockRepoLookup mocks (*Client).RepoLookup for tests. @@ -132,75 +101,29 @@ func (c *Client) RepoLookup( tr.EndWithErr(&err) }() - if conf.IsGRPCEnabled(ctx) { - client, err := c.grpcClient() - if err != nil { - return nil, err - } - resp, err := client.RepoLookup(ctx, args.ToProto()) - if err != nil { - return nil, errors.Wrapf(err, "RepoLookup for %+v failed", args) - } - res := protocol.RepoLookupResultFromProto(resp) - switch { - case resp.GetErrorNotFound(): - return res, &ErrNotFound{Repo: args.Repo, IsNotFound: true} - case resp.GetErrorUnauthorized(): - return res, &ErrUnauthorized{Repo: args.Repo, NoAuthz: true} - case resp.GetErrorTemporarilyUnavailable(): - return res, &ErrTemporary{Repo: args.Repo, IsTemporary: true} - case resp.GetErrorRepoDenied() != "": - return res, &ErrRepoDenied{ - Repo: args.Repo, - Reason: resp.GetErrorRepoDenied(), - } - } - return res, nil - } - - resp, err := c.httpPost(ctx, "repo-lookup", args) + client, err := c.grpcClient() if err != nil { return nil, err } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - // best-effort inclusion of body in error message - body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) - return nil, errors.Errorf( - "RepoLookup for %+v failed with http status %d: %s", - args, - resp.StatusCode, - string(body), - ) + resp, err := client.RepoLookup(ctx, args.ToProto()) + if err != nil { + return nil, errors.Wrapf(err, "RepoLookup for %+v failed", args) } - - err = json.NewDecoder(resp.Body).Decode(&result) - if err == nil && result != nil { - switch { - case result.ErrorNotFound: - err = &ErrNotFound{ - Repo: args.Repo, - IsNotFound: true, - } - case result.ErrorUnauthorized: - err = &ErrUnauthorized{ - Repo: args.Repo, - NoAuthz: true, - } - case result.ErrorTemporarilyUnavailable: - err = &ErrTemporary{ - Repo: args.Repo, - IsTemporary: true, - } - case result.ErrorRepoDenied != "": - err = &ErrRepoDenied{ - Repo: args.Repo, - Reason: result.ErrorRepoDenied, - } + res := protocol.RepoLookupResultFromProto(resp) + switch { + case resp.GetErrorNotFound(): + return res, &ErrNotFound{Repo: args.Repo, IsNotFound: true} + case resp.GetErrorUnauthorized(): + return res, &ErrUnauthorized{Repo: args.Repo, NoAuthz: true} + case resp.GetErrorTemporarilyUnavailable(): + return res, &ErrTemporary{Repo: args.Repo, IsTemporary: true} + case resp.GetErrorRepoDenied() != "": + return res, &ErrRepoDenied{ + Repo: args.Repo, + Reason: resp.GetErrorRepoDenied(), } } - return result, err + return res, nil } // MockEnqueueRepoUpdate mocks (*Client).EnqueueRepoUpdate for tests. @@ -213,50 +136,22 @@ func (c *Client) EnqueueRepoUpdate(ctx context.Context, repo api.RepoName) (*pro return MockEnqueueRepoUpdate(ctx, repo) } - if conf.IsGRPCEnabled(ctx) { - client, err := c.grpcClient() - if err != nil { - return nil, err - } - - req := proto.EnqueueRepoUpdateRequest{Repo: string(repo)} - resp, err := client.EnqueueRepoUpdate(ctx, &req) - if err != nil { - if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { - return nil, &repoNotFoundError{repo: string(repo), responseBody: s.Message()} - } - - return nil, err - } - - return protocol.RepoUpdateResponseFromProto(resp), nil - } - - req := &protocol.RepoUpdateRequest{ - Repo: repo, - } - - resp, err := c.httpPost(ctx, "enqueue-repo-update", req) + client, err := c.grpcClient() if err != nil { return nil, err } - defer resp.Body.Close() - bs, err := io.ReadAll(resp.Body) + req := proto.EnqueueRepoUpdateRequest{Repo: string(repo)} + resp, err := client.EnqueueRepoUpdate(ctx, &req) if err != nil { - return nil, errors.Wrap(err, "failed to read response body") - } + if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound { + return nil, &repoNotFoundError{repo: string(repo), responseBody: s.Message()} + } - var res protocol.RepoUpdateResponse - if resp.StatusCode == http.StatusNotFound { - return nil, &repoNotFoundError{string(repo), string(bs)} - } else if resp.StatusCode < 200 || resp.StatusCode >= 400 { - return nil, errors.New(string(bs)) - } else if err = json.Unmarshal(bs, &res); err != nil { return nil, err } - return &res, nil + return protocol.RepoUpdateResponseFromProto(resp), nil } type repoNotFoundError struct { @@ -277,66 +172,12 @@ func (c *Client) EnqueueChangesetSync(ctx context.Context, ids []int64) error { return MockEnqueueChangesetSync(ctx, ids) } - if conf.IsGRPCEnabled(ctx) { - client, err := c.grpcClient() - if err != nil { - return err - } - - // empty response can be ignored - _, err = client.EnqueueChangesetSync(ctx, &proto.EnqueueChangesetSyncRequest{Ids: ids}) - return err - } - - req := protocol.ChangesetSyncRequest{IDs: ids} - resp, err := c.httpPost(ctx, "enqueue-changeset-sync", req) + client, err := c.grpcClient() if err != nil { return err } - defer resp.Body.Close() - bs, err := io.ReadAll(resp.Body) - if err != nil { - return errors.Wrap(err, "failed to read response body") - } - - var res protocol.ChangesetSyncResponse - if resp.StatusCode < 200 || resp.StatusCode >= 400 { - return errors.New(string(bs)) - } else if err = json.Unmarshal(bs, &res); err != nil { - return err - } - - if res.Error == "" { - return nil - } - return errors.New(res.Error) -} - -func (c *Client) httpPost(ctx context.Context, method string, payload any) (resp *http.Response, err error) { - reqBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("POST", c.URL+"/"+method, bytes.NewReader(reqBody)) - if err != nil { - return nil, err - } - - return c.do(ctx, req) -} - -func (c *Client) do(ctx context.Context, req *http.Request) (_ *http.Response, err error) { - tr, ctx := trace.New(ctx, "repoupdater.do") - defer tr.EndWithErr(&err) - - req.Header.Set("Content-Type", "application/json") - - req = req.WithContext(ctx) - - if c.HTTPClient != nil { - return c.HTTPClient.Do(req) - } - return http.DefaultClient.Do(req) + // empty response can be ignored + _, err = client.EnqueueChangesetSync(ctx, &proto.EnqueueChangesetSyncRequest{Ids: ids}) + return err } diff --git a/internal/repoupdater/client_test.go b/internal/repoupdater/client_test.go deleted file mode 100644 index fb424638721..00000000000 --- a/internal/repoupdater/client_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package repoupdater - -import "testing" - -func TestNewClient(t *testing.T) { - t.Run("successful creation of client with custom URL", func(t *testing.T) { - expected := "foo" - c := NewClient(expected) - - if c.URL != expected { - t.Errorf("Expected URL %q, but got %q", expected, c.URL) - } - }) -} diff --git a/internal/search/alert/BUILD.bazel b/internal/search/alert/BUILD.bazel index 05b13a09f23..1b211da775b 100644 --- a/internal/search/alert/BUILD.bazel +++ b/internal/search/alert/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//internal/endpoint", "//internal/gitserver", "//internal/gitserver/gitdomain", + "//internal/grpc/defaults", "//internal/search", "//internal/search/query", "//internal/search/repos", diff --git a/internal/search/alert/observer.go b/internal/search/alert/observer.go index bcdfc77d286..f90d97aefea 100644 --- a/internal/search/alert/observer.go +++ b/internal/search/alert/observer.go @@ -17,6 +17,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/endpoint" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" + "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/query" searchrepos "github.com/sourcegraph/sourcegraph/internal/search/repos" @@ -25,10 +26,11 @@ import ( ) type Observer struct { - Logger log.Logger - Db database.DB - Zoekt zoekt.Streamer - Searcher *endpoint.Map + Logger log.Logger + Db database.DB + Zoekt zoekt.Streamer + Searcher *endpoint.Map + SearcherGRPCConnectionCache *defaults.ConnectionCache // Inputs are used to generate alert messages based on the query. *search.Inputs @@ -47,7 +49,7 @@ type Observer struct { // raising NoResolvedRepos alerts with suggestions when we know the original // query does not contain any repos to search. func (o *Observer) reposExist(ctx context.Context, options search.RepoOptions) bool { - repositoryResolver := searchrepos.NewResolver(o.Logger, o.Db, gitserver.NewClient("search.alertobserver"), o.Searcher, o.Zoekt) + repositoryResolver := searchrepos.NewResolver(o.Logger, o.Db, gitserver.NewClient("search.alertobserver"), o.Searcher, o.SearcherGRPCConnectionCache, o.Zoekt) it := repositoryResolver.Iterator(ctx, options) for it.Next() { resolved := it.Current() diff --git a/internal/search/backend/BUILD.bazel b/internal/search/backend/BUILD.bazel index efdb3a86c29..6e7952250e5 100644 --- a/internal/search/backend/BUILD.bazel +++ b/internal/search/backend/BUILD.bazel @@ -25,7 +25,6 @@ go_library( "//internal/ctags_config", "//internal/grpc/defaults", "//internal/honey", - "//internal/httpcli", "//internal/trace", "//internal/types", "//lib/errors", @@ -41,7 +40,6 @@ go_library( "@com_github_sourcegraph_zoekt//cmd/zoekt-sourcegraph-indexserver/protos/sourcegraph/zoekt/configuration/v1:configuration", "@com_github_sourcegraph_zoekt//grpc/protos/zoekt/webserver/v1:webserver", "@com_github_sourcegraph_zoekt//query", - "@com_github_sourcegraph_zoekt//rpc", "@com_github_sourcegraph_zoekt//stream", "@io_opentelemetry_go_otel//attribute", "@org_golang_google_grpc//:go_default_library", diff --git a/internal/search/backend/grpc.go b/internal/search/backend/grpc.go index a6bee9e5226..c66e4289135 100644 --- a/internal/search/backend/grpc.go +++ b/internal/search/backend/grpc.go @@ -9,49 +9,8 @@ import ( "github.com/sourcegraph/zoekt/query" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - - "github.com/sourcegraph/sourcegraph/internal/conf" ) -// switchableZoektGRPCClient is a zoekt.Streamer that can switch between -// gRPC and HTTP backends. -type switchableZoektGRPCClient struct { - httpClient zoekt.Streamer - grpcClient zoekt.Streamer -} - -func (c *switchableZoektGRPCClient) StreamSearch(ctx context.Context, q query.Q, opts *zoekt.SearchOptions, sender zoekt.Sender) error { - if conf.IsGRPCEnabled(ctx) { - return c.grpcClient.StreamSearch(ctx, q, opts, sender) - } else { - return c.httpClient.StreamSearch(ctx, q, opts, sender) - } -} - -func (c *switchableZoektGRPCClient) Search(ctx context.Context, q query.Q, opts *zoekt.SearchOptions) (*zoekt.SearchResult, error) { - if conf.IsGRPCEnabled(ctx) { - return c.grpcClient.Search(ctx, q, opts) - } else { - return c.httpClient.Search(ctx, q, opts) - } -} - -func (c *switchableZoektGRPCClient) List(ctx context.Context, q query.Q, opts *zoekt.ListOptions) (*zoekt.RepoList, error) { - if conf.IsGRPCEnabled(ctx) { - return c.grpcClient.List(ctx, q, opts) - } else { - return c.httpClient.List(ctx, q, opts) - } -} - -func (c *switchableZoektGRPCClient) Close() { - c.httpClient.Close() -} - -func (c *switchableZoektGRPCClient) String() string { - return c.httpClient.String() -} - // zoektGRPCClient is a zoekt.Streamer that uses gRPC for its RPC layer type zoektGRPCClient struct { endpoint string diff --git a/internal/search/backend/horizontal.go b/internal/search/backend/horizontal.go index da6938de044..32f2a2f9aa5 100644 --- a/internal/search/backend/horizontal.go +++ b/internal/search/backend/horizontal.go @@ -60,13 +60,6 @@ func (s *HorizontalSearcher) StreamSearch(ctx context.Context, q query.Q, opts * var mu sync.Mutex dedupper := dedupper{} - // GobCache exists, so we only pay the cost of marshalling a query once - // when we aggregate it out over all the replicas. Zoekt's RPC layers - // unwrap this before passing it on to the Zoekt evaluation layers. - if !conf.IsGRPCEnabled(ctx) { - q = &query.GobCache{Q: q} - } - ch := make(chan error, len(clients)) for endpoint, c := range clients { go func(endpoint string, c zoekt.Streamer) { diff --git a/internal/search/backend/zoekt.go b/internal/search/backend/zoekt.go index 78c88257350..68032a86a44 100644 --- a/internal/search/backend/zoekt.go +++ b/internal/search/backend/zoekt.go @@ -6,30 +6,11 @@ import ( "github.com/sourcegraph/log" "github.com/sourcegraph/zoekt" proto "github.com/sourcegraph/zoekt/grpc/protos/zoekt/webserver/v1" - "github.com/sourcegraph/zoekt/rpc" - zoektstream "github.com/sourcegraph/zoekt/stream" "google.golang.org/grpc" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpcli" ) -// We don't use the normal factory for internal requests because we disable -// retries. Currently our retry framework copies the full body on every -// request, this is prohibitive when zoekt generates a large query. -// -// Once our retry framework supports the use of Request.GetBody we can switch -// back to the normal internal request factory. -var zoektHTTPClient, _ = httpcli.NewFactory( - httpcli.NewMiddleware( - httpcli.ContextErrorMiddleware, - ), - httpcli.NewMaxIdleConnsPerHostOpt(500), - // This will also generate a metric named "src_zoekt_webserver_requests_total". - httpcli.MeteredTransportOpt("zoekt_webserver"), - httpcli.TracedTransportOpt, -).Client() - // ZoektStreamFunc is a convenience function to create a stream receiver from a // function. type ZoektStreamFunc func(*zoekt.SearchResult) @@ -92,29 +73,14 @@ func (c *cachedStreamerCloser) Close() { c.Streamer.Close() } -// ZoektDial connects to a Searcher HTTP RPC server at address (host:port). -func ZoektDial(endpoint string) zoekt.Streamer { - return &switchableZoektGRPCClient{ - httpClient: ZoektDialHTTP(endpoint), - grpcClient: ZoektDialGRPC(endpoint), - } -} - -// ZoektDialHTTP connects to a Searcher HTTP RPC server at address (host:port). -func ZoektDialHTTP(endpoint string) zoekt.Streamer { - client := rpc.Client(endpoint) - streamClient := zoektstream.NewClient("http://"+endpoint, zoektHTTPClient).WithSearcher(client) - return NewMeteredSearcher(endpoint, streamClient) -} - // maxRecvMsgSize is the max message size we can receive from Zoekt without erroring. // By default, this caps at 4MB, but Zoekt can send payloads significantly larger // than that depending on the type of search being executed. // 128MiB is a best guess at reasonable size that will rarely fail. const maxRecvMsgSize = 128 * 1024 * 1024 // 128MiB -// ZoektDialGRPC connects to a Searcher gRPC server at address (host:port). -func ZoektDialGRPC(endpoint string) zoekt.Streamer { +// ZoektDial connects to a Searcher gRPC server at address (host:port). +func ZoektDial(endpoint string) zoekt.Streamer { conn, err := defaults.Dial( endpoint, log.Scoped("zoekt"), diff --git a/internal/search/commit/commit.go b/internal/search/commit/commit.go index ed87a557f29..bea2f3c04e2 100644 --- a/internal/search/commit/commit.go +++ b/internal/search/commit/commit.go @@ -96,7 +96,7 @@ func (j *SearchJob) Run(ctx context.Context, clients job.RuntimeClients, stream return doSearch(args) } - repos := searchrepos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.Zoekt) + repos := searchrepos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.SearcherGRPCConnectionCache, clients.Zoekt) it := repos.Iterator(ctx, j.RepoOpts) p := pool.New().WithContext(ctx).WithMaxGoroutines(4).WithFirstError() diff --git a/internal/search/job/jobutil/BUILD.bazel b/internal/search/job/jobutil/BUILD.bazel index 6072113d68c..c7d1f256763 100644 --- a/internal/search/job/jobutil/BUILD.bazel +++ b/internal/search/job/jobutil/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "//internal/featureflag", "//internal/gitserver", "//internal/gitserver/gitdomain", + "//internal/grpc/defaults", "//internal/own/search", "//internal/search", "//internal/search/alert", diff --git a/internal/search/job/jobutil/alert.go b/internal/search/job/jobutil/alert.go index 1fb207c415c..bfb49d2253d 100644 --- a/internal/search/job/jobutil/alert.go +++ b/internal/search/job/jobutil/alert.go @@ -41,12 +41,13 @@ func (j *alertJob) Run(ctx context.Context, clients job.RuntimeClients, stream s jobAlert, err := j.child.Run(ctx, clients, statsObserver) ao := searchalert.Observer{ - Logger: clients.Logger, - Db: clients.DB, - Zoekt: clients.Zoekt, - Searcher: clients.SearcherURLs, - Inputs: j.inputs, - HasResults: countingStream.Count() > 0, + Logger: clients.Logger, + Db: clients.DB, + Zoekt: clients.Zoekt, + Searcher: clients.SearcherURLs, + SearcherGRPCConnectionCache: clients.SearcherGRPCConnectionCache, + Inputs: j.inputs, + HasResults: countingStream.Count() > 0, } if err != nil { ao.Error(ctx, err) diff --git a/internal/search/job/jobutil/exhaustive_job.go b/internal/search/job/jobutil/exhaustive_job.go index 4480948b701..dfd5d01f8d4 100644 --- a/internal/search/job/jobutil/exhaustive_job.go +++ b/internal/search/job/jobutil/exhaustive_job.go @@ -105,5 +105,5 @@ func (e Exhaustive) ResolveRepositoryRevSpec(ctx context.Context, clients job.Ru } func reposNewResolver(clients job.RuntimeClients) *repos.Resolver { - return repos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.Zoekt) + return repos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.SearcherGRPCConnectionCache, clients.Zoekt) } diff --git a/internal/search/job/jobutil/filter_file_contains.go b/internal/search/job/jobutil/filter_file_contains.go index d585e9e0f1d..0e9dfbf208f 100644 --- a/internal/search/job/jobutil/filter_file_contains.go +++ b/internal/search/job/jobutil/filter_file_contains.go @@ -12,6 +12,7 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" "github.com/sourcegraph/sourcegraph/internal/endpoint" + "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/job" "github.com/sourcegraph/sourcegraph/internal/search/query" @@ -93,14 +94,14 @@ func (j *fileContainsFilterJob) Run(ctx context.Context, clients job.RuntimeClie defer func() { finish(alert, err) }() filteredStream := streaming.StreamFunc(func(event streaming.SearchEvent) { - event = j.filterEvent(ctx, clients.SearcherURLs, event) + event = j.filterEvent(ctx, clients.SearcherURLs, clients.SearcherGRPCConnectionCache, event) stream.Send(event) }) return j.child.Run(ctx, clients, filteredStream) } -func (j *fileContainsFilterJob) filterEvent(ctx context.Context, searcherURLs *endpoint.Map, event streaming.SearchEvent) streaming.SearchEvent { +func (j *fileContainsFilterJob) filterEvent(ctx context.Context, searcherURLs *endpoint.Map, searcherGRPCConnectionCache *defaults.ConnectionCache, event streaming.SearchEvent) streaming.SearchEvent { // Don't filter out files with zero chunks because if the file contained // a result, we still want to return a match for the file even if it // has no matched ranges left. @@ -110,7 +111,7 @@ func (j *fileContainsFilterJob) filterEvent(ctx context.Context, searcherURLs *e case *result.FileMatch: filtered = append(filtered, j.filterFileMatch(v)) case *result.CommitMatch: - cm := j.filterCommitMatch(ctx, searcherURLs, v) + cm := j.filterCommitMatch(ctx, searcherURLs, searcherGRPCConnectionCache, v) if cm != nil { filtered = append(filtered, cm) } @@ -159,7 +160,7 @@ func matchesAny(val string, matchers []*regexp.Regexp) bool { return false } -func (j *fileContainsFilterJob) filterCommitMatch(ctx context.Context, searcherURLs *endpoint.Map, cm *result.CommitMatch) result.Match { +func (j *fileContainsFilterJob) filterCommitMatch(ctx context.Context, searcherURLs *endpoint.Map, searcherGRPCConnectionCache *defaults.ConnectionCache, cm *result.CommitMatch) result.Match { // Skip any commit matches -- we only handle diff matches if cm.DiffPreview == nil { return nil @@ -187,15 +188,14 @@ func (j *fileContainsFilterJob) filterCommitMatch(ctx context.Context, searcherU PatternMatchesContent: true, } - onMatch := func(fms []*protocol.FileMatch) { - for _, fm := range fms { - matchedFileCounts[fm.Path] += 1 - } + onMatch := func(fm *protocol.FileMatch) { + matchedFileCounts[fm.Path] += 1 } _, err := searcher.Search( ctx, searcherURLs, + searcherGRPCConnectionCache, cm.Repo.Name, cm.Repo.ID, "", diff --git a/internal/search/job/jobutil/filter_file_contains_test.go b/internal/search/job/jobutil/filter_file_contains_test.go index 877d432aeb8..1bebf60ea08 100644 --- a/internal/search/job/jobutil/filter_file_contains_test.go +++ b/internal/search/job/jobutil/filter_file_contains_test.go @@ -256,9 +256,9 @@ func TestFileContainsFilterJob(t *testing.T) { s.Send(tc.inputEvent) return nil, nil }) - searcher.MockSearch = func(_ context.Context, _ api.RepoName, _ api.RepoID, _ api.CommitID, p *search.TextPatternInfo, _ time.Duration, onMatches func([]*protocol.FileMatch)) (limitHit bool, err error) { + searcher.MockSearch = func(_ context.Context, _ api.RepoName, _ api.RepoID, _ api.CommitID, p *search.TextPatternInfo, _ time.Duration, onMatch func(*protocol.FileMatch)) (limitHit bool, err error) { if len(p.IncludePatterns) > 0 { - onMatches([]*protocol.FileMatch{{Path: "file4"}}) + onMatch(&protocol.FileMatch{Path: "file4"}) } return false, nil } diff --git a/internal/search/job/jobutil/repo_pager_job.go b/internal/search/job/jobutil/repo_pager_job.go index 90b9a7bcd81..1aff3905a4b 100644 --- a/internal/search/job/jobutil/repo_pager_job.go +++ b/internal/search/job/jobutil/repo_pager_job.go @@ -83,7 +83,7 @@ func (p *repoPagerJob) Run(ctx context.Context, clients job.RuntimeClients, stre var maxAlerter search.MaxAlerter - repoResolver := repos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.Zoekt) + repoResolver := repos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.SearcherGRPCConnectionCache, clients.Zoekt) it := repoResolver.Iterator(ctx, p.repoOpts) for it.Next() { diff --git a/internal/search/job/jobutil/repos.go b/internal/search/job/jobutil/repos.go index 879d231dc6c..1aa3cf3f0a9 100644 --- a/internal/search/job/jobutil/repos.go +++ b/internal/search/job/jobutil/repos.go @@ -28,7 +28,7 @@ func (s *RepoSearchJob) Run(ctx context.Context, clients job.RuntimeClients, str tr, ctx, stream, finish := job.StartSpan(ctx, stream, s) defer func() { finish(alert, err) }() - repos := searchrepos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.Zoekt) + repos := searchrepos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.SearcherGRPCConnectionCache, clients.Zoekt) it := repos.Iterator(ctx, s.RepoOpts) for it.Next() { diff --git a/internal/search/repos/BUILD.bazel b/internal/search/repos/BUILD.bazel index 68750d3102d..8ad51062358 100644 --- a/internal/search/repos/BUILD.bazel +++ b/internal/search/repos/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "//internal/endpoint", "//internal/gitserver", "//internal/gitserver/gitdomain", + "//internal/grpc/defaults", "//internal/search", "//internal/search/job", "//internal/search/limits", @@ -64,6 +65,7 @@ go_test( "//internal/endpoint", "//internal/gitserver", "//internal/gitserver/gitdomain", + "//internal/grpc/defaults", "//internal/search", "//internal/search/job", "//internal/search/query", diff --git a/internal/search/repos/repos.go b/internal/search/repos/repos.go index c3911a3d06a..5824a2246f2 100644 --- a/internal/search/repos/repos.go +++ b/internal/search/repos/repos.go @@ -27,6 +27,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/endpoint" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" + "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/limits" "github.com/sourcegraph/sourcegraph/internal/search/query" @@ -68,22 +69,24 @@ func (r *Resolved) String() string { return fmt.Sprintf("Resolved{RepoRevs=%d BackendsMissing=%d}", len(r.RepoRevs), r.BackendsMissing) } -func NewResolver(logger log.Logger, db database.DB, gitserverClient gitserver.Client, searcher *endpoint.Map, zoekt zoekt.Streamer) *Resolver { +func NewResolver(logger log.Logger, db database.DB, gitserverClient gitserver.Client, searcher *endpoint.Map, searcherGRPCConnectionCache *defaults.ConnectionCache, zoekt zoekt.Streamer) *Resolver { return &Resolver{ - logger: logger, - db: db, - gitserver: gitserverClient, - zoekt: zoekt, - searcher: searcher, + logger: logger, + db: db, + gitserver: gitserverClient, + zoekt: zoekt, + searcher: searcher, + searcherGRPCConnectionCache: searcherGRPCConnectionCache, } } type Resolver struct { - logger log.Logger - db database.DB - gitserver gitserver.Client - zoekt zoekt.Streamer - searcher *endpoint.Map + logger log.Logger + db database.DB + gitserver gitserver.Client + zoekt zoekt.Streamer + searcher *endpoint.Map + searcherGRPCConnectionCache *defaults.ConnectionCache } // Iterator returns an iterator of Resolved for opts. @@ -176,7 +179,7 @@ func (r *Resolver) ResolveRevSpecs(ctx context.Context, op search.RepoOptions, r Associated: repoRevSpecs, } - resolved, err := r.doFilterDBResolved(ctx, tr, op, result) + resolved, err := r.doFilterDBResolved(ctx, tr, r.searcherGRPCConnectionCache, op, result) return resolved, err } @@ -203,7 +206,7 @@ func (r *Resolver) resolve(ctx context.Context, op search.RepoOptions) (_ Resolv // We then speak to gitserver (and others) to convert revspecs into // revisions to search. - resolved, err := r.doFilterDBResolved(ctx, tr, op, result) + resolved, err := r.doFilterDBResolved(ctx, tr, r.searcherGRPCConnectionCache, op, result) return resolved, next, err } @@ -357,7 +360,7 @@ func (r *Resolver) doQueryDB(ctx context.Context, tr trace.Trace, op search.Repo // // NOTE: This API is not idiomatic and can return non-nil error with a useful // Resolved. -func (r *Resolver) doFilterDBResolved(ctx context.Context, tr trace.Trace, op search.RepoOptions, result dbResolved) (Resolved, error) { +func (r *Resolver) doFilterDBResolved(ctx context.Context, tr trace.Trace, searcherGRPCConnectionCache *defaults.ConnectionCache, op search.RepoOptions, result dbResolved) (Resolved, error) { // At each step we will discover RepoRevSpecs that do not actually exist. // We keep appending to this. missing := result.Missing @@ -369,7 +372,7 @@ func (r *Resolver) doFilterDBResolved(ctx context.Context, tr trace.Trace, op se missing = append(missing, filteredMissing...) tr.AddEvent("starting contains filtering") - filteredRepoRevs, missingHasFileContentRevs, backendsMissing, err := r.filterRepoHasFileContent(ctx, filteredRepoRevs, op) + filteredRepoRevs, missingHasFileContentRevs, backendsMissing, err := r.filterRepoHasFileContent(ctx, searcherGRPCConnectionCache, filteredRepoRevs, op) missing = append(missing, missingHasFileContentRevs...) if err != nil { return Resolved{}, errors.Wrap(err, "filter has file content") @@ -644,6 +647,7 @@ func (r *Resolver) filterHasCommitAfter( // 4) We collect the set of revisions that matched all contains predicates and return them. func (r *Resolver) filterRepoHasFileContent( ctx context.Context, + searcherGRPCConnectionCache *defaults.ConnectionCache, repoRevs []*search.RepositoryRevisions, op search.RepoOptions, ) ( @@ -788,7 +792,7 @@ func (r *Resolver) filterRepoHasFileContent( return false, nil } - return r.repoHasFileContentAtCommit(ctx, repo, commitID, arg) + return r.repoHasFileContentAtCommit(ctx, searcherGRPCConnectionCache, repo, commitID, arg) } for _, repoRevs := range unindexed { @@ -835,7 +839,7 @@ func (r *Resolver) filterRepoHasFileContent( return matchedRepoRevs, missing, backendsMissing, nil } -func (r *Resolver) repoHasFileContentAtCommit(ctx context.Context, repo types.MinimalRepo, commitID api.CommitID, args query.RepoHasFileContentArgs) (bool, error) { +func (r *Resolver) repoHasFileContentAtCommit(ctx context.Context, searcherGRPCConnectionCache *defaults.ConnectionCache, repo types.MinimalRepo, commitID api.CommitID, args query.RepoHasFileContentArgs) (bool, error) { patternInfo := search.TextPatternInfo{ Pattern: args.Content, IsNegated: args.Negated, @@ -851,15 +855,14 @@ func (r *Resolver) repoHasFileContentAtCommit(ctx context.Context, repo types.Mi } foundMatches := false - onMatches := func(fms []*protocol.FileMatch) { - if len(fms) > 0 { - foundMatches = true - } + onMatch := func(fm *protocol.FileMatch) { + foundMatches = true } _, err := searcher.Search( ctx, r.searcher, + searcherGRPCConnectionCache, repo.Name, repo.ID, "", // not using zoekt, don't need branch @@ -869,7 +872,7 @@ func (r *Resolver) repoHasFileContentAtCommit(ctx context.Context, repo types.Mi time.Hour, // depend on context for timeout search.Features{}, // not using any search features 0, // don't care about the actual content, so don't fetch extra context - onMatches, + onMatch, ) return foundMatches, err } diff --git a/internal/search/repos/repos_test.go b/internal/search/repos/repos_test.go index 5d11f95e3e6..451a964e9bb 100644 --- a/internal/search/repos/repos_test.go +++ b/internal/search/repos/repos_test.go @@ -26,6 +26,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/endpoint" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain" + "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/query" "github.com/sourcegraph/sourcegraph/internal/search/searcher" @@ -147,7 +148,7 @@ func TestRevisionValidation(t *testing.T) { db.ReposFunc.SetDefaultReturn(repos) op := search.RepoOptions{RepoFilters: toParsedRepoFilters(tt.repoFilters...)} - repositoryResolver := NewResolver(logtest.Scoped(t), db, nil, nil, nil) + repositoryResolver := NewResolver(logtest.Scoped(t), db, nil, nil, defaults.NewConnectionCache(logtest.Scoped(t)), nil) repositoryResolver.gitserver = mockGitserver resolved, _, err := repositoryResolver.resolve(context.Background(), op) if !errors.Is(err, tt.wantErr) { @@ -298,7 +299,7 @@ func TestResolverIterator(t *testing.T) { return "", nil }) - resolver := NewResolver(logtest.Scoped(t), db, gsClient, nil, nil) + resolver := NewResolver(logtest.Scoped(t), db, gsClient, nil, defaults.NewConnectionCache(logtest.Scoped(t)), nil) all, _, err := resolver.resolve(ctx, search.RepoOptions{}) if err != nil { t.Fatal(err) @@ -402,7 +403,7 @@ func TestResolverIterator(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - r := NewResolver(logtest.Scoped(t), db, gsClient, nil, nil) + r := NewResolver(logtest.Scoped(t), db, gsClient, nil, defaults.NewConnectionCache(logtest.Scoped(t)), nil) it := r.Iterator(ctx, tc.opts) var pages []Resolved @@ -521,7 +522,7 @@ func TestResolverIterateRepoRevs(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - r := NewResolver(logger, db, gsClient, nil, nil) + r := NewResolver(logger, db, gsClient, nil, defaults.NewConnectionCache(logtest.Scoped(t)), nil) got, err := iterator.Collect(r.IterateRepoRevs(ctx, tc.opts)) var gotErr string @@ -592,7 +593,7 @@ func TestResolveRepositoriesWithSearchContext(t *testing.T) { op := search.RepoOptions{ SearchContextSpec: "searchcontext", } - repositoryResolver := NewResolver(logtest.Scoped(t), db, gsClient, nil, nil) + repositoryResolver := NewResolver(logtest.Scoped(t), db, gsClient, nil, defaults.NewConnectionCache(logtest.Scoped(t)), nil) resolved, _, err := repositoryResolver.resolve(context.Background(), op) if err != nil { t.Fatal(err) @@ -652,13 +653,13 @@ func TestRepoHasFileContent(t *testing.T) { }, }, } - searcher.MockSearch = func(ctx context.Context, repo api.RepoName, repoID api.RepoID, commit api.CommitID, p *search.TextPatternInfo, fetchTimeout time.Duration, onMatches func([]*protocol.FileMatch)) (limitHit bool, err error) { + searcher.MockSearch = func(ctx context.Context, repo api.RepoName, repoID api.RepoID, commit api.CommitID, p *search.TextPatternInfo, fetchTimeout time.Duration, onMatch func(*protocol.FileMatch)) (limitHit bool, err error) { if r, ok := unindexedCorpus[string(repo)]; ok { for path, lines := range r { if len(p.IncludePatterns) == 0 || p.IncludePatterns[0] == path { for line := range lines { if p.Pattern == line || p.Pattern == "" { - onMatches([]*protocol.FileMatch{{}}) + onMatch(&protocol.FileMatch{}) } } } @@ -764,7 +765,7 @@ func TestRepoHasFileContent(t *testing.T) { ReposMap: tc.matchingRepos, }, nil) - res := NewResolver(logtest.Scoped(t), db, mockGitserver, endpoint.Static("test"), mockZoekt) + res := NewResolver(logtest.Scoped(t), db, mockGitserver, endpoint.Static("test"), defaults.NewConnectionCache(logtest.Scoped(t)), mockZoekt) resolved, _, err := res.resolve(context.Background(), search.RepoOptions{ RepoFilters: toParsedRepoFilters(".*"), HasFileContent: tc.filters, @@ -867,7 +868,7 @@ func TestRepoHasCommitAfter(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - res := NewResolver(logtest.Scoped(t), db, nil, endpoint.Static("test"), nil) + res := NewResolver(logtest.Scoped(t), db, nil, endpoint.Static("test"), defaults.NewConnectionCache(logtest.Scoped(t)), nil) res.gitserver = mockGitserver resolved, _, err := res.resolve(context.Background(), search.RepoOptions{ RepoFilters: toParsedRepoFilters(tc.nameFilter), diff --git a/internal/search/searcher/BUILD.bazel b/internal/search/searcher/BUILD.bazel index 9d44de8432f..9d5ea571e7a 100644 --- a/internal/search/searcher/BUILD.bazel +++ b/internal/search/searcher/BUILD.bazel @@ -5,10 +5,8 @@ go_library( name = "searcher", srcs = [ "client.go", - "client_grpc.go", "retry_grpc.go", "search.go", - "stream.go", "symbol_search_job.go", ], importpath = "github.com/sourcegraph/sourcegraph/internal/search/searcher", @@ -21,13 +19,11 @@ go_library( "//internal/errcode", "//internal/gitserver", "//internal/grpc/defaults", - "//internal/httpcli", "//internal/limiter", "//internal/search", "//internal/search/job", "//internal/search/result", "//internal/search/streaming", - "//internal/search/streaming/http", "//internal/searcher/v1:searcher", "//internal/symbols", "//internal/trace", diff --git a/internal/search/searcher/client.go b/internal/search/searcher/client.go index 57cff6a9b4a..b50823769b0 100644 --- a/internal/search/searcher/client.go +++ b/internal/search/searcher/client.go @@ -3,34 +3,33 @@ package searcher import ( - "bytes" "context" - "encoding/json" "io" - "net/http" + "net/url" "time" - "go.opentelemetry.io/otel/attribute" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/endpoint" "github.com/sourcegraph/sourcegraph/internal/errcode" - "github.com/sourcegraph/sourcegraph/internal/httpcli" + "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" "github.com/sourcegraph/sourcegraph/internal/search" - "github.com/sourcegraph/sourcegraph/internal/trace" + proto "github.com/sourcegraph/sourcegraph/internal/searcher/v1" "github.com/sourcegraph/sourcegraph/lib/errors" ) var ( - searchDoer, _ = httpcli.NewInternalClientFactory("search").Doer() - MockSearch func(ctx context.Context, repo api.RepoName, repoID api.RepoID, commit api.CommitID, p *search.TextPatternInfo, fetchTimeout time.Duration, onMatches func([]*protocol.FileMatch)) (limitHit bool, err error) + MockSearch func(ctx context.Context, repo api.RepoName, repoID api.RepoID, commit api.CommitID, p *search.TextPatternInfo, fetchTimeout time.Duration, onMatch func(*protocol.FileMatch)) (limitHit bool, err error) ) // Search searches repo@commit with p. func Search( ctx context.Context, searcherURLs *endpoint.Map, + connectionCache *defaults.ConnectionCache, repo api.RepoName, repoID api.RepoID, branch string, @@ -40,16 +39,13 @@ func Search( fetchTimeout time.Duration, features search.Features, contextLines int, - onMatches func([]*protocol.FileMatch), + onMatch func(*protocol.FileMatch), ) (limitHit bool, err error) { if MockSearch != nil { - return MockSearch(ctx, repo, repoID, commit, p, fetchTimeout, onMatches) + return MockSearch(ctx, repo, repoID, commit, p, fetchTimeout, onMatch) } - tr, ctx := trace.New(ctx, "searcher.client", repo.Attr(), commit.Attr()) - defer tr.EndWithErr(&err) - - r := protocol.Request{ + r := (&protocol.Request{ Repo: repo, RepoID: repoID, Commit: commit, @@ -73,18 +69,12 @@ func Search( Indexed: indexed, FetchTimeout: fetchTimeout, NumContextLines: int32(contextLines), - } - - body, err := json.Marshal(r) - if err != nil { - return false, err - } + }).ToProto() // Searcher caches the file contents for repo@commit since it is // relatively expensive to fetch from gitserver. So we use consistent // hashing to increase cache hits. consistentHashKey := string(repo) + "@" + string(commit) - tr.AddEvent("calculated hash", attribute.String("consistentHashKey", consistentHashKey)) nodes, err := searcherURLs.Endpoints() if err != nil { @@ -96,90 +86,50 @@ func Search( return false, err } - for attempt := 0; attempt < 2; attempt++ { - url := urls[attempt%len(urls)] - - tr.AddEvent("attempting text search", attribute.String("url", url), attribute.Int("attempt", attempt)) - limitHit, err = textSearchStream(ctx, url, body, onMatches) - if err == nil || errcode.IsTimeout(err) { - return limitHit, err + trySearch := func(attempt int) (bool, error) { + parsed, err := url.Parse(urls[attempt%len(urls)]) + if err != nil { + return false, errors.Wrap(err, "failed to parse URL") } - // If we are canceled, return that error. - if err = ctx.Err(); err != nil { - return false, err - } - - // If not temporary or our last attempt then don't try again. - if !errcode.IsTemporary(err) { - return false, err - } - - tr.AddEvent("transient error", trace.Error(err)) - } - - return false, err -} - -func textSearchStream(ctx context.Context, url string, body []byte, cb func([]*protocol.FileMatch)) (_ bool, err error) { - tr, ctx := trace.New(ctx, "searcher.textSearchStream") - defer tr.EndWithErr(&err) - - req, err := http.NewRequestWithContext(ctx, "GET", url, bytes.NewReader(body)) - if err != nil { - return false, err - } - - resp, err := searchDoer.Do(req) - if err != nil { - // If we failed due to cancellation or timeout (with no partial results in the response - // body), return just that. - if ctx.Err() != nil { - err = ctx.Err() - } - return false, errors.Wrap(err, "streaming searcher request failed") - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - body, err := io.ReadAll(resp.Body) + conn, err := connectionCache.GetConnection(parsed.Host) if err != nil { return false, err } - return false, errors.WithStack(&searcherError{StatusCode: resp.StatusCode, Message: string(body)}) + + client := &automaticRetryClient{proto.NewSearcherServiceClient(conn)} + resp, err := client.Search(ctx, r) + if err != nil { + return false, err + } + + for { + msg, err := resp.Recv() + if errors.Is(err, io.EOF) { + return false, nil + } else if status.Code(err) == codes.Canceled { + return false, context.Canceled + } else if err != nil { + return false, err + } + + switch v := msg.Message.(type) { + case *proto.SearchResponse_FileMatch: + var fm protocol.FileMatch + fm.FromProto(v.FileMatch) + onMatch(&fm) + case *proto.SearchResponse_DoneMessage: + return v.DoneMessage.LimitHit, nil + default: + return false, errors.Newf("unknown SearchResponse message %T", v) + } + } } - var ed EventDone - dec := StreamDecoder{ - OnMatches: cb, - OnDone: func(e EventDone) { - ed = e - }, - OnUnknown: func(event []byte, _ []byte) { - err = errors.Errorf("unknown event %q", event) - }, + limitHit, err = trySearch(0) + if err != nil && errcode.IsTemporary(err) { + // Retry once if we get a temporary error back + limitHit, err = trySearch(1) } - if err := dec.ReadAll(resp.Body); err != nil { - return false, err - } - if ed.Error != "" { - return false, errors.New(ed.Error) - } - return ed.LimitHit, err -} - -type searcherError struct { - StatusCode int - Message string -} - -func (e *searcherError) BadRequest() bool { - return e.StatusCode == http.StatusBadRequest -} - -func (e *searcherError) Temporary() bool { - return e.StatusCode == http.StatusServiceUnavailable -} - -func (e *searcherError) Error() string { - return e.Message + return limitHit, err } diff --git a/internal/search/searcher/client_grpc.go b/internal/search/searcher/client_grpc.go deleted file mode 100644 index d4d43bac6dc..00000000000 --- a/internal/search/searcher/client_grpc.go +++ /dev/null @@ -1,125 +0,0 @@ -// Package searcher provides a client for our just in time text searching -// service "searcher". -package searcher - -import ( - "context" - "io" - "net/url" - "time" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" - "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/endpoint" - "github.com/sourcegraph/sourcegraph/internal/errcode" - "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/search" - proto "github.com/sourcegraph/sourcegraph/internal/searcher/v1" - "github.com/sourcegraph/sourcegraph/lib/errors" -) - -// Search searches repo@commit with p. -func SearchGRPC( - ctx context.Context, - searcherURLs *endpoint.Map, - connectionCache *defaults.ConnectionCache, - repo api.RepoName, - repoID api.RepoID, - branch string, - commit api.CommitID, - indexed bool, - p *search.TextPatternInfo, - fetchTimeout time.Duration, - features search.Features, - contextLines int, - onMatch func(*proto.FileMatch), -) (limitHit bool, err error) { - r := (&protocol.Request{ - Repo: repo, - RepoID: repoID, - Commit: commit, - Branch: branch, - PatternInfo: protocol.PatternInfo{ - Pattern: p.Pattern, - ExcludePattern: p.ExcludePattern, - IncludePatterns: p.IncludePatterns, - Languages: p.Languages, - CombyRule: p.CombyRule, - Select: p.Select.Root(), - Limit: int(p.FileMatchLimit), - IsRegExp: p.IsRegExp, - IsStructuralPat: p.IsStructuralPat, - IsCaseSensitive: p.IsCaseSensitive, - PathPatternsAreCaseSensitive: p.PathPatternsAreCaseSensitive, - IsNegated: p.IsNegated, - PatternMatchesContent: p.PatternMatchesContent, - PatternMatchesPath: p.PatternMatchesPath, - }, - Indexed: indexed, - FetchTimeout: fetchTimeout, - NumContextLines: int32(contextLines), - }).ToProto() - - // Searcher caches the file contents for repo@commit since it is - // relatively expensive to fetch from gitserver. So we use consistent - // hashing to increase cache hits. - consistentHashKey := string(repo) + "@" + string(commit) - - nodes, err := searcherURLs.Endpoints() - if err != nil { - return false, err - } - - urls, err := searcherURLs.GetN(consistentHashKey, len(nodes)) - if err != nil { - return false, err - } - - trySearch := func(attempt int) (bool, error) { - parsed, err := url.Parse(urls[attempt%len(urls)]) - if err != nil { - return false, errors.Wrap(err, "failed to parse URL") - } - - conn, err := connectionCache.GetConnection(parsed.Host) - if err != nil { - return false, err - } - - client := &automaticRetryClient{proto.NewSearcherServiceClient(conn)} - resp, err := client.Search(ctx, r) - if err != nil { - return false, err - } - - for { - msg, err := resp.Recv() - if errors.Is(err, io.EOF) { - return false, nil - } else if status.Code(err) == codes.Canceled { - return false, context.Canceled - } else if err != nil { - return false, err - } - - switch v := msg.Message.(type) { - case *proto.SearchResponse_FileMatch: - onMatch(v.FileMatch) - case *proto.SearchResponse_DoneMessage: - return v.DoneMessage.LimitHit, nil - default: - return false, errors.Newf("unknown SearchResponse message %T", v) - } - } - } - - limitHit, err = trySearch(0) - if err != nil && errcode.IsTemporary(err) { - // Retry once if we get a temporary error back - limitHit, err = trySearch(1) - } - return limitHit, err -} diff --git a/internal/search/searcher/search.go b/internal/search/searcher/search.go index cc797ea1c4d..3d1a2edb46c 100644 --- a/internal/search/searcher/search.go +++ b/internal/search/searcher/search.go @@ -12,7 +12,6 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/endpoint" "github.com/sourcegraph/sourcegraph/internal/errcode" "github.com/sourcegraph/sourcegraph/internal/gitserver" @@ -22,7 +21,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/search/job" "github.com/sourcegraph/sourcegraph/internal/search/result" "github.com/sourcegraph/sourcegraph/internal/search/streaming" - proto "github.com/sourcegraph/sourcegraph/internal/searcher/v1" "github.com/sourcegraph/sourcegraph/internal/trace" "github.com/sourcegraph/sourcegraph/internal/types" ) @@ -202,86 +200,13 @@ func (s *TextSearchJob) searchFilesInRepo( return false, err } - onMatches := func(searcherMatches []*protocol.FileMatch) { + onMatch := func(searcherMatch *protocol.FileMatch) { stream.Send(streaming.SearchEvent{ - Results: convertMatches(repo, commit, &rev, searcherMatches, s.PathRegexps), + Results: convertMatches(repo, commit, &rev, []*protocol.FileMatch{searcherMatch}, s.PathRegexps), }) } - onMatchGRPC := func(searcherMatch *proto.FileMatch) { - stream.Send(streaming.SearchEvent{ - Results: []result.Match{convertProtoMatch(repo, commit, &rev, searcherMatch, s.PathRegexps)}, - }) - } - - if conf.IsGRPCEnabled(ctx) { - return SearchGRPC(ctx, searcherURLs, searcherGRPCConnectionCache, gitserverRepo, repo.ID, rev, commit, index, info, fetchTimeout, s.Features, contextLines, onMatchGRPC) - } else { - return Search(ctx, searcherURLs, gitserverRepo, repo.ID, rev, commit, index, info, fetchTimeout, s.Features, contextLines, onMatches) - } -} - -func convertProtoMatch(repo types.MinimalRepo, commit api.CommitID, rev *string, fm *proto.FileMatch, pathRegexps []*regexp.Regexp) result.Match { - chunkMatches := make(result.ChunkMatches, 0, len(fm.GetChunkMatches())) - for _, cm := range fm.GetChunkMatches() { - ranges := make(result.Ranges, 0, len(cm.GetRanges())) - for _, rr := range cm.Ranges { - ranges = append(ranges, result.Range{ - Start: result.Location{ - Offset: int(rr.GetStart().GetOffset()), - Line: int(rr.GetStart().GetLine()), - Column: int(rr.GetStart().GetColumn()), - }, - End: result.Location{ - Offset: int(rr.GetEnd().GetOffset()), - Line: int(rr.GetEnd().GetLine()), - Column: int(rr.GetEnd().GetColumn()), - }, - }) - } - - chunkMatches = append(chunkMatches, result.ChunkMatch{ - Content: string(cm.GetContent()), - ContentStart: result.Location{ - Offset: int(cm.GetContentStart().GetOffset()), - Line: int(cm.GetContentStart().GetLine()), - Column: 0, - }, - Ranges: ranges, - }) - - } - - var pathMatches []result.Range - for _, pathRe := range pathRegexps { - pathSubmatches := pathRe.FindAllStringSubmatchIndex(string(fm.GetPath()), -1) - for _, sm := range pathSubmatches { - pathMatches = append(pathMatches, result.Range{ - Start: result.Location{ - Offset: sm[0], - Line: 0, - Column: utf8.RuneCountInString(string(fm.GetPath()[:sm[0]])), - }, - End: result.Location{ - Offset: sm[1], - Line: 0, - Column: utf8.RuneCountInString(string(fm.GetPath()[:sm[1]])), - }, - }) - } - } - - return &result.FileMatch{ - File: result.File{ - Path: string(fm.GetPath()), - Repo: repo, - CommitID: commit, - InputRev: rev, - }, - ChunkMatches: chunkMatches, - PathMatches: pathMatches, - LimitHit: fm.GetLimitHit(), - } + return Search(ctx, searcherURLs, searcherGRPCConnectionCache, gitserverRepo, repo.ID, rev, commit, index, info, fetchTimeout, s.Features, contextLines, onMatch) } // convert converts a set of searcher matches into []result.Match diff --git a/internal/search/searcher/stream.go b/internal/search/searcher/stream.go deleted file mode 100644 index 84f78823194..00000000000 --- a/internal/search/searcher/stream.go +++ /dev/null @@ -1,56 +0,0 @@ -package searcher - -import ( - "bytes" - "encoding/json" - "io" - - "github.com/sourcegraph/sourcegraph/cmd/searcher/protocol" - streamhttp "github.com/sourcegraph/sourcegraph/internal/search/streaming/http" - "github.com/sourcegraph/sourcegraph/lib/errors" -) - -type StreamDecoder struct { - OnMatches func([]*protocol.FileMatch) - OnDone func(EventDone) - OnUnknown func(event, data []byte) -} - -func (rr StreamDecoder) ReadAll(r io.Reader) error { - dec := streamhttp.NewDecoder(r) - for dec.Scan() { - event := dec.Event() - data := dec.Data() - if bytes.Equal(event, []byte("matches")) { - if rr.OnMatches == nil { - continue - } - var d []*protocol.FileMatch - if err := json.Unmarshal(data, &d); err != nil { - return errors.Wrap(err, "decode matches payload") - } - rr.OnMatches(d) - } else if bytes.Equal(event, []byte("done")) { - if rr.OnDone == nil { - continue - } - var e EventDone - if err := json.Unmarshal(data, &e); err != nil { - return errors.Wrap(err, "decode done payload") - } - rr.OnDone(e) - break // done will always be the last event - } else { - if rr.OnUnknown == nil { - continue - } - rr.OnUnknown(event, data) - } - } - return dec.Err() -} - -type EventDone struct { - LimitHit bool `json:"limit_hit"` - Error string `json:"error"` -} diff --git a/internal/search/structural/structural.go b/internal/search/structural/structural.go index 00c13aceb9e..6d483e6e504 100644 --- a/internal/search/structural/structural.go +++ b/internal/search/structural/structural.go @@ -170,7 +170,7 @@ func (s *SearchJob) Run(ctx context.Context, clients job.RuntimeClients, stream _, ctx, stream, finish := job.StartSpan(ctx, stream, s) defer func() { finish(alert, err) }() - repos := searchrepos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.Zoekt) + repos := searchrepos.NewResolver(clients.Logger, clients.DB, clients.Gitserver, clients.SearcherURLs, clients.SearcherGRPCConnectionCache, clients.Zoekt) it := repos.Iterator(ctx, s.RepoOpts) for it.Next() { diff --git a/internal/symbols/BUILD.bazel b/internal/symbols/BUILD.bazel index 5b0a502c35d..7ccf3baa362 100644 --- a/internal/symbols/BUILD.bazel +++ b/internal/symbols/BUILD.bazel @@ -13,11 +13,9 @@ go_library( "//internal/actor", "//internal/api", "//internal/authz", - "//internal/conf", "//internal/conf/conftypes", "//internal/endpoint", "//internal/grpc/defaults", - "//internal/httpcli", "//internal/limiter", "//internal/search", "//internal/search/result", diff --git a/internal/symbols/client.go b/internal/symbols/client.go index 8e282fbac56..59a4a4301a5 100644 --- a/internal/symbols/client.go +++ b/internal/symbols/client.go @@ -1,12 +1,8 @@ package symbols import ( - "bytes" "context" - "encoding/json" "io" - "net/http" - "strings" "github.com/sourcegraph/log" "go.opentelemetry.io/otel/attribute" @@ -16,11 +12,9 @@ import ( "github.com/sourcegraph/sourcegraph/internal/actor" "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/authz" - "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/endpoint" "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/internal/httpcli" "github.com/sourcegraph/sourcegraph/internal/limiter" "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/result" @@ -40,7 +34,6 @@ func LoadConfig() { DefaultClient = &Client{ Endpoints: defaultEndpoints(), GRPCConnectionCache: defaults.NewConnectionCache(log.Scoped("symbolsConnectionCache")), - HTTPClient: defaultDoer, HTTPLimiter: limiter.New(500), SubRepoPermsChecker: func() authz.SubRepoPermissionChecker { return authz.DefaultSubRepoPermsChecker }, } @@ -50,14 +43,6 @@ func LoadConfig() { // SYMBOLS_URL environment variable. var DefaultClient *Client -var defaultDoer = func() httpcli.Doer { - d, err := httpcli.NewInternalClientFactory("symbols").Doer() - if err != nil { - panic(err) - } - return d -}() - // Client is a symbols service client. type Client struct { // Endpoints to symbols service. @@ -65,9 +50,6 @@ type Client struct { GRPCConnectionCache *defaults.ConnectionCache - // HTTP client to use - HTTPClient httpcli.Doer - // Limits concurrency of outstanding HTTP posts HTTPLimiter limiter.Limiter @@ -84,14 +66,7 @@ func (c *Client) Search(ctx context.Context, args search.SymbolsParameters) (sym args.CommitID.Attr()) defer tr.EndWithErr(&err) - var response search.SymbolsResponse - - if conf.IsGRPCEnabled(ctx) { - response, err = c.searchGRPC(ctx, args) - } else { - response, err = c.searchJSON(ctx, args) - } - + response, err := c.searchGRPC(ctx, args) if err != nil { return nil, errors.Wrap(err, "executing symbols search request") } @@ -147,49 +122,12 @@ func (c *Client) searchGRPC(ctx context.Context, args search.SymbolsParameters) return response, nil } -func (c *Client) searchJSON(ctx context.Context, args search.SymbolsParameters) (search.SymbolsResponse, error) { - resp, err := c.httpPost(ctx, "search", args.Repo, args) - if err != nil { - return search.SymbolsResponse{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - // best-effort inclusion of body in error message - body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) - return search.SymbolsResponse{}, errors.Errorf( - "Symbol.Search http status %d: %s", - resp.StatusCode, - string(body), - ) - } - - var response search.SymbolsResponse - err = json.NewDecoder(resp.Body).Decode(&response) - if err != nil { - return search.SymbolsResponse{}, err - } - if response.Err != "" { - return search.SymbolsResponse{}, errors.New(response.Err) - } - - return response, nil -} - -func (c *Client) LocalCodeIntel(ctx context.Context, args types.RepoCommitPath) (result *types.LocalCodeIntelPayload, err error) { +func (c *Client) LocalCodeIntel(ctx context.Context, path types.RepoCommitPath) (result *types.LocalCodeIntelPayload, err error) { tr, ctx := trace.New(ctx, "symbols.LocalCodeIntel", - attribute.String("repo", args.Repo), - attribute.String("commitID", args.Commit)) + attribute.String("repo", path.Repo), + attribute.String("commitID", path.Commit)) defer tr.EndWithErr(&err) - if conf.IsGRPCEnabled(ctx) { - return c.localCodeIntelGRPC(ctx, args) - } - - return c.localCodeIntelJSON(ctx, args) -} - -func (c *Client) localCodeIntelGRPC(ctx context.Context, path types.RepoCommitPath) (result *types.LocalCodeIntelPayload, err error) { grpcClient, err := c.gRPCClient(path.Repo) if err != nil { return nil, errors.Wrap(err, "getting gRPC symbols client") @@ -238,43 +176,13 @@ func (c *Client) localCodeIntelGRPC(ctx context.Context, path types.RepoCommitPa } } -func (c *Client) localCodeIntelJSON(ctx context.Context, args types.RepoCommitPath) (result *types.LocalCodeIntelPayload, err error) { - resp, err := c.httpPost(ctx, "localCodeIntel", api.RepoName(args.Repo), args) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - // best-effort inclusion of body in error message - body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) - return nil, errors.Errorf( - "Squirrel.LocalCodeIntel http status %d: %s", - resp.StatusCode, - string(body), - ) - } - - err = json.NewDecoder(resp.Body).Decode(&result) - if err != nil { - return nil, errors.Wrap(err, "decoding response body") - } - - return result, nil -} - func (c *Client) SymbolInfo(ctx context.Context, args types.RepoCommitPathPoint) (result *types.SymbolInfo, err error) { tr, ctx := trace.New(ctx, "squirrel.SymbolInfo", attribute.String("repo", args.Repo), attribute.String("commitID", args.Commit)) defer tr.EndWithErr(&err) - if conf.IsGRPCEnabled(ctx) { - result, err = c.symbolInfoGRPC(ctx, args) - } else { - result, err = c.symbolInfoJSON(ctx, args) - } - + result, err = c.symbolInfoGRPC(ctx, args) if err != nil { return nil, errors.Wrap(err, "executing symbol info request") } @@ -336,71 +244,6 @@ func (c *Client) symbolInfoGRPC(ctx context.Context, args types.RepoCommitPathPo return protoResponse.ToInternal(), nil } -func (c *Client) symbolInfoJSON(ctx context.Context, args types.RepoCommitPathPoint) (result *types.SymbolInfo, err error) { - resp, err := c.httpPost(ctx, "symbolInfo", api.RepoName(args.Repo), args) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - // best-effort inclusion of body in error message - body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) - return nil, errors.Errorf( - "Squirrel.SymbolInfo http status %d: %s", - resp.StatusCode, - string(body), - ) - } - - err = json.NewDecoder(resp.Body).Decode(&result) - if err != nil { - return nil, errors.Wrap(err, "decoding response body") - } - - return result, nil -} - -func (c *Client) httpPost( - ctx context.Context, - method string, - repo api.RepoName, - payload any, -) (resp *http.Response, err error) { - tr, ctx := trace.New(ctx, "symbols.httpPost", - attribute.String("method", method), - repo.Attr()) - defer tr.EndWithErr(&err) - - symbolsURL, err := c.url(repo) - if err != nil { - return nil, err - } - - reqBody, err := json.Marshal(payload) - if err != nil { - return nil, err - } - - if !strings.HasSuffix(symbolsURL, "/") { - symbolsURL += "/" - } - req, err := http.NewRequest("POST", symbolsURL+method, bytes.NewReader(reqBody)) - if err != nil { - return nil, err - } - - req.Header.Set("Content-Type", "application/json") - req = req.WithContext(ctx) - - tr.AddEvent("Waiting on HTTP limiter") - c.HTTPLimiter.Acquire() - defer c.HTTPLimiter.Release() - tr.AddEvent("Acquired HTTP limiter") - - return c.HTTPClient.Do(req) -} - func (c *Client) gRPCClient(repo string) (proto.SymbolsServiceClient, error) { address, err := c.Endpoints.Get(repo) if err != nil { @@ -415,13 +258,6 @@ func (c *Client) gRPCClient(repo string) (proto.SymbolsServiceClient, error) { return &automaticRetryClient{base: proto.NewSymbolsServiceClient(conn)}, nil } -func (c *Client) url(repo api.RepoName) (string, error) { - if c.Endpoints == nil { - return "", errors.New("a symbols service has not been configured") - } - return c.Endpoints.Get(string(repo)) -} - // translateGRPCError translates gRPC errors to their corresponding context errors, if applicable. func translateGRPCError(err error) error { st, ok := status.FromError(err) diff --git a/mockgen.temp.yaml b/mockgen.temp.yaml index cfb1abddb6d..a4e2ebdb77f 100644 --- a/mockgen.temp.yaml +++ b/mockgen.temp.yaml @@ -136,3 +136,4 @@ path: github.com/sourcegraph/sourcegraph/internal/gitserver/v1 interfaces: - GitserverServiceClient + - GitserverService_ExecServer diff --git a/monitoring/definitions/embeddings.go b/monitoring/definitions/embeddings.go index ca9d1447eb5..92a682806c2 100644 --- a/monitoring/definitions/embeddings.go +++ b/monitoring/definitions/embeddings.go @@ -30,7 +30,6 @@ func Embeddings() *monitoring.Dashboard { InstanceFilterRegex: `${instance:regex}`, }, monitoring.ObservableOwnerInfraOrg), shared.NewDatabaseConnectionsMonitoringGroup(containerName, monitoring.ObservableOwnerCody), - shared.NewFrontendInternalAPIErrorResponseMonitoringGroup(containerName, monitoring.ObservableOwnerCody, nil), shared.NewContainerMonitoringGroup(containerName, monitoring.ObservableOwnerCody, nil), shared.NewProvisioningIndicatorsGroup(containerName, monitoring.ObservableOwnerCody, nil), shared.NewGolangMonitoringGroup(containerName, monitoring.ObservableOwnerCody, nil), diff --git a/monitoring/definitions/frontend.go b/monitoring/definitions/frontend.go index 6c66c0b1f98..d27271a6dfc 100644 --- a/monitoring/definitions/frontend.go +++ b/monitoring/definitions/frontend.go @@ -472,17 +472,6 @@ func Frontend() *monitoring.Dashboard { - Check the Searcher dashboard for indications it might be unhealthy. `, }, - { - Name: "internalapi_error_responses", - Description: "internal API error responses every 5m by route", - Query: `sum by(category) (increase(src_frontend_internal_request_duration_seconds_count{code!~"2.."}[5m])) / ignoring(code) group_left sum(increase(src_frontend_internal_request_duration_seconds_count[5m])) * 100`, - Warning: monitoring.Alert().GreaterOrEqual(5).For(15 * time.Minute), - Panel: monitoring.Panel().LegendFormat("{{category}}").Unit(monitoring.Percentage), - Owner: monitoring.ObservableOwnerSource, - NextSteps: ` - - May not be a substantial issue, check the 'frontend' logs for potential causes. - `, - }, }, { { diff --git a/monitoring/definitions/git_server.go b/monitoring/definitions/git_server.go index 982a7e08085..810854a0d5c 100644 --- a/monitoring/definitions/git_server.go +++ b/monitoring/definitions/git_server.go @@ -264,7 +264,6 @@ func GitServer() *monitoring.Dashboard { - **Kubernetes and Docker Compose:** Check that you are running a similar number of git server replicas and that their CPU/memory limits are allocated according to what is shown in the [Sourcegraph resource estimator](../deploy/resource_estimator.md). `, }, - shared.FrontendInternalAPIErrorResponses("gitserver", monitoring.ObservableOwnerSource).Observable(), }, { { diff --git a/monitoring/definitions/precise_code_intel_worker.go b/monitoring/definitions/precise_code_intel_worker.go index 0cc479df6ad..5fdffd44e5a 100644 --- a/monitoring/definitions/precise_code_intel_worker.go +++ b/monitoring/definitions/precise_code_intel_worker.go @@ -22,7 +22,6 @@ func PreciseCodeIntelWorker() *monitoring.Dashboard { shared.CodeIntelligence.NewUploadStoreGroup(containerName), // Resource monitoring - shared.NewFrontendInternalAPIErrorResponseMonitoringGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), shared.NewDatabaseConnectionsMonitoringGroup(containerName, monitoring.ObservableOwnerInfraOrg), shared.NewContainerMonitoringGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), shared.NewProvisioningIndicatorsGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), diff --git a/monitoring/definitions/repo_updater.go b/monitoring/definitions/repo_updater.go index 39310e6b0c1..caf326ad0f4 100644 --- a/monitoring/definitions/repo_updater.go +++ b/monitoring/definitions/repo_updater.go @@ -601,7 +601,6 @@ func RepoUpdater() *monitoring.Dashboard { InstanceFilterRegex: `${instance:regex}`, }, monitoring.ObservableOwnerInfraOrg), shared.HTTP.NewHandlersGroup(containerName), - shared.NewFrontendInternalAPIErrorResponseMonitoringGroup(containerName, monitoring.ObservableOwnerSource, nil), shared.NewDatabaseConnectionsMonitoringGroup(containerName, monitoring.ObservableOwnerSource), shared.NewContainerMonitoringGroup(containerName, monitoring.ObservableOwnerSource, containerMonitoringOptions), shared.NewProvisioningIndicatorsGroup(containerName, monitoring.ObservableOwnerSource, nil), diff --git a/monitoring/definitions/searcher.go b/monitoring/definitions/searcher.go index 62c69ab499a..5d946c2b83d 100644 --- a/monitoring/definitions/searcher.go +++ b/monitoring/definitions/searcher.go @@ -245,7 +245,6 @@ regularly above 0 it is a sign for further investigation.`, InstanceFilterRegex: `${instance:regex}`, }, monitoring.ObservableOwnerInfraOrg), shared.NewDatabaseConnectionsMonitoringGroup(containerName, monitoring.ObservableOwnerInfraOrg), - shared.NewFrontendInternalAPIErrorResponseMonitoringGroup(containerName, monitoring.ObservableOwnerSearchCore, nil), shared.NewContainerMonitoringGroup(containerName, monitoring.ObservableOwnerSearchCore, nil), shared.NewProvisioningIndicatorsGroup(containerName, monitoring.ObservableOwnerSearchCore, nil), shared.NewGolangMonitoringGroup(containerName, monitoring.ObservableOwnerSearchCore, nil), diff --git a/monitoring/definitions/shared/BUILD.bazel b/monitoring/definitions/shared/BUILD.bazel index 552e7e9d759..f532b2e1150 100644 --- a/monitoring/definitions/shared/BUILD.bazel +++ b/monitoring/definitions/shared/BUILD.bazel @@ -16,7 +16,6 @@ go_library( "dbconns.go", "diskmetrics.go", "executors.go", - "frontend.go", "gitserver.go", "go.go", "grpc.go", diff --git a/monitoring/definitions/shared/frontend.go b/monitoring/definitions/shared/frontend.go deleted file mode 100644 index 14cac7bbfe7..00000000000 --- a/monitoring/definitions/shared/frontend.go +++ /dev/null @@ -1,52 +0,0 @@ -package shared - -import ( - "fmt" - "strings" - "time" - - "github.com/sourcegraph/sourcegraph/monitoring/monitoring" -) - -var FrontendInternalAPIErrorResponses sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable { - return Observable{ - Name: "frontend_internal_api_error_responses", - Description: "frontend-internal API error responses every 5m by route", - Query: fmt.Sprintf(`sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="%[1]s",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="%[1]s"}[5m]))`, containerName), - Warning: monitoring.Alert().GreaterOrEqual(2).For(5 * time.Minute), - Panel: monitoring.Panel().LegendFormat("{{category}}").Unit(monitoring.Percentage), - Owner: owner, - NextSteps: strings.ReplaceAll(` - - **Single-container deployments:** Check 'docker logs $CONTAINER_ID' for logs starting with 'repo-updater' that indicate requests to the frontend service are failing. - - **Kubernetes:** - - Confirm that 'kubectl get pods' shows the 'frontend' pods are healthy. - - Check 'kubectl logs {{CONTAINER_NAME}}' for logs indicate request failures to 'frontend' or 'frontend-internal'. - - **Docker Compose:** - - Confirm that 'docker ps' shows the 'frontend-internal' container is healthy. - - Check 'docker logs {{CONTAINER_NAME}}' for logs indicating request failures to 'frontend' or 'frontend-internal'. - `, "{{CONTAINER_NAME}}", containerName), - } -} - -type FrontendInternalAPIERrorResponseMonitoringOptions struct { - // ErrorResponses transforms the default observable used to construct the error responses panel. - ErrorResponses ObservableOption -} - -// NewProvisioningIndicatorsGroup creates a group containing panels displaying -// internal API error response metrics for the given container. -func NewFrontendInternalAPIErrorResponseMonitoringGroup(containerName string, owner monitoring.ObservableOwner, options *FrontendInternalAPIERrorResponseMonitoringOptions) monitoring.Group { - if options == nil { - options = &FrontendInternalAPIERrorResponseMonitoringOptions{} - } - - return monitoring.Group{ - Title: "Internal service requests", - Hidden: true, - Rows: []monitoring.Row{ - { - options.ErrorResponses.safeApply(FrontendInternalAPIErrorResponses(containerName, owner)).Observable(), - }, - }, - } -} diff --git a/monitoring/definitions/symbols.go b/monitoring/definitions/symbols.go index 48dbf2590cc..5ac02514777 100644 --- a/monitoring/definitions/symbols.go +++ b/monitoring/definitions/symbols.go @@ -63,7 +63,6 @@ func Symbols() *monitoring.Dashboard { InstanceFilterRegex: `${instance:regex}`, }, monitoring.ObservableOwnerInfraOrg), shared.NewDatabaseConnectionsMonitoringGroup(containerName, monitoring.ObservableOwnerInfraOrg), - shared.NewFrontendInternalAPIErrorResponseMonitoringGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), shared.NewContainerMonitoringGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), shared.NewProvisioningIndicatorsGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), shared.NewGolangMonitoringGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), diff --git a/monitoring/definitions/worker.go b/monitoring/definitions/worker.go index e683d0e79d1..8031fb8f67d 100644 --- a/monitoring/definitions/worker.go +++ b/monitoring/definitions/worker.go @@ -253,7 +253,6 @@ func Worker() *monitoring.Dashboard { }, // Resource monitoring - shared.NewFrontendInternalAPIErrorResponseMonitoringGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), shared.NewDatabaseConnectionsMonitoringGroup(containerName, monitoring.ObservableOwnerInfraOrg), shared.NewContainerMonitoringGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), shared.NewProvisioningIndicatorsGroup(containerName, monitoring.ObservableOwnerCodeIntel, nil), diff --git a/schema/schema.go b/schema/schema.go index 2bcb0ae9f5a..71b7c7b925f 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -903,8 +903,6 @@ type ExperimentalFeatures struct { CustomGitFetch []*CustomGitFetchMapping `json:"customGitFetch,omitempty"` // DebugLog description: Turns on debug logging for specific debugging scenarios. DebugLog *DebugLog `json:"debug.log,omitempty"` - // EnableGRPC description: Enables gRPC for communication between internal services - EnableGRPC *bool `json:"enableGRPC,omitempty"` // EnableGithubInternalRepoVisibility description: Enable support for visibility of internal Github repositories EnableGithubInternalRepoVisibility bool `json:"enableGithubInternalRepoVisibility,omitempty"` // EnablePermissionsWebhooks description: DEPRECATED: No longer has any effect. @@ -998,7 +996,6 @@ func (v *ExperimentalFeatures) UnmarshalJSON(data []byte) error { delete(m, "batchChanges.enablePerforce") delete(m, "customGitFetch") delete(m, "debug.log") - delete(m, "enableGRPC") delete(m, "enableGithubInternalRepoVisibility") delete(m, "enablePermissionsWebhooks") delete(m, "enableStorm") diff --git a/schema/site.schema.json b/schema/site.schema.json index f39c6572244..02c6af97743 100644 --- a/schema/site.schema.json +++ b/schema/site.schema.json @@ -428,14 +428,6 @@ "type": "boolean", "default": false }, - "enableGRPC": { - "description": "Enables gRPC for communication between internal services", - "type": "boolean", - "default": true, - "!go": { - "pointer": true - } - }, "enablePermissionsWebhooks": { "description": "DEPRECATED: No longer has any effect.", "type": "boolean", diff --git a/sg.config.yaml b/sg.config.yaml index 312e5ebfbdc..f66df65b3a1 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -43,7 +43,6 @@ env: PROMETHEUS_URL: http://localhost:9090 JAEGER_SERVER_URL: http://localhost:16686 - SRC_GRPC_ENABLE_CONF: 'true' SRC_DEVELOPMENT: 'true' SRC_PROF_HTTP: '' SRC_PROF_SERVICES: | diff --git a/testing/BUILD.bazel b/testing/BUILD.bazel index a138cbc2009..034430a336f 100644 --- a/testing/BUILD.bazel +++ b/testing/BUILD.bazel @@ -15,7 +15,6 @@ server_integration_test( "//cmd/server:image_tarball", ], env = { - "SG_FEATURE_FLAG_GRPC": "true", "TEST_USER_EMAIL": "test@sourcegraph.com", "TEST_USER_PASSWORD": "supersecurepassword", "SOURCEGRAPH_SUDO_TOKEN": "fake-sg-token", @@ -54,9 +53,6 @@ server_integration_test( "//dev/authtest:authtest_test", "//dev/gqltest:gqltest_test", ], - env = { - "SG_FEATURE_FLAG_GRPC": "false", - }, env_inherit = [ "AWS_ACCESS_KEY_ID", "AWS_CODE_COMMIT_PASSWORD", @@ -78,46 +74,6 @@ server_integration_test( runner_src = ":backend_integration_test.sh", ) -server_integration_test( - name = "grpc_backend_integration_test", - timeout = "long", - args = [ - "$(location //dev/gqltest:gqltest_test)", # actual test - "$(location //dev/authtest:authtest_test)", # actual test - ], - data = [ - "//cmd/server:image_tarball", - "//dev/authtest:authtest_test", - "//dev/gqltest:gqltest_test", - ], - env = { - "SG_FEATURE_FLAG_GRPC": "true", - }, - env_inherit = [ - "AWS_ACCESS_KEY_ID", - "AWS_CODE_COMMIT_PASSWORD", - "AWS_CODE_COMMIT_USERNAME", - "AWS_SECRET_ACCESS_KEY", - "AZURE_DEVOPS_TOKEN", - "AZURE_DEVOPS_USERNAME", - "BITBUCKET_SERVER_TOKEN", - "BITBUCKET_SERVER_URL", - "BITBUCKET_SERVER_USERNAME", - "GHE_GITHUB_TOKEN", - "PERFORCE_PASSWORD", - "PERFORCE_PORT", - "PERFORCE_USER", - "SOURCEGRAPH_LICENSE_GENERATION_KEY", - "SOURCEGRAPH_LICENSE_KEY", - ], - port = "7082", - runner_src = ":backend_integration_test.sh", - tags = [ - "exclusive", - "manual", - ], -) - server_integration_test( name = "codeintel_integration_test", args = [ @@ -140,7 +96,6 @@ server_integration_test( "//internal/cmd/init-sg", ], env = { - "SG_FEATURE_FLAG_GRPC": "true", "TEST_USER_EMAIL": "test@sourcegraph.com", "TEST_USER_PASSWORD": "supersecurepassword", "SOURCEGRAPH_SUDO_USER": "admin", @@ -225,7 +180,6 @@ server_integration_test( ], }), env = { - "SG_FEATURE_FLAG_GRPC": "true", "TEST_USER_EMAIL": "test@sourcegraph.com", "TEST_USER_PASSWORD": "supersecurepassword", "SOURCEGRAPH_SUDO_USER": "admin", diff --git a/testing/tools/integration_runner.sh b/testing/tools/integration_runner.sh index a64a0cf3784..448f8274425 100755 --- a/testing/tools/integration_runner.sh +++ b/testing/tools/integration_runner.sh @@ -61,9 +61,9 @@ function generate_unique_container_name() { local ident # try generate a unique identifier with openssl otherwise fallback to dd and hexdump - if command -v openssl &> /dev/null; then + if command -v openssl &>/dev/null; then ident="$(openssl rand -hex 12)" - elif command -v hexdump &> /dev/null; then + elif command -v hexdump &>/dev/null; then ident="$(dd if=/dev/urandom bs=12 count=1 2>/dev/null | hexdump -e '24/1 "%02x"')" else echo "⚠️ Missing openssl or hexdump. Unable to generate unique id" @@ -110,7 +110,6 @@ function _run_server_image() { echo "License key present: $(is_present "$SOURCEGRAPH_LICENSE_GENERATION_KEY")" echo "Allow single docker image code insights: $ALLOW_SINGLE_DOCKER_CODE_INSIGHTS" - echo "GRPC Feature flag: $SG_FEATURE_FLAG_GRPC" # shellcheck disable=SC2086 docker run $docker_args \ @@ -121,7 +120,6 @@ function _run_server_image() { -e BAZEL_SKIP_OOB_INFER_VERSION=true \ -e ALLOW_SINGLE_DOCKER_CODE_INSIGHTS="$ALLOW_SINGLE_DOCKER_CODE_INSIGHTS" \ -e SOURCEGRAPH_LICENSE_GENERATION_KEY="$SOURCEGRAPH_LICENSE_GENERATION_KEY" \ - -e SG_FEATURE_FLAG_GRPC="$SG_FEATURE_FLAG_GRPC" \ -e DB_STARTUP_TIMEOUT="$DB_STARTUP_TIMEOUT" \ "$image_name" @@ -145,7 +143,7 @@ function wait_until_container_ready() { # shellcheck disable=SC2181 while [ ! $? -eq 0 ]; do sleep 5 - t=$(( t + 5 )) + t=$((t + 5)) if [ "$t" -gt "$timeout" ]; then echo "$url was not accessible within $timeout." docker inspect "$name"