gitserver: Add debug endpoint to inspect repository locker (#58479)

There's little insight into what's currently being done by the syncer, so adding this endpoint as a stopgap until we have a DB backed queue and more insight for free through that.

cmd/gitserver/internal/integration_tests/mocks.go

@@ -273,6 +273,9 @@ func (c RepositoryLockSetStatusFuncCall) Results() []interface{} {
 // github.com/sourcegraph/sourcegraph/cmd/gitserver/internal) used for unit
 // testing.
 type MockRepositoryLocker struct {
+	// AllStatusesFunc is an instance of a mock function object controlling
+	// the behavior of the method AllStatuses.
+	AllStatusesFunc *RepositoryLockerAllStatusesFunc
 	// StatusFunc is an instance of a mock function object controlling the
 	// behavior of the method Status.
 	StatusFunc *RepositoryLockerStatusFunc
@@ -286,6 +289,11 @@ type MockRepositoryLocker struct {
 // overwritten.
 func NewMockRepositoryLocker() *MockRepositoryLocker {
 	return &MockRepositoryLocker{
+		AllStatusesFunc: &RepositoryLockerAllStatusesFunc{
+			defaultHook: func() (r0 map[common.GitDir]string) {
+				return
+			},
+		},
 		StatusFunc: &RepositoryLockerStatusFunc{
 			defaultHook: func(common.GitDir) (r0 string, r1 bool) {
 				return
@@ -303,6 +311,11 @@ func NewMockRepositoryLocker() *MockRepositoryLocker {
 // interface. All methods panic on invocation, unless overwritten.
 func NewStrictMockRepositoryLocker() *MockRepositoryLocker {
 	return &MockRepositoryLocker{
+		AllStatusesFunc: &RepositoryLockerAllStatusesFunc{
+			defaultHook: func() map[common.GitDir]string {
+				panic("unexpected invocation of MockRepositoryLocker.AllStatuses")
+			},
+		},
 		StatusFunc: &RepositoryLockerStatusFunc{
 			defaultHook: func(common.GitDir) (string, bool) {
 				panic("unexpected invocation of MockRepositoryLocker.Status")
@@ -321,6 +334,9 @@ func NewStrictMockRepositoryLocker() *MockRepositoryLocker {
 // implementation, unless overwritten.
 func NewMockRepositoryLockerFrom(i internal.RepositoryLocker) *MockRepositoryLocker {
 	return &MockRepositoryLocker{
+		AllStatusesFunc: &RepositoryLockerAllStatusesFunc{
+			defaultHook: i.AllStatuses,
+		},
 		StatusFunc: &RepositoryLockerStatusFunc{
 			defaultHook: i.Status,
 		},
@@ -330,6 +346,106 @@ func NewMockRepositoryLockerFrom(i internal.RepositoryLocker) *MockRepositoryLoc
 	}
 }
 
+// RepositoryLockerAllStatusesFunc describes the behavior when the
+// AllStatuses method of the parent MockRepositoryLocker instance is
+// invoked.
+type RepositoryLockerAllStatusesFunc struct {
+	defaultHook func() map[common.GitDir]string
+	hooks       []func() map[common.GitDir]string
+	history     []RepositoryLockerAllStatusesFuncCall
+	mutex       sync.Mutex
+}
+
+// AllStatuses delegates to the next hook function in the queue and stores
+// the parameter and result values of this invocation.
+func (m *MockRepositoryLocker) AllStatuses() map[common.GitDir]string {
+	r0 := m.AllStatusesFunc.nextHook()()
+	m.AllStatusesFunc.appendCall(RepositoryLockerAllStatusesFuncCall{r0})
+	return r0
+}
+
+// SetDefaultHook sets function that is called when the AllStatuses method
+// of the parent MockRepositoryLocker instance is invoked and the hook queue
+// is empty.
+func (f *RepositoryLockerAllStatusesFunc) SetDefaultHook(hook func() map[common.GitDir]string) {
+	f.defaultHook = hook
+}
+
+// PushHook adds a function to the end of hook queue. Each invocation of the
+// AllStatuses method of the parent MockRepositoryLocker instance invokes
+// the hook at the front of the queue and discards it. After the queue is
+// empty, the default hook function is invoked for any future action.
+func (f *RepositoryLockerAllStatusesFunc) PushHook(hook func() map[common.GitDir]string) {
+	f.mutex.Lock()
+	f.hooks = append(f.hooks, hook)
+	f.mutex.Unlock()
+}
+
+// SetDefaultReturn calls SetDefaultHook with a function that returns the
+// given values.
+func (f *RepositoryLockerAllStatusesFunc) SetDefaultReturn(r0 map[common.GitDir]string) {
+	f.SetDefaultHook(func() map[common.GitDir]string {
+		return r0
+	})
+}
+
+// PushReturn calls PushHook with a function that returns the given values.
+func (f *RepositoryLockerAllStatusesFunc) PushReturn(r0 map[common.GitDir]string) {
+	f.PushHook(func() map[common.GitDir]string {
+		return r0
+	})
+}
+
+func (f *RepositoryLockerAllStatusesFunc) nextHook() func() map[common.GitDir]string {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	if len(f.hooks) == 0 {
+		return f.defaultHook
+	}
+
+	hook := f.hooks[0]
+	f.hooks = f.hooks[1:]
+	return hook
+}
+
+func (f *RepositoryLockerAllStatusesFunc) appendCall(r0 RepositoryLockerAllStatusesFuncCall) {
+	f.mutex.Lock()
+	f.history = append(f.history, r0)
+	f.mutex.Unlock()
+}
+
+// History returns a sequence of RepositoryLockerAllStatusesFuncCall objects
+// describing the invocations of this function.
+func (f *RepositoryLockerAllStatusesFunc) History() []RepositoryLockerAllStatusesFuncCall {
+	f.mutex.Lock()
+	history := make([]RepositoryLockerAllStatusesFuncCall, len(f.history))
+	copy(history, f.history)
+	f.mutex.Unlock()
+
+	return history
+}
+
+// RepositoryLockerAllStatusesFuncCall is an object that describes an
+// invocation of method AllStatuses on an instance of MockRepositoryLocker.
+type RepositoryLockerAllStatusesFuncCall struct {
+	// Result0 is the value of the 1st result returned from this method
+	// invocation.
+	Result0 map[common.GitDir]string
+}
+
+// Args returns an interface slice containing the arguments of this
+// invocation.
+func (c RepositoryLockerAllStatusesFuncCall) Args() []interface{} {
+	return []interface{}{}
+}
+
+// Results returns an interface slice containing the results of this
+// invocation.
+func (c RepositoryLockerAllStatusesFuncCall) Results() []interface{} {
+	return []interface{}{c.Result0}
+}
+
 // RepositoryLockerStatusFunc describes the behavior when the Status method
 // of the parent MockRepositoryLocker instance is invoked.
 type RepositoryLockerStatusFunc struct {

cmd/gitserver/internal/lock.go

@@ -35,6 +35,8 @@ type RepositoryLocker interface {
 	// Status returns the status of the locked directory dir. If dir is not
 	// locked, then locked is false.
 	Status(dir common.GitDir) (status string, locked bool)
+	// AllStatuses returns the status of all locked directories.
+	AllStatuses() map[common.GitDir]string
 }
 
 func NewRepositoryLocker() RepositoryLocker {
@@ -88,6 +90,18 @@ func (rl *repositoryLocker) Status(dir common.GitDir) (status string, locked boo
 	return
 }
 
+func (rl *repositoryLocker) AllStatuses() map[common.GitDir]string {
+	rl.mu.RLock()
+	defer rl.mu.RUnlock()
+
+	statuses := make(map[common.GitDir]string, len(rl.status))
+	for dir, status := range rl.status {
+		statuses[dir] = status
+	}
+
+	return statuses
+}
+
 type repositoryLock struct {
 	unlock    func()
 	setStatus func(status string)

cmd/gitserver/shared/debug.go

@@ -1,11 +1,23 @@
 package shared
 
 import (
+	"net/http"
+
 	"github.com/sourcegraph/sourcegraph/internal/debugserver"
 )
 
-// GRPCWebUIDebugEndpoint returns a debug endpoint that serves the GRPCWebUI that targets
-// this gitserver instance.
-func GRPCWebUIDebugEndpoint(addr string) debugserver.Endpoint {
-	return debugserver.NewGRPCWebUIEndpoint("gitserver", addr)
+func createDebugServerEndpoints(ready chan struct{}, addr string, debugserverEndpoints *LazyDebugserverEndpoint) []debugserver.Endpoint {
+	return []debugserver.Endpoint{
+		debugserver.NewGRPCWebUIEndpoint("gitserver", addr),
+		{
+			Name: "Repository Locker State",
+			Path: "/repository-locker-state",
+			Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				// wait until we're healthy to respond
+				<-ready
+				// lockerStatusEndpoint is guaranteed to be assigned now
+				debugserverEndpoints.lockerStatusEndpoint(w, r)
+			}),
+		},
+	}
 }

cmd/gitserver/shared/service.go

@@ -9,21 +9,31 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/service"
 )
 
-type svc struct{}
+type svc struct {
+	ready                chan struct{}
+	debugServerEndpoints LazyDebugserverEndpoint
+}
 
 func (svc) Name() string { return "gitserver" }
 
-func (svc) Configure() (env.Config, []debugserver.Endpoint) {
+func (s *svc) Configure() (env.Config, []debugserver.Endpoint) {
+	s.ready = make(chan struct{})
+
 	c := LoadConfig()
-	endpoints := []debugserver.Endpoint{
-		GRPCWebUIDebugEndpoint(c.ListenAddress),
-	}
 
-	return c, endpoints
+	return c, createDebugServerEndpoints(s.ready, c.ListenAddress, &s.debugServerEndpoints)
 }
 
-func (svc) Start(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config env.Config) error {
-	return Main(ctx, observationCtx, ready, config.(*Config))
+func (s *svc) Start(ctx context.Context, observationCtx *observation.Context, signalReadyToParent service.ReadyFunc, config env.Config) error {
+	// This service's debugserver endpoints should start responding when this service is ready (and
+	// not ewait for *all* services to be ready). Therefore, we need to track whether we are ready
+	// separately.
+	ready := service.ReadyFunc(func() {
+		close(s.ready)
+		signalReadyToParent()
+	})
+
+	return Main(ctx, observationCtx, ready, &s.debugServerEndpoints, config.(*Config))
 }
 
-var Service service.Service = svc{}
+var Service service.Service = &svc{}

cmd/gitserver/shared/shared.go

@@ -5,6 +5,7 @@ import (
 	"container/list"
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"os/exec"
@@ -50,7 +51,11 @@ import (
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
-func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, config *Config) error {
+type LazyDebugserverEndpoint struct {
+	lockerStatusEndpoint http.HandlerFunc
+}
+
+func Main(ctx context.Context, observationCtx *observation.Context, ready service.ReadyFunc, debugserverEndpoints *LazyDebugserverEndpoint, config *Config) error {
 	logger := observationCtx.Logger
 
 	// Load and validate configuration.
@@ -194,6 +199,12 @@ func Main(ctx context.Context, observationCtx *observation.Context, ready servic
 	}
 	rec.RegistrationDone()
 
+	debugserverEndpoints.lockerStatusEndpoint = func(w http.ResponseWriter, r *http.Request) {
+		if err := json.NewEncoder(w).Encode(locker.AllStatuses()); err != nil {
+			logger.Error("failed to encode locker statuses", log.Error(err))
+		}
+	}
+
 	logger.Info("git-server: listening", log.String("addr", config.ListenAddress))
 
 	// We're ready!