gitserver: Add RepoCloneProgress endpoint (#10041)

This is similar to /repos, but only returns clone progress and is much
faster as it only needs to check the disk once.

cmd/frontend/graphqlbackend/repositories.go

@@ -151,7 +151,7 @@ func (r *repositoryConnectionResolver) compute(ctx context.Context) ([]*types.Re
 				for i, repo := range repos {
 					repoNames[i] = repo.Name
 				}
-				response, err := gitserver.DefaultClient.RepoInfo(ctx, repoNames...)
+				response, err := gitserver.DefaultClient.RepoCloneProgress(ctx, repoNames...)
 				if err != nil {
 					r.err = err
 					return

cmd/gitserver/server/repo_info.go

@@ -51,6 +51,19 @@ func (s *Server) repoInfo(ctx context.Context, repo api.RepoName) (*protocol.Rep
 	return &resp, nil
 }
 
+func (s *Server) repoCloneProgress(repo api.RepoName) (*protocol.RepoCloneProgress, error) {
+	dir := s.dir(repo)
+	resp := protocol.RepoCloneProgress{
+		Cloned: repoCloned(dir),
+	}
+	resp.CloneProgress, resp.CloneInProgress = s.locker.Status(dir)
+	if isAlwaysCloningTest(repo) {
+		resp.CloneInProgress = true
+		resp.CloneProgress = "This will never finish cloning"
+	}
+	return &resp, nil
+}
+
 func (s *Server) handleRepoInfo(w http.ResponseWriter, r *http.Request) {
 	var req protocol.RepoInfoRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
@@ -76,6 +89,31 @@ func (s *Server) handleRepoInfo(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *Server) handleRepoCloneProgress(w http.ResponseWriter, r *http.Request) {
+	var req protocol.RepoCloneProgressRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	resp := protocol.RepoCloneProgressResponse{
+		Results: make(map[api.RepoName]*protocol.RepoCloneProgress, len(req.Repos)),
+	}
+	for _, repoName := range req.Repos {
+		result, err := s.repoCloneProgress(repoName)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		resp.Results[repoName] = result
+	}
+
+	if err := json.NewEncoder(w).Encode(resp); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
 func (s *Server) handleRepoDelete(w http.ResponseWriter, r *http.Request) {
 	var req protocol.RepoDeleteRequest
 	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {

cmd/gitserver/server/server.go

@@ -230,6 +230,7 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("/is-repo-cloneable", s.handleIsRepoCloneable)
 	mux.HandleFunc("/is-repo-cloned", s.handleIsRepoCloned)
 	mux.HandleFunc("/repos", s.handleRepoInfo)
+	mux.HandleFunc("/repo-clone-progress", s.handleRepoCloneProgress)
 	mux.HandleFunc("/delete", s.handleRepoDelete)
 	mux.HandleFunc("/repo-update", s.handleRepoUpdate)
 	mux.HandleFunc("/getGitolitePhabricatorMetadata", s.handleGetGitolitePhabricatorMetadata)

internal/gitserver/client.go

@@ -659,6 +659,76 @@ func (c *Client) IsRepoCloned(ctx context.Context, repo api.RepoName) (bool, err
 	return cloned, nil
 }
 
+func (c *Client) RepoCloneProgress(ctx context.Context, repos ...api.RepoName) (*protocol.RepoCloneProgressResponse, error) {
+	numPossibleShards := len(c.Addrs(ctx))
+	shards := make(map[string]*protocol.RepoCloneProgressRequest, (len(repos)/numPossibleShards)*2) // 2x because it may not be a perfect division
+
+	for _, r := range repos {
+		addr := c.AddrForRepo(ctx, r)
+		shard := shards[addr]
+
+		if shard == nil {
+			shard = new(protocol.RepoCloneProgressRequest)
+			shards[addr] = shard
+		}
+
+		shard.Repos = append(shard.Repos, r)
+	}
+
+	type op struct {
+		req *protocol.RepoCloneProgressRequest
+		res *protocol.RepoCloneProgressResponse
+		err error
+	}
+
+	ch := make(chan op, len(shards))
+	for _, req := range shards {
+		go func(o op) {
+			var resp *http.Response
+			resp, o.err = c.httpPost(ctx, o.req.Repos[0], "repo-clone-progress", o.req)
+			if o.err != nil {
+				ch <- o
+				return
+			}
+
+			defer resp.Body.Close()
+			if resp.StatusCode != http.StatusOK {
+				o.err = &url.Error{
+					URL: resp.Request.URL.String(),
+					Op:  "RepoCloneProgress",
+					Err: errors.Errorf("RepoCloneProgress: http status %d", resp.StatusCode),
+				}
+				ch <- o
+				return // we never get an error status code AND result
+			}
+
+			o.res = new(protocol.RepoCloneProgressResponse)
+			o.err = json.NewDecoder(resp.Body).Decode(o.res)
+			ch <- o
+		}(op{req: req})
+	}
+
+	err := new(multierror.Error)
+	res := protocol.RepoCloneProgressResponse{
+		Results: make(map[api.RepoName]*protocol.RepoCloneProgress),
+	}
+
+	for i := 0; i < cap(ch); i++ {
+		o := <-ch
+
+		if o.err != nil {
+			err = multierror.Append(err, o.err)
+			continue
+		}
+
+		for repo, info := range o.res.Results {
+			res.Results[repo] = info
+		}
+	}
+
+	return &res, err.ErrorOrNil()
+}
+
 // RepoInfo retrieves information about one or more repositories on gitserver.
 //
 // The repository not existing is not an error; in that case, RepoInfoResponse.Results[i].Cloned

internal/gitserver/protocol/gitserver.go

@@ -96,18 +96,18 @@ type IsRepoClonedRequest struct {
 	Repo api.RepoName
 }
 
-// RepoInfoRequest is a request for information about multiple repositories on gitserver.
-type RepoInfoRequest struct {
-	// Repos are the repositories to get information about.
-	Repos []api.RepoName
-}
-
 // RepoDeleteRequest is a request to delete a repository clone on gitserver
 type RepoDeleteRequest struct {
 	// Repo is the repository to delete.
 	Repo api.RepoName
 }
 
+// RepoInfoRequest is a request for information about multiple repositories on gitserver.
+type RepoInfoRequest struct {
+	// Repos are the repositories to get information about.
+	Repos []api.RepoName
+}
+
 // RepoInfo is the information requests about a single repository
 // via a RepoInfoRequest.
 type RepoInfo struct {
@@ -131,6 +131,25 @@ type RepoInfoResponse struct {
 	Results map[api.RepoName]*RepoInfo
 }
 
+// RepoCloneProgressRequest is a request for information about the clone progress of multiple
+// repositories on gitserver.
+type RepoCloneProgressRequest struct {
+	Repos []api.RepoName
+}
+
+// RepoCloneProgress is information about the clone progress of a repo
+type RepoCloneProgress struct {
+	CloneInProgress bool   // whether the repository is currently being cloned
+	CloneProgress   string // a progress message from the running clone command.
+	Cloned          bool   // whether the repository has been cloned successfully
+}
+
+// RepoCloneProgressResponse is the response to a repository clone progress request
+// for multiple repositories at the same time.
+type RepoCloneProgressResponse struct {
+	Results map[api.RepoName]*RepoCloneProgress
+}
+
 // CreateCommitFromPatchRequest is the request information needed for creating
 // the simulated staging area git object for a repo.
 type CreateCommitFromPatchRequest struct {