mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 15:12:02 +00:00
remove replacer service (#12812)
This commit is contained in:
Rijnard van Tonder
GitHub
parent
79490fdc54
commit
dd1d5dd5f3
3
.github/CODEOWNERS
vendored
3
.github/CODEOWNERS
vendored
@ -147,13 +147,12 @@
|
||||
/internal/campaigns @sourcegraph/campaigns
|
||||
/web/**/campaigns/** @sourcegraph/campaigns
|
||||
|
||||
# Search and code mod
|
||||
# Search
|
||||
*/search/**/* @sourcegraph/search
|
||||
/cmd/frontend/internal/pkg/search @sourcegraph/search
|
||||
/cmd/frontend/graphqlbackend/*search* @sourcegraph/search
|
||||
/cmd/frontend/graphqlbackend/*zoekt* @sourcegraph/search
|
||||
/cmd/query-runner/ @sourcegraph/search
|
||||
/cmd/replacer/ @sourcegraph/search @rvantonder
|
||||
/cmd/searcher/ @sourcegraph/search
|
||||
/cmd/symbols/ @sourcegraph/search
|
||||
/internal/search/ @sourcegraph/search
|
||||
|
||||
@ -1,330 +0,0 @@
|
||||
package graphqlbackend
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/inconshreveable/log15"
|
||||
"github.com/opentracing-contrib/go-stdlib/nethttp"
|
||||
otlog "github.com/opentracing/opentracing-go/log"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sourcegraph/go-diff/diff"
|
||||
"github.com/sourcegraph/sourcegraph/internal/env"
|
||||
"github.com/sourcegraph/sourcegraph/internal/errcode"
|
||||
"github.com/sourcegraph/sourcegraph/internal/goroutine"
|
||||
"github.com/sourcegraph/sourcegraph/internal/lazyregexp"
|
||||
"github.com/sourcegraph/sourcegraph/internal/search"
|
||||
"github.com/sourcegraph/sourcegraph/internal/search/query"
|
||||
"github.com/sourcegraph/sourcegraph/internal/trace"
|
||||
"github.com/sourcegraph/sourcegraph/internal/trace/ot"
|
||||
"github.com/sourcegraph/sourcegraph/internal/vcs/git"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
)
|
||||
|
||||
type rawCodemodResult struct {
|
||||
URI string `json:"uri"`
|
||||
Diff string
|
||||
}
|
||||
|
||||
type args struct {
|
||||
matchTemplate string
|
||||
rewriteTemplate string
|
||||
includeFileFilter string
|
||||
excludeFileFilter string
|
||||
}
|
||||
|
||||
// codemodResultResolver is a resolver for the GraphQL type `CodemodResult`
|
||||
type codemodResultResolver struct {
|
||||
commit *GitCommitResolver
|
||||
path string
|
||||
fileURL string
|
||||
diff string
|
||||
matches []*searchResultMatchResolver
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) ToRepository() (*RepositoryResolver, bool) { return nil, false }
|
||||
func (r *codemodResultResolver) ToFileMatch() (*FileMatchResolver, bool) { return nil, false }
|
||||
func (r *codemodResultResolver) ToCommitSearchResult() (*commitSearchResultResolver, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) ToCodemodResult() (*codemodResultResolver, bool) {
|
||||
return r, true
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) searchResultURIs() (string, string) {
|
||||
return string(r.commit.repoResolver.repo.Name), r.path
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) resultCount() int32 {
|
||||
return int32(len(r.matches))
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) Icon() string {
|
||||
return "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='width:24px;height:24px' viewBox='0 0 24 24'%3E%3Cpath fill='%23a2b0cd' d='M11,6C12.38,6 13.63,6.56 14.54,7.46L12,10H18V4L15.95,6.05C14.68,4.78 12.93,4 11,4C7.47,4 4.57,6.61 4.08,10H6.1C6.56,7.72 8.58,6 11,6M16.64,15.14C17.3,14.24 17.76,13.17 17.92,12H15.9C15.44,14.28 13.42,16 11,16C9.62,16 8.37,15.44 7.46,14.54L10,12H4V18L6.05,15.95C7.32,17.22 9.07,18 11,18C12.55,18 14,17.5 15.14,16.64L20,21.5L21.5,20L16.64,15.14Z' /%3E%3C/svg%3E"
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) Label() (*markdownResolver, error) {
|
||||
commitURL, err := r.commit.URL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
text := fmt.Sprintf("[%s](%s) › [%s](%s)", r.commit.repoResolver.Name(), commitURL, r.path, r.fileURL)
|
||||
return &markdownResolver{text: text}, nil
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) URL() string {
|
||||
return r.fileURL
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) Detail() (*markdownResolver, error) {
|
||||
diff, err := diff.ParseFileDiff([]byte(r.diff))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat := diff.Stat()
|
||||
return &markdownResolver{text: stat.String()}, nil
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) Matches() []*searchResultMatchResolver {
|
||||
return r.matches
|
||||
}
|
||||
|
||||
func (r *codemodResultResolver) Commit() *GitCommitResolver { return r.commit }
|
||||
|
||||
func (r *codemodResultResolver) RawDiff() string { return r.diff }
|
||||
|
||||
func validateQuery(q query.QueryInfo) (*args, error) {
|
||||
matchValues := q.Values(query.FieldDefault)
|
||||
var matchTemplates []string
|
||||
for _, v := range matchValues {
|
||||
if v.String != nil && *v.String != "" {
|
||||
matchTemplates = append(matchTemplates, *v.String)
|
||||
}
|
||||
if v.Regexp != nil || v.Bool != nil {
|
||||
return nil, errors.New("this looks like a regex search pattern. Structural search is active because 'replace:' was specified. Please enclose your search string with quotes when using 'replace:'.")
|
||||
}
|
||||
}
|
||||
matchTemplate := strings.Join(matchTemplates, " ")
|
||||
|
||||
replacementValues, _ := q.StringValues(query.FieldReplace)
|
||||
var rewriteTemplate string
|
||||
if len(replacementValues) > 0 {
|
||||
rewriteTemplate = replacementValues[0]
|
||||
}
|
||||
|
||||
includeFileFilter, excludeFileFilter := q.RegexpPatterns(query.FieldFile)
|
||||
var includeFileFilterText string
|
||||
if len(includeFileFilter) > 0 {
|
||||
includeFileFilterText = includeFileFilter[0]
|
||||
// only file names or files with extensions in the following characterset are allowed
|
||||
IsAlphanumericWithPeriod := lazyregexp.New(`^[a-zA-Z0-9_.]+$`).MatchString
|
||||
if !IsAlphanumericWithPeriod(includeFileFilterText) {
|
||||
return nil, errors.New("the 'file:' filter cannot contain regex when using the 'replace:' filter currently. Only alphanumeric characters or '.'")
|
||||
}
|
||||
}
|
||||
|
||||
var excludeFileFilterText string
|
||||
if len(excludeFileFilter) > 0 {
|
||||
excludeFileFilterText = excludeFileFilter[0]
|
||||
IsAlphanumericWithPeriod := lazyregexp.New(`^[a-zA-Z_.]+$`).MatchString
|
||||
if !IsAlphanumericWithPeriod(includeFileFilterText) {
|
||||
return nil, errors.New("the '-file:' filter cannot contain regex when using the 'replace:' filter currently. Only alphanumeric characters or '.'")
|
||||
}
|
||||
}
|
||||
|
||||
return &args{matchTemplate, rewriteTemplate, includeFileFilterText, excludeFileFilterText}, nil
|
||||
}
|
||||
|
||||
// Calls the codemod backend replacer service for a set of repository revisions.
|
||||
func performCodemod(ctx context.Context, args *search.TextParameters) ([]SearchResultResolver, *searchResultsCommon, error) {
|
||||
cmodArgs, err := validateQuery(args.Query)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
title := fmt.Sprintf("pattern: %+v, replace: %+v, includeFileFilter: %+v, excludeFileFilter: %+v, numRepoRevs: %d", cmodArgs.matchTemplate, cmodArgs.rewriteTemplate, cmodArgs.includeFileFilter, cmodArgs.excludeFileFilter, len(args.Repos))
|
||||
tr, ctx := trace.New(ctx, "callCodemod", title)
|
||||
defer func() {
|
||||
tr.SetError(err)
|
||||
tr.Finish()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
unflattened [][]codemodResultResolver
|
||||
common = &searchResultsCommon{}
|
||||
)
|
||||
for _, repoRev := range args.Repos {
|
||||
wg.Add(1)
|
||||
repoRev := repoRev // shadow variable so it doesn't change while goroutine is running
|
||||
goroutine.Go(func() {
|
||||
defer wg.Done()
|
||||
results, searchErr := callCodemodInRepo(ctx, repoRev, cmodArgs)
|
||||
if ctx.Err() == context.Canceled {
|
||||
// Our request has been canceled (either because another one of args.repos had a
|
||||
// fatal error, or otherwise), so we can just ignore these results.
|
||||
return
|
||||
}
|
||||
repoTimedOut := ctx.Err() == context.DeadlineExceeded
|
||||
if searchErr != nil {
|
||||
tr.LogFields(otlog.String("repo", string(repoRev.Repo.Name)), otlog.String("searchErr", searchErr.Error()), otlog.Bool("timeout", errcode.IsTimeout(searchErr)), otlog.Bool("temporary", errcode.IsTemporary(searchErr)))
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if fatalErr := handleRepoSearchResult(common, repoRev, false, repoTimedOut, searchErr); fatalErr != nil {
|
||||
err = errors.Wrapf(searchErr, "failed to call codemod %s", repoRev)
|
||||
cancel()
|
||||
}
|
||||
if len(results) > 0 {
|
||||
unflattened = append(unflattened, results)
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var results []SearchResultResolver
|
||||
for _, ur := range unflattened {
|
||||
for _, resolver := range ur {
|
||||
v := resolver
|
||||
results = append(results, &v)
|
||||
}
|
||||
}
|
||||
|
||||
return results, common, nil
|
||||
}
|
||||
|
||||
var ReplacerURL = env.Get("REPLACER_URL", "http://replacer:3185", "replacer server URL")
|
||||
|
||||
func toMatchResolver(fileURL string, raw *rawCodemodResult) ([]*searchResultMatchResolver, error) {
|
||||
if !strings.Contains(raw.Diff, "@@") {
|
||||
return nil, errors.Errorf("Invalid diff does not contain expected @@: %v", raw.Diff)
|
||||
}
|
||||
strippedDiff := raw.Diff[strings.Index(raw.Diff, "@@"):]
|
||||
|
||||
return []*searchResultMatchResolver{
|
||||
{
|
||||
url: fileURL,
|
||||
body: "```diff\n" + strippedDiff + "\n```",
|
||||
highlights: nil,
|
||||
},
|
||||
},
|
||||
nil
|
||||
}
|
||||
|
||||
func callCodemodInRepo(ctx context.Context, repoRevs *search.RepositoryRevisions, args *args) (results []codemodResultResolver, err error) {
|
||||
tr, ctx := trace.New(ctx, "callCodemodInRepo", fmt.Sprintf("repoRevs: %v, pattern %+v, replace: %+v", repoRevs, args.matchTemplate, args.rewriteTemplate))
|
||||
defer func() {
|
||||
tr.LazyPrintf("%d results", len(results))
|
||||
tr.SetError(err)
|
||||
tr.Finish()
|
||||
}()
|
||||
|
||||
// For performance, assume repo is cloned in gitserver and do not trigger a repo-updater lookup (this call fails if repo is not on gitserver).
|
||||
commit, err := git.ResolveRevision(ctx, repoRevs.GitserverRepo(), nil, repoRevs.Revs[0].RevSpec, git.ResolveRevisionOptions{NoEnsureRevision: true})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "codemod repo lookup failed: it's possible that the repo is not cloned in gitserver. Try force a repo update another way.")
|
||||
}
|
||||
|
||||
u, err := url.Parse(ReplacerURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("repo", string(repoRevs.Repo.Name))
|
||||
q.Set("commit", string(commit))
|
||||
q.Set("matchtemplate", args.matchTemplate)
|
||||
q.Set("rewritetemplate", args.rewriteTemplate)
|
||||
q.Set("fileextension", args.includeFileFilter)
|
||||
q.Set("directoryexclude", args.excludeFileFilter)
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
req, err := http.NewRequest("GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
req, ht := nethttp.TraceRequest(ot.GetTracer(ctx), req,
|
||||
nethttp.OperationName("Codemod client"),
|
||||
nethttp.ClientTrace(false))
|
||||
defer ht.Finish()
|
||||
|
||||
// TODO(RVT): Use a separate HTTP client here dedicated to codemod,
|
||||
// not doing so means codemod and searcher share the same HTTP limits
|
||||
// etc. which is fine for now but not if codemod goes in front of users.
|
||||
// Once separated please fix cmd/frontend/graphqlbackend/textsearch:50 in #6586
|
||||
resp, err := ctxhttp.Do(ctx, searchHTTPClient, req)
|
||||
if err != nil {
|
||||
// If we failed due to cancellation or timeout (with no partial results in the response body), return just that.
|
||||
if ctx.Err() != nil {
|
||||
err = ctx.Err()
|
||||
}
|
||||
return nil, errors.Wrap(err, "codemod request failed")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, errors.WithStack(&searcherError{StatusCode: resp.StatusCode, Message: string(body)})
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
// TODO(RVT): Remove buffer inefficiency introduced here. Results are
|
||||
// line encoded JSON. Malformed JSON can happen, for extremely long
|
||||
// lines. Unless we know where this and subsequent malformed lines end,
|
||||
// we can't continue decoding the next one. As an inefficient but robust
|
||||
// solution, we allow to buffer a very high maximum length line and can
|
||||
// skip over very long malformed lines. It is set to 10 * 64K.
|
||||
scanner.Buffer(make([]byte, 100), 10*bufio.MaxScanTokenSize)
|
||||
|
||||
repoResolver := &RepositoryResolver{repo: repoRevs.Repo}
|
||||
|
||||
for scanner.Scan() {
|
||||
var raw *rawCodemodResult
|
||||
b := scanner.Bytes()
|
||||
if err := scanner.Err(); err != nil {
|
||||
log15.Info(fmt.Sprintf("Skipping codemod scanner error (line too long?): %s", err.Error()))
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(b, &raw); err != nil {
|
||||
// skip on other decode errors (including e.g., empty
|
||||
// responses if dependencies are not installed)
|
||||
continue
|
||||
}
|
||||
fileURL := fileMatchURI(repoRevs.Repo.Name, repoRevs.Revs[0].RevSpec, raw.URI)
|
||||
matches, err := toMatchResolver(fileURL, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, codemodResultResolver{
|
||||
commit: &GitCommitResolver{
|
||||
repoResolver: repoResolver,
|
||||
inputRev: &repoRevs.Revs[0].RevSpec,
|
||||
oid: GitObjectID(commit),
|
||||
},
|
||||
path: raw.URI,
|
||||
fileURL: fileURL,
|
||||
diff: raw.Diff,
|
||||
matches: matches,
|
||||
})
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
package graphqlbackend
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/internal/search/query"
|
||||
)
|
||||
|
||||
func TestCodemod_validateArgsNoRegex(t *testing.T) {
|
||||
q, _ := query.ParseAndCheck("re.*gex")
|
||||
_, err := validateQuery(q)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected query %v to fail", q)
|
||||
}
|
||||
if !strings.HasPrefix(err.Error(), "this looks like a regex search pattern.") {
|
||||
t.Fatalf("%v expected complaint about regex pattern. Got %s", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodemod_validateArgsOk(t *testing.T) {
|
||||
q, _ := query.ParseAndCheck(`"not regex"`)
|
||||
_, err := validateQuery(q)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected query %v to to be OK", q)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCodemod_resolver(t *testing.T) {
|
||||
raw := &rawCodemodResult{
|
||||
URI: "",
|
||||
Diff: "Not a valid diff",
|
||||
}
|
||||
_, err := toMatchResolver("", raw)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected invalid diff for %v", raw.Diff)
|
||||
}
|
||||
if !strings.HasPrefix(err.Error(), "Invalid diff") {
|
||||
t.Fatalf("Expected error %q", err)
|
||||
}
|
||||
}
|
||||
@ -71,7 +71,6 @@ func TestResolverTo(t *testing.T) {
|
||||
&NamespaceResolver{},
|
||||
&NodeResolver{},
|
||||
&RepositoryResolver{},
|
||||
&codemodResultResolver{},
|
||||
&commitSearchResultResolver{},
|
||||
&gitRevSpec{},
|
||||
&searchSuggestionResolver{},
|
||||
|
||||
@ -279,9 +279,6 @@ func (r *RepositoryResolver) ToFileMatch() (*FileMatchResolver, bool) { return
|
||||
func (r *RepositoryResolver) ToCommitSearchResult() (*commitSearchResultResolver, bool) {
|
||||
return nil, false
|
||||
}
|
||||
func (r *RepositoryResolver) ToCodemodResult() (*codemodResultResolver, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (r *RepositoryResolver) searchResultURIs() (string, string) {
|
||||
return string(r.repo.Name), ""
|
||||
|
||||
20
cmd/frontend/graphqlbackend/schema.go
generated
20
cmd/frontend/graphqlbackend/schema.go
generated
@ -1511,7 +1511,7 @@ type SearchFilterSuggestions {
|
||||
}
|
||||
|
||||
# A search result.
|
||||
union SearchResult = FileMatch | CommitSearchResult | Repository | CodemodResult
|
||||
union SearchResult = FileMatch | CommitSearchResult | Repository
|
||||
|
||||
# An object representing a markdown string.
|
||||
type Markdown {
|
||||
@ -1741,24 +1741,6 @@ type CommitSearchResult implements GenericSearchResultInterface {
|
||||
diffPreview: HighlightedString
|
||||
}
|
||||
|
||||
# The result of a code modification query.
|
||||
type CodemodResult implements GenericSearchResultInterface {
|
||||
# URL to an icon that is displayed with every search result.
|
||||
icon: String!
|
||||
# A markdown string that is rendered prominently.
|
||||
label: Markdown!
|
||||
# The URL of the result.
|
||||
url: String!
|
||||
# A markdown string that is rendered less prominently.
|
||||
detail: Markdown!
|
||||
# A list of matches in this search result.
|
||||
matches: [SearchResultMatch!]!
|
||||
# The commit whose contents the codemod was run against.
|
||||
commit: GitCommit!
|
||||
# The raw diff of the modification.
|
||||
rawDiff: String!
|
||||
}
|
||||
|
||||
# A search result that is a diff between two diffable Git objects.
|
||||
type DiffSearchResult {
|
||||
# The diff that matched the search query.
|
||||
|
||||
@ -1518,7 +1518,7 @@ type SearchFilterSuggestions {
|
||||
}
|
||||
|
||||
# A search result.
|
||||
union SearchResult = FileMatch | CommitSearchResult | Repository | CodemodResult
|
||||
union SearchResult = FileMatch | CommitSearchResult | Repository
|
||||
|
||||
# An object representing a markdown string.
|
||||
type Markdown {
|
||||
@ -1748,24 +1748,6 @@ type CommitSearchResult implements GenericSearchResultInterface {
|
||||
diffPreview: HighlightedString
|
||||
}
|
||||
|
||||
# The result of a code modification query.
|
||||
type CodemodResult implements GenericSearchResultInterface {
|
||||
# URL to an icon that is displayed with every search result.
|
||||
icon: String!
|
||||
# A markdown string that is rendered prominently.
|
||||
label: Markdown!
|
||||
# The URL of the result.
|
||||
url: String!
|
||||
# A markdown string that is rendered less prominently.
|
||||
detail: Markdown!
|
||||
# A list of matches in this search result.
|
||||
matches: [SearchResultMatch!]!
|
||||
# The commit whose contents the codemod was run against.
|
||||
commit: GitCommit!
|
||||
# The raw diff of the modification.
|
||||
rawDiff: String!
|
||||
}
|
||||
|
||||
# A search result that is a diff between two diffable Git objects.
|
||||
type DiffSearchResult {
|
||||
# The diff that matched the search query.
|
||||
|
||||
@ -69,10 +69,6 @@ func (r *commitSearchResultResolver) ToCommitSearchResult() (*commitSearchResult
|
||||
return r, true
|
||||
}
|
||||
|
||||
func (r *commitSearchResultResolver) ToCodemodResult() (*codemodResultResolver, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (r *commitSearchResultResolver) searchResultURIs() (string, string) {
|
||||
// Diffs aren't going to be returned with other types of results
|
||||
// and are already ordered in the desired order, so we'll just leave them in place.
|
||||
|
||||
@ -504,8 +504,6 @@ loop:
|
||||
}
|
||||
addPoint(t)
|
||||
})
|
||||
case *codemodResultResolver:
|
||||
continue
|
||||
default:
|
||||
panic("SearchResults.Sparkline unexpected union type state")
|
||||
}
|
||||
@ -1388,8 +1386,6 @@ func (r *searchResolver) determineResultTypes(args search.TextParameters, forceO
|
||||
// Determine which types of results to return.
|
||||
if forceOnlyResultType != "" {
|
||||
resultTypes = []string{forceOnlyResultType}
|
||||
} else if len(r.query.Values(query.FieldReplace)) > 0 {
|
||||
resultTypes = []string{"codemod"}
|
||||
} else {
|
||||
resultTypes, _ = r.query.StringValues(query.FieldType)
|
||||
if len(resultTypes) == 0 {
|
||||
@ -1767,30 +1763,6 @@ func (r *searchResolver) doResults(ctx context.Context, forceOnlyResultType stri
|
||||
commonMu.Unlock()
|
||||
}
|
||||
})
|
||||
case "codemod":
|
||||
wg := waitGroup(true)
|
||||
wg.Add(1)
|
||||
goroutine.Go(func() {
|
||||
defer wg.Done()
|
||||
|
||||
codemodResults, codemodCommon, err := performCodemod(ctx, &args)
|
||||
// Timeouts are reported through searchResultsCommon so don't report an error for them
|
||||
if err != nil && !isContextError(ctx, err) {
|
||||
multiErrMu.Lock()
|
||||
multiErr = multierror.Append(multiErr, errors.Wrap(err, "codemod search failed"))
|
||||
multiErrMu.Unlock()
|
||||
}
|
||||
if codemodResults != nil {
|
||||
resultsMu.Lock()
|
||||
results = append(results, codemodResults...)
|
||||
resultsMu.Unlock()
|
||||
}
|
||||
if codemodCommon != nil {
|
||||
commonMu.Lock()
|
||||
common.update(*codemodCommon)
|
||||
commonMu.Unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -1866,7 +1838,6 @@ func isContextError(ctx context.Context, err error) bool {
|
||||
// - *RepositoryResolver // repo name match
|
||||
// - *fileMatchResolver // text match
|
||||
// - *commitSearchResultResolver // diff or commit match
|
||||
// - *codemodResultResolver // code modification
|
||||
//
|
||||
// Note: Any new result types added here also need to be handled properly in search_results.go:301 (sparklines)
|
||||
type SearchResultResolver interface {
|
||||
@ -1875,7 +1846,6 @@ type SearchResultResolver interface {
|
||||
ToRepository() (*RepositoryResolver, bool)
|
||||
ToFileMatch() (*FileMatchResolver, bool)
|
||||
ToCommitSearchResult() (*commitSearchResultResolver, bool)
|
||||
ToCodemodResult() (*codemodResultResolver, bool)
|
||||
|
||||
resultCount() int32
|
||||
}
|
||||
|
||||
@ -49,10 +49,6 @@ var (
|
||||
// Default is 2, but we can send many concurrent requests
|
||||
MaxIdleConnsPerHost: 500,
|
||||
}, func(u *url.URL) string {
|
||||
// TODO(uwedeportivo): remove once codemod has its own client
|
||||
if strings.Contains(u.String(), "replacer") {
|
||||
return "replace"
|
||||
}
|
||||
return "search"
|
||||
}),
|
||||
},
|
||||
@ -139,10 +135,6 @@ func (fm *FileMatchResolver) ToCommitSearchResult() (*commitSearchResultResolver
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (r *FileMatchResolver) ToCodemodResult() (*codemodResultResolver, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (fm *FileMatchResolver) searchResultURIs() (string, string) {
|
||||
return fm.Repo.Name(), fm.JPath
|
||||
}
|
||||
|
||||
@ -1,24 +0,0 @@
|
||||
FROM sourcegraph/alpine:3.10@sha256:4d05cd5669726fc38823e92320659a6d1ef7879e62268adec5df658a0bacf65c
|
||||
|
||||
# hadolint ignore=DL3018
|
||||
RUN apk --no-cache add pcre
|
||||
|
||||
# The comby/comby image is a small binary-only distribution. See the bin and src directories
|
||||
# here: https://github.com/comby-tools/comby/tree/master/dockerfiles/alpine
|
||||
# hadolint ignore=DL3022
|
||||
COPY --from=comby/comby:0.15.0@sha256:b3b57ed810be1489ccd1ef8fa42210d87d2d396e416c62cee345f615a0dcb09b /usr/local/bin/comby /usr/local/bin/comby
|
||||
|
||||
ARG COMMIT_SHA="unknown"
|
||||
ARG DATE="unknown"
|
||||
ARG VERSION="unknown"
|
||||
|
||||
LABEL org.opencontainers.image.revision=${COMMIT_SHA}
|
||||
LABEL org.opencontainers.image.created=${DATE}
|
||||
LABEL org.opencontainers.image.version=${VERSION}
|
||||
LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}
|
||||
|
||||
ENV CACHE_DIR=/mnt/cache/replacer
|
||||
RUN mkdir -p ${CACHE_DIR} && chown -R sourcegraph:sourcegraph ${CACHE_DIR}
|
||||
USER sourcegraph
|
||||
ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/replacer"]
|
||||
COPY replacer /usr/local/bin/
|
||||
@ -1,26 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# We want to build multiple go binaries, so we use a custom build step on CI.
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"/../..
|
||||
set -ex
|
||||
|
||||
OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
|
||||
cleanup() {
|
||||
rm -rf "$OUTPUT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Environment for building linux binaries
|
||||
export GO111MODULE=on
|
||||
export GOARCH=amd64
|
||||
export GOOS=linux
|
||||
export CGO_ENABLED=0
|
||||
|
||||
pkg="github.com/sourcegraph/sourcegraph/cmd/replacer"
|
||||
go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg"
|
||||
|
||||
docker build -f cmd/replacer/Dockerfile -t "$IMAGE" "$OUTPUT" \
|
||||
--progress=plain \
|
||||
--build-arg COMMIT_SHA \
|
||||
--build-arg DATE \
|
||||
--build-arg VERSION
|
||||
@ -1,104 +0,0 @@
|
||||
// Command replacer is an interface to replace and rewrite code. It passes a zipped repo
|
||||
// to external tools and streams back JSON lines results.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/inconshreveable/log15"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/cmd/replacer/replace"
|
||||
"github.com/sourcegraph/sourcegraph/internal/api"
|
||||
"github.com/sourcegraph/sourcegraph/internal/debugserver"
|
||||
"github.com/sourcegraph/sourcegraph/internal/env"
|
||||
"github.com/sourcegraph/sourcegraph/internal/gitserver"
|
||||
"github.com/sourcegraph/sourcegraph/internal/store"
|
||||
"github.com/sourcegraph/sourcegraph/internal/trace/ot"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/internal/tracer"
|
||||
)
|
||||
|
||||
var cacheDir = env.Get("CACHE_DIR", "/tmp", "directory to store cached archives.")
|
||||
var cacheSizeMB = env.Get("REPLACER_CACHE_SIZE_MB", "100000", "maximum size of the on disk cache in megabytes")
|
||||
|
||||
const port = "3185"
|
||||
|
||||
func main() {
|
||||
env.Lock()
|
||||
env.HandleHelpFlag()
|
||||
log.SetFlags(0)
|
||||
tracer.Init()
|
||||
|
||||
go debugserver.Start()
|
||||
|
||||
var cacheSizeBytes int64
|
||||
if i, err := strconv.ParseInt(cacheSizeMB, 10, 64); err != nil {
|
||||
log.Fatalf("invalid int %q for REPLACER_CACHE_SIZE_MB: %s", cacheSizeMB, err)
|
||||
} else {
|
||||
cacheSizeBytes = i * 1000 * 1000
|
||||
}
|
||||
|
||||
store := store.Store{
|
||||
FetchTar: func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) {
|
||||
return gitserver.DefaultClient.Archive(ctx, repo, gitserver.ArchiveOptions{Treeish: string(commit), Format: "tar"})
|
||||
},
|
||||
Path: filepath.Join(cacheDir, "replacer-archives"),
|
||||
MaxCacheSizeBytes: cacheSizeBytes,
|
||||
}
|
||||
store.SetMaxConcurrentFetchTar(10)
|
||||
store.Start()
|
||||
service := &replace.Service{
|
||||
Store: &store,
|
||||
Log: log15.Root(),
|
||||
}
|
||||
handler := ot.Middleware(service)
|
||||
|
||||
host := ""
|
||||
if env.InsecureDev {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
addr := net.JoinHostPort(host, port)
|
||||
server := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// For cluster liveness and readiness probes
|
||||
if r.URL.Path == "/healthz" {
|
||||
w.WriteHeader(200)
|
||||
_, err := w.Write([]byte("ok"))
|
||||
if err != nil {
|
||||
log15.Info("Error checking /healthz: " + err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
handler.ServeHTTP(w, r)
|
||||
}),
|
||||
}
|
||||
go shutdownOnSIGINT(server)
|
||||
|
||||
log15.Info("replacer: listening", "addr", server.Addr)
|
||||
if err := server.ListenAndServe(); err != http.ErrServerClosed {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func shutdownOnSIGINT(s *http.Server) {
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, os.Interrupt)
|
||||
<-c
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
err := s.Shutdown(ctx)
|
||||
if err != nil {
|
||||
log.Fatal("graceful server shutdown failed, will exit:", err)
|
||||
}
|
||||
}
|
||||
@ -1,55 +0,0 @@
|
||||
// Package protocol contains structures used by the replacer API.
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"github.com/sourcegraph/sourcegraph/internal/api"
|
||||
"github.com/sourcegraph/sourcegraph/internal/gitserver"
|
||||
)
|
||||
|
||||
// Request represents a request to replacer
|
||||
type Request struct {
|
||||
// Repo is the name of the repository to search. eg "github.com/gorilla/mux"
|
||||
Repo api.RepoName
|
||||
|
||||
// URL specifies the repository's Git remote URL (for gitserver). It is optional. See
|
||||
// (gitserver.ExecRequest).URL for documentation on what it is used for.
|
||||
URL string
|
||||
|
||||
// Commit is which commit to search. It is required to be resolved,
|
||||
// not a ref like HEAD or master. eg
|
||||
// "599cba5e7b6137d46ddf58fb1765f5d928e69604"
|
||||
Commit api.CommitID
|
||||
|
||||
// The amount of time to wait for a repo archive to fetch.
|
||||
// It is parsed with time.ParseDuration.
|
||||
//
|
||||
// This timeout should be low when searching across many repos
|
||||
// so that unfetched repos don't delay the search, and because we are likely
|
||||
// to get results from the repos that have already been fetched.
|
||||
//
|
||||
// This timeout should be high when searching across a single repo
|
||||
// because returning results slowly is better than returning no results at all.
|
||||
//
|
||||
// This only times out how long we wait for the fetch request;
|
||||
// the fetch will still happen in the background so future requests don't have to wait.
|
||||
FetchTimeout string
|
||||
|
||||
RewriteSpecification
|
||||
}
|
||||
|
||||
type RewriteSpecification struct {
|
||||
// A template pattern that expresses what to match.
|
||||
MatchTemplate string
|
||||
|
||||
// A template pattern that expresses how matches should be rewritten.
|
||||
RewriteTemplate string
|
||||
|
||||
// A file extension suffix filtering which files to process (e.g., ".go")
|
||||
FileExtension string
|
||||
|
||||
// A directory prefix to exclude (e.g., vendor)
|
||||
DirectoryExclude string
|
||||
}
|
||||
|
||||
// GitserverRepo returns the repository information necessary to perform gitserver requests.
|
||||
func (r Request) GitserverRepo() gitserver.Repo { return gitserver.Repo{Name: r.Repo, URL: r.URL} }
|
||||
@ -1,309 +0,0 @@
|
||||
// Package replace is a service exposing an API to replace file contents in a repo.
|
||||
// It streams back results with JSON lines.
|
||||
//
|
||||
// Architecture Notes:
|
||||
// - The following are the same as cmd/searcher/search.go:
|
||||
// * Archive is fetched from gitserver
|
||||
// * Simple HTTP API exposed
|
||||
// * Currently no concept of authorization
|
||||
// * On disk cache of fetched archives to reduce load on gitserver
|
||||
//
|
||||
// - Here is where replacer.go differs
|
||||
// * Pass the zip file path to external replacer tool(s) after validating
|
||||
// * Read tool stdout and write it out on the HTTP connection
|
||||
// * Input from stdout is expected to use JSON lines format, but the format isn't checked here: line-buffering is done on the frontend
|
||||
|
||||
package replace
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
nettrace "golang.org/x/net/trace"
|
||||
|
||||
"github.com/inconshreveable/log15"
|
||||
"github.com/opentracing/opentracing-go/ext"
|
||||
otlog "github.com/opentracing/opentracing-go/log"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/sourcegraph/sourcegraph/cmd/replacer/protocol"
|
||||
"github.com/sourcegraph/sourcegraph/internal/store"
|
||||
"github.com/sourcegraph/sourcegraph/internal/trace/ot"
|
||||
|
||||
"github.com/gorilla/schema"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
Store *store.Store
|
||||
Log log15.Logger
|
||||
}
|
||||
|
||||
type ExternalTool struct {
|
||||
Name string
|
||||
BinaryPath string
|
||||
}
|
||||
|
||||
// Configure the command line options and return the command to execute using an external tool
|
||||
func (t *ExternalTool) command(ctx context.Context, spec *protocol.RewriteSpecification, zipPath string) (cmd *exec.Cmd, err error) {
|
||||
switch t.Name {
|
||||
case "comby":
|
||||
_, err = exec.LookPath("comby")
|
||||
if err != nil {
|
||||
return nil, errors.New("comby is not installed on the PATH. Try running 'bash <(curl -sL get.comby.dev)'.")
|
||||
}
|
||||
|
||||
var args []string
|
||||
args = append(args, spec.MatchTemplate, spec.RewriteTemplate)
|
||||
|
||||
if spec.FileExtension != "" {
|
||||
args = append(args, spec.FileExtension)
|
||||
}
|
||||
|
||||
args = append(args, "-zip", zipPath, "-json-lines", "-json-only-diff")
|
||||
|
||||
if spec.DirectoryExclude != "" {
|
||||
args = append(args, "-exclude-dir", spec.DirectoryExclude)
|
||||
}
|
||||
|
||||
log15.Info(fmt.Sprintf("running command: comby %q", strings.Join(args[:], " ")))
|
||||
return exec.CommandContext(ctx, t.BinaryPath, args...), nil
|
||||
|
||||
default:
|
||||
return nil, errors.Errorf("Unknown external replace tool %q.", t.Name)
|
||||
}
|
||||
}
|
||||
|
||||
var decoder = schema.NewDecoder()
|
||||
|
||||
func init() {
|
||||
decoder.IgnoreUnknownKeys(true)
|
||||
}
|
||||
|
||||
// ServeHTTP handles HTTP based replace requests
|
||||
func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
running.Inc()
|
||||
defer running.Dec()
|
||||
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
log15.Info("Didn't parse" + err.Error())
|
||||
http.Error(w, "failed to parse form: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var p protocol.Request
|
||||
err = decoder.Decode(&p, r.Form)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to decode form: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err = validateParams(&p); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
deadlineHit, err := s.replace(ctx, &p, w, r)
|
||||
if err != nil {
|
||||
code := http.StatusInternalServerError
|
||||
if isBadRequest(err) || ctx.Err() == context.Canceled {
|
||||
code = http.StatusBadRequest
|
||||
} else if isTemporary(err) {
|
||||
code = http.StatusServiceUnavailable
|
||||
} else {
|
||||
log.Printf("internal error serving %#+v: %s", p, err)
|
||||
}
|
||||
http.Error(w, err.Error(), code)
|
||||
return
|
||||
}
|
||||
|
||||
if deadlineHit {
|
||||
log15.Info("Deadline hit")
|
||||
http.Error(w, "Deadline hit", http.StatusRequestTimeout)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) replace(ctx context.Context, p *protocol.Request, w http.ResponseWriter, r *http.Request) (deadlineHit bool, err error) {
|
||||
tr := nettrace.New("replace", fmt.Sprintf("%s@%s", p.Repo, p.Commit))
|
||||
tr.LazyPrintf("%s", p.RewriteSpecification)
|
||||
|
||||
span, ctx := ot.StartSpanFromContext(ctx, "Replace")
|
||||
ext.Component.Set(span, "service")
|
||||
span.SetTag("repo", p.Repo)
|
||||
span.SetTag("url", p.URL)
|
||||
span.SetTag("commit", p.Commit)
|
||||
span.SetTag("rewriteSpecification", p.RewriteSpecification)
|
||||
defer func(start time.Time) {
|
||||
code := "200"
|
||||
// We often have canceled and timed out requests. We do not want to
|
||||
// record them as errors to avoid noise
|
||||
if ctx.Err() == context.Canceled {
|
||||
code = "canceled"
|
||||
span.SetTag("err", err)
|
||||
} else if ctx.Err() == context.DeadlineExceeded {
|
||||
code = "timedout"
|
||||
span.SetTag("err", err)
|
||||
deadlineHit = true
|
||||
err = nil // error is fully described by deadlineHit=true return value
|
||||
} else if err != nil {
|
||||
tr.LazyPrintf("error: %v", err)
|
||||
tr.SetError()
|
||||
ext.Error.Set(span, true)
|
||||
span.SetTag("err", err.Error())
|
||||
if isBadRequest(err) {
|
||||
code = "400"
|
||||
} else if isTemporary(err) {
|
||||
code = "503"
|
||||
} else {
|
||||
code = "500"
|
||||
}
|
||||
}
|
||||
tr.LazyPrintf("code=%s deadlineHit=%v", code, deadlineHit)
|
||||
tr.Finish()
|
||||
requestTotal.WithLabelValues(code).Inc()
|
||||
span.SetTag("deadlineHit", deadlineHit)
|
||||
span.Finish()
|
||||
|
||||
if s.Log != nil {
|
||||
s.Log.Debug("replace request", "repo", p.Repo, "commit", p.Commit, "rewriteSpecification", p.RewriteSpecification, "code", code, "duration", time.Since(start), "err", err)
|
||||
}
|
||||
}(time.Now())
|
||||
|
||||
if p.FetchTimeout == "" {
|
||||
p.FetchTimeout = "500ms"
|
||||
}
|
||||
fetchTimeout, err := time.ParseDuration(p.FetchTimeout)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
prepareCtx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
getZf := func() (string, *store.ZipFile, error) {
|
||||
path, err := s.Store.PrepareZip(prepareCtx, p.GitserverRepo(), p.Commit)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
zf, err := s.Store.ZipCache.Get(path)
|
||||
return path, zf, err
|
||||
}
|
||||
|
||||
zipPath, zf, err := store.GetZipFileWithRetry(getZf)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "failed to get archive")
|
||||
}
|
||||
defer zf.Close()
|
||||
|
||||
nFiles := uint64(len(zf.Files))
|
||||
bytes := int64(len(zf.Data))
|
||||
tr.LazyPrintf("files=%d bytes=%d", nFiles, bytes)
|
||||
span.LogFields(
|
||||
otlog.Uint64("archive.files", nFiles),
|
||||
otlog.Int64("archive.size", bytes))
|
||||
archiveFiles.Observe(float64(nFiles))
|
||||
archiveSize.Observe(float64(bytes))
|
||||
|
||||
w.Header().Set("Transfer-Encoding", "chunked")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
t := &ExternalTool{
|
||||
Name: "comby",
|
||||
BinaryPath: "comby",
|
||||
}
|
||||
|
||||
cmd, err := t.command(ctx, &p.RewriteSpecification, zipPath)
|
||||
if err != nil {
|
||||
log15.Info("Invalid command: " + err.Error())
|
||||
return false, errors.Wrap(err, "invalid command")
|
||||
}
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
log15.Info("Could not connect to command stdout: " + err.Error())
|
||||
return false, errors.Wrap(err, "failed to connect to command stdout")
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
log15.Info("Error starting command: " + err.Error())
|
||||
return false, errors.Wrap(err, "failed to start command")
|
||||
}
|
||||
|
||||
_, err = io.Copy(w, stdout)
|
||||
if err != nil {
|
||||
log15.Info("Error copying external command output to HTTP writer: " + err.Error())
|
||||
return false, errors.Wrap(err, "failed while copying command output to HTTP")
|
||||
}
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
log15.Info("Error after executing command: " + string(err.(*exec.ExitError).Stderr))
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func validateParams(p *protocol.Request) error {
|
||||
if p.Repo == "" {
|
||||
return errors.New("Repo must be non-empty")
|
||||
}
|
||||
// Surprisingly this is the same sanity check used in the git source.
|
||||
if len(p.Commit) != 40 {
|
||||
return errors.Errorf("Commit must be resolved (Commit=%q)", p.Commit)
|
||||
}
|
||||
|
||||
if p.RewriteSpecification.MatchTemplate == "" {
|
||||
return errors.New("MatchTemplate must be non-empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const megabyte = float64(1000 * 1000)
|
||||
|
||||
var (
|
||||
running = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "replacer_service_running",
|
||||
Help: "Number of running search requests.",
|
||||
})
|
||||
archiveSize = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "replacer_service_archive_size_bytes",
|
||||
Help: "Observes the size when an archive is searched.",
|
||||
Buckets: []float64{1 * megabyte, 10 * megabyte, 100 * megabyte, 500 * megabyte, 1000 * megabyte, 5000 * megabyte},
|
||||
})
|
||||
archiveFiles = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "replacer_service_archive_files",
|
||||
Help: "Observes the number of files when an archive is searched.",
|
||||
Buckets: []float64{100, 1000, 10000, 50000, 100000},
|
||||
})
|
||||
requestTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "replacer_service_request_total",
|
||||
Help: "Number of returned replace requests.",
|
||||
}, []string{"code"})
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.MustRegister(running)
|
||||
prometheus.MustRegister(archiveSize)
|
||||
prometheus.MustRegister(archiveFiles)
|
||||
prometheus.MustRegister(requestTotal)
|
||||
}
|
||||
|
||||
func isBadRequest(err error) bool {
|
||||
e, ok := errors.Cause(err).(interface {
|
||||
BadRequest() bool
|
||||
})
|
||||
return ok && e.BadRequest()
|
||||
}
|
||||
|
||||
func isTemporary(err error) bool {
|
||||
e, ok := errors.Cause(err).(interface {
|
||||
Temporary() bool
|
||||
})
|
||||
return ok && e.Temporary()
|
||||
}
|
||||
@ -1,145 +0,0 @@
|
||||
package replace_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/cmd/replacer/protocol"
|
||||
"github.com/sourcegraph/sourcegraph/cmd/replacer/replace"
|
||||
"github.com/sourcegraph/sourcegraph/internal/testutil"
|
||||
)
|
||||
|
||||
func TestReplace(t *testing.T) {
|
||||
// If we are not on CI skip the test if comby is not installed.
|
||||
if _, err := exec.LookPath("comby"); os.Getenv("CI") == "" && err != nil {
|
||||
t.Skip("comby is not installed on the PATH. Try running 'bash <(curl -sL get.comby.dev)'.")
|
||||
}
|
||||
|
||||
files := map[string]string{
|
||||
|
||||
"README.md": `# Hello World
|
||||
|
||||
Hello world example in go`,
|
||||
"main.go": `package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func main() {
|
||||
fmt.Println("Hello foo")
|
||||
}
|
||||
`,
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
arg protocol.RewriteSpecification
|
||||
want string
|
||||
}{
|
||||
{protocol.RewriteSpecification{
|
||||
MatchTemplate: "func",
|
||||
RewriteTemplate: "derp",
|
||||
FileExtension: ".go",
|
||||
}, `
|
||||
{"uri":"main.go","diff":"--- main.go\n+++ main.go\n@@ -2,6 +2,6 @@\n \n import \"fmt\"\n \n-func main() {\n+derp main() {\n \tfmt.Println(\"Hello foo\")\n }"}
|
||||
`},
|
||||
}
|
||||
|
||||
store, cleanup, err := testutil.NewStore(files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cleanup()
|
||||
ts := httptest.NewServer(&replace.Service{Store: store})
|
||||
defer ts.Close()
|
||||
|
||||
for _, test := range cases {
|
||||
req := protocol.Request{
|
||||
Repo: "foo",
|
||||
URL: "u",
|
||||
Commit: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
||||
RewriteSpecification: test.arg,
|
||||
FetchTimeout: "5000ms",
|
||||
}
|
||||
got, err := doReplace(ts.URL, &req)
|
||||
if err != nil {
|
||||
t.Errorf("%v failed: %s", test.arg, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// We have an extra newline to make expected readable
|
||||
if len(test.want) > 0 {
|
||||
test.want = test.want[1:]
|
||||
}
|
||||
|
||||
if got != test.want {
|
||||
d, err := testutil.Diff(test.want, got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Errorf("%v unexpected response:\n%s", test.arg, d)
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplace_badrequest(t *testing.T) {
|
||||
cases := []protocol.Request{
|
||||
{
|
||||
Repo: "foo",
|
||||
URL: "u",
|
||||
Commit: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
||||
// No MatchTemplate
|
||||
},
|
||||
}
|
||||
|
||||
store, cleanup, err := testutil.NewStore(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cleanup()
|
||||
ts := httptest.NewServer(&replace.Service{Store: store})
|
||||
defer ts.Close()
|
||||
|
||||
for _, p := range cases {
|
||||
_, err := doReplace(ts.URL, &p)
|
||||
if err == nil {
|
||||
t.Fatalf("%v expected to fail", p)
|
||||
}
|
||||
if !strings.HasPrefix(err.Error(), "non-200 response: code=400 ") {
|
||||
t.Fatalf("%v expected to have HTTP 400 response. Got %s", p, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func doReplace(u string, p *protocol.Request) (string, error) {
|
||||
form := url.Values{
|
||||
"Repo": []string{string(p.Repo)},
|
||||
"URL": []string{string(p.URL)},
|
||||
"Commit": []string{string(p.Commit)},
|
||||
"FetchTimeout": []string{p.FetchTimeout},
|
||||
"MatchTemplate": []string{p.RewriteSpecification.MatchTemplate},
|
||||
"RewriteTemplate": []string{p.RewriteSpecification.RewriteTemplate},
|
||||
"FileExtension": []string{p.RewriteSpecification.FileExtension},
|
||||
}
|
||||
resp, err := http.PostForm(u, form)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("non-200 response: code=%d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return string(body), err
|
||||
}
|
||||
@ -61,7 +61,6 @@ PACKAGES=(
|
||||
github.com/sourcegraph/sourcegraph/cmd/github-proxy
|
||||
github.com/sourcegraph/sourcegraph/cmd/gitserver
|
||||
github.com/sourcegraph/sourcegraph/cmd/query-runner
|
||||
github.com/sourcegraph/sourcegraph/cmd/replacer
|
||||
github.com/sourcegraph/sourcegraph/cmd/searcher
|
||||
github.com/sourcegraph/sourcegraph/cmd/symbols
|
||||
github.com/google/zoekt/cmd/zoekt-archive-index
|
||||
|
||||
@ -16,7 +16,6 @@ var SrcProfServices = []map[string]string{
|
||||
{"Name": "symbols", "Host": "127.0.0.1:6071"},
|
||||
{"Name": "repo-updater", "Host": "127.0.0.1:6074"},
|
||||
{"Name": "query-runner", "Host": "127.0.0.1:6067"},
|
||||
{"Name": "replacer", "Host": "127.0.0.1:6076"},
|
||||
{"Name": "zoekt-indexserver", "Host": "127.0.0.1:6072"},
|
||||
{"Name": "zoekt-webserver", "Host": "127.0.0.1:3070", "DefaultPath": "/debug/requests/"},
|
||||
}
|
||||
|
||||
@ -33,7 +33,6 @@ var DefaultEnv = map[string]string{
|
||||
"QUERY_RUNNER_URL": "http://127.0.0.1:3183",
|
||||
"SRC_SYNTECT_SERVER": "http://127.0.0.1:9238",
|
||||
"SYMBOLS_URL": "http://127.0.0.1:3184",
|
||||
"REPLACER_URL": "http://127.0.0.1:3185",
|
||||
"SRC_HTTP_ADDR": ":8080",
|
||||
"SRC_HTTPS_ADDR": ":8443",
|
||||
"SRC_FRONTEND_INTERNAL": FrontendInternalHost,
|
||||
@ -46,7 +45,6 @@ var DefaultEnv = map[string]string{
|
||||
// searcher/symbols to ensure this value isn't larger than the volume for
|
||||
// CACHE_DIR.
|
||||
"SEARCHER_CACHE_SIZE_MB": "50000",
|
||||
"REPLACER_CACHE_SIZE_MB": "50000",
|
||||
"SYMBOLS_CACHE_SIZE_MB": "50000",
|
||||
|
||||
// Used to differentiate between deployments on dev, Docker, and Kubernetes.
|
||||
@ -138,7 +136,6 @@ func Main() {
|
||||
`query-runner: query-runner`,
|
||||
`symbols: symbols`,
|
||||
`searcher: searcher`,
|
||||
`replacer: replacer`,
|
||||
`github-proxy: github-proxy`,
|
||||
`repo-updater: repo-updater`,
|
||||
`syntect_server: sh -c 'env QUIET=true ROCKET_ENV=production ROCKET_PORT=9238 ROCKET_LIMITS='"'"'{json=10485760}'"'"' ROCKET_SECRET_KEY='"'"'SeerutKeyIsI7releuantAndknvsuZPluaseIgnorYA='"'"' ROCKET_KEEP_ALIVE=0 ROCKET_ADDRESS='"'"'"127.0.0.1"'"'"' syntect_server | grep -v "Rocket has launched" | grep -v "Warning: environment is"' | grep -v 'Configured for production'`,
|
||||
|
||||
@ -2,7 +2,6 @@ gitserver: gitserver
|
||||
query-runner: query-runner
|
||||
repo-updater: repo-updater
|
||||
searcher: searcher
|
||||
replacer: replacer
|
||||
symbols: symbols
|
||||
github-proxy: github-proxy
|
||||
frontend: env CONFIGURATION_MODE=server SITE_CONFIG_ESCAPE_HATCH_PATH=$HOME/.sourcegraph/site-config.json frontend
|
||||
|
||||
@ -6,7 +6,7 @@ set -e
|
||||
echo "--- build libsqlite"
|
||||
./dev/libsqlite3-pcre/build.sh
|
||||
|
||||
# For searcher and replacer tests
|
||||
# For searcher
|
||||
echo "--- comby install"
|
||||
./dev/comby-install-or-upgrade.sh
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This function installs the comby dependency for cmd/searcher and
|
||||
# cmd/replacer. The CI pipeline calls this script to install or upgrade comby
|
||||
# This function installs the comby dependency for cmd/searcher.
|
||||
# The CI pipeline calls this script to install or upgrade comby
|
||||
# for tests or development environments.
|
||||
REQUIRE_VERSION="0.14.1"
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
# This will install binaries into the `.bin` directory under the repository root.
|
||||
#
|
||||
|
||||
all_oss_commands=" gitserver query-runner github-proxy searcher replacer frontend repo-updater symbols "
|
||||
all_oss_commands=" gitserver query-runner github-proxy searcher frontend repo-updater symbols "
|
||||
all_commands="${all_oss_commands}${ENTERPRISE_ONLY_COMMANDS}"
|
||||
|
||||
# handle options
|
||||
|
||||
@ -54,8 +54,3 @@
|
||||
targets:
|
||||
# postgres exporter
|
||||
- host.docker.internal:9187
|
||||
- labels:
|
||||
job: replacer
|
||||
targets:
|
||||
# replacer
|
||||
- host.docker.internal:6076
|
||||
|
||||
@ -54,8 +54,3 @@
|
||||
targets:
|
||||
# postgres exporter
|
||||
- 127.0.0.1:9187
|
||||
- labels:
|
||||
job: replacer
|
||||
targets:
|
||||
# replacer
|
||||
- 127.0.0.1:6076
|
||||
|
||||
@ -5,7 +5,6 @@
|
||||
{ "Name": "symbols", "Host": "127.0.0.1:6071" },
|
||||
{ "Name": "repo-updater", "Host": "127.0.0.1:6074" },
|
||||
{ "Name": "query-runner", "Host": "127.0.0.1:6067" },
|
||||
{ "Name": "replacer", "Host": "127.0.0.1:6076" },
|
||||
{ "Name": "precise-code-intel-bundle-manager", "Host": "127.0.0.1:6087" },
|
||||
{ "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" },
|
||||
{ "Name": "precise-code-intel-indexer", "Host": "127.0.0.1:6089" },
|
||||
|
||||
@ -63,7 +63,6 @@ export INSECURE_DEV=1
|
||||
export SRC_GIT_SERVERS=127.0.0.1:3178
|
||||
export GOLANGSERVER_SRC_GIT_SERVERS=host.docker.internal:3178
|
||||
export SEARCHER_URL=http://127.0.0.1:3181
|
||||
export REPLACER_URL=http://127.0.0.1:3185
|
||||
export REPO_UPDATER_URL=http://127.0.0.1:3182
|
||||
export REDIS_ENDPOINT=127.0.0.1:6379
|
||||
export QUERY_RUNNER_URL=http://localhost:3183
|
||||
|
||||
@ -2850,229 +2850,6 @@ To learn more about Sourcegraph's alerting, see [our alerting documentation](htt
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: frontend_internal_api_error_responses
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 5+ frontend-internal API error responses every 5m by route_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing.
|
||||
- **Kubernetes:**
|
||||
- Confirm that `kubectl get pods` shows the `frontend` pods are healthy.
|
||||
- Check `kubectl logs replacer` for logs indicate request failures to `frontend` or `frontend-internal`.
|
||||
- **Docker Compose:**
|
||||
- Confirm that `docker ps` shows the `frontend-internal` container is healthy.
|
||||
- Check `docker logs replacer` for logs indicating request failures to `frontend` or `frontend-internal`.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_frontend_internal_api_error_responses"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: container_cpu_usage
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 99%+ container cpu usage total (1m average) across all cores by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`.
|
||||
- **Docker Compose:** Consider increasing `cpus:` of the replacer container in `docker-compose.yml`.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_container_cpu_usage"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: container_memory_usage
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 99%+ container memory usage by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`.
|
||||
- **Docker Compose:** Consider increasing `memory:` of replacer container in `docker-compose.yml`.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_container_memory_usage"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: container_restarts
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 1+ container restarts every 5m by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Kubernetes:**
|
||||
- Determine if the pod was OOM killed using `kubectl describe pod replacer` (look for `OOMKilled: true`) and, if so, consider increasing the memory limit in the relevant `Deployment.yaml`.
|
||||
- Check the logs before the container restarted to see if there are `panic:` messages or similar using `kubectl logs -p replacer`.
|
||||
- **Docker Compose:**
|
||||
- Determine if the pod was OOM killed using `docker inspect -f '{{json .State}}' replacer` (look for `"OOMKilled":true`) and, if so, consider increasing the memory limit of the replacer container in `docker-compose.yml`.
|
||||
- Check the logs before the container restarted to see if there are `panic:` messages or similar using `docker logs replacer` (note this will include logs from the previous and currently running container).
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_container_restarts"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: fs_inodes_used
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 3e+06+ fs inodes in use by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- - Refer to your OS or cloud provider`s documentation for how to increase inodes.
|
||||
- **Kubernetes:** consider provisioning more machines with less resources.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_fs_inodes_used"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: provisioning_container_cpu_usage_7d
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 80%+ or less than 30% container cpu usage total (7d maximum) across all cores by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- If usage is high:
|
||||
- **Kubernetes:** Consider decreasing CPU limits in the the relevant `Deployment.yaml`.
|
||||
- **Docker Compose:** Consider descreasing `cpus:` of the replacer container in `docker-compose.yml`.
|
||||
- If usage is low, consider decreasing the above values.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_provisioning_container_cpu_usage_7d"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: provisioning_container_memory_usage_7d
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 80%+ or less than 30% container memory usage (7d maximum) by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- If usage is high:
|
||||
- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`.
|
||||
- **Docker Compose:** Consider increasing `memory:` of replacer container in `docker-compose.yml`.
|
||||
- If usage is low, consider decreasing the above values.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_provisioning_container_memory_usage_7d"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: provisioning_container_cpu_usage_5m
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 90%+ container cpu usage total (5m maximum) across all cores by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`.
|
||||
- **Docker Compose:** Consider increasing `cpus:` of the replacer container in `docker-compose.yml`.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_provisioning_container_cpu_usage_5m"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: provisioning_container_memory_usage_5m
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 90%+ container memory usage (5m maximum) by instance_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`.
|
||||
- **Docker Compose:** Consider increasing `memory:` of replacer container in `docker-compose.yml`.
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_provisioning_container_memory_usage_5m"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: go_goroutines
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 10000+ maximum active goroutines for 10m_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_go_goroutines"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: go_gc_duration_seconds
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: 2s+ maximum go garbage collection duration_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"warning_replacer_go_gc_duration_seconds"
|
||||
]
|
||||
```
|
||||
|
||||
## replacer: pods_available_percentage
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
- _replacer: less than 90% percentage pods available for 10m_
|
||||
|
||||
**Possible solutions:**
|
||||
|
||||
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
|
||||
|
||||
```json
|
||||
"observability.silenceAlerts": [
|
||||
"critical_replacer_pods_available_percentage"
|
||||
]
|
||||
```
|
||||
|
||||
## repo-updater: frontend_internal_api_error_responses
|
||||
|
||||
**Descriptions:**
|
||||
|
||||
@ -32,7 +32,6 @@ To set up notifications for these alerts, see: [alerting](alerting.md).
|
||||
- `"gitserver"`
|
||||
- `"precise-code-intel"`
|
||||
- `"query-runner"`
|
||||
- `"replacer"`
|
||||
- `"repo-updater"`
|
||||
- `"searcher"`
|
||||
- `"symbols"`
|
||||
|
||||
@ -75,7 +75,6 @@ This is a table of Sourcegraph backend debug ports in the two deployment context
|
||||
| query-runner | 6060 | 6067 |
|
||||
| zoekt-indexserver | 6060 | 6072 |
|
||||
| zoekt-webserver | 6060 | 3070 |
|
||||
| replacer | 6060 | 6076 |
|
||||
|
||||
|
||||
## Profiling kinds
|
||||
|
||||
@ -61,12 +61,6 @@ digraph architecture {
|
||||
shape="box3d"
|
||||
URL="https://github.com/sourcegraph/sourcegraph/tree/master/cmd/searcher"
|
||||
]
|
||||
replacer [
|
||||
label="replacer"
|
||||
fillcolor="5"
|
||||
shape="box3d"
|
||||
URL="https://github.com/sourcegraph/sourcegraph/tree/master/cmd/replacer"
|
||||
]
|
||||
query_runner [
|
||||
label="query\nrunner"
|
||||
fillcolor="6"
|
||||
@ -152,7 +146,6 @@ digraph architecture {
|
||||
gitserver
|
||||
query_runner
|
||||
searcher
|
||||
replacer
|
||||
repo_updater
|
||||
github_proxy
|
||||
zoekt_webserver
|
||||
@ -173,13 +166,6 @@ digraph architecture {
|
||||
fillcolor="4"
|
||||
]
|
||||
|
||||
replacer -> {
|
||||
frontend
|
||||
gitserver
|
||||
} [
|
||||
fillcolor="5"
|
||||
]
|
||||
|
||||
query_runner -> frontend [fillcolor="6"]
|
||||
|
||||
symbols -> {
|
||||
|
||||
@ -99,24 +99,6 @@
|
||||
<path fill="none" stroke="black" stroke-width="0.6" d="M352.71,-560.99C328.73,-552 300.1,-538.19 279.94,-517.73 259.3,-496.77 247.52,-464.89 238.39,-441.56"/>
|
||||
<polygon fill="#8dd3c7" stroke="black" stroke-width="0.6" points="242.73,-439.69 233.27,-428.97 233.98,-443.25 242.73,-439.69"/>
|
||||
</g>
|
||||
<!-- replacer -->
|
||||
<g id="node5" class="node">
|
||||
<title>replacer</title>
|
||||
<g id="a_node5"><a xlink:href="https://github.com/sourcegraph/sourcegraph/tree/master/cmd/replacer" xlink:title="replacer" target="_blank">
|
||||
<polygon fill="#80b1d3" stroke="black" points="424.44,-428.73 371.44,-428.73 367.44,-424.73 367.44,-392.73 420.44,-392.73 424.44,-396.73 424.44,-428.73"/>
|
||||
<polyline fill="none" stroke="black" points="420.44,-424.73 367.44,-424.73 "/>
|
||||
<polyline fill="none" stroke="black" points="420.44,-424.73 420.44,-392.73 "/>
|
||||
<polyline fill="none" stroke="black" points="420.44,-424.73 424.44,-428.73 "/>
|
||||
<text text-anchor="middle" x="395.94" y="-408.23" font-family="Iosevka" font-size="10.00">replacer</text>
|
||||
</a>
|
||||
</g>
|
||||
</g>
|
||||
<!-- frontend->replacer -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>frontend->replacer</title>
|
||||
<path fill="none" stroke="black" stroke-width="0.6" d="M405.92,-533.49C407.89,-505.09 407.57,-468.29 404.96,-442.61"/>
|
||||
<polygon fill="#8dd3c7" stroke="black" stroke-width="0.6" points="409.61,-441.69 403.17,-428.93 400.24,-442.93 409.61,-441.69"/>
|
||||
</g>
|
||||
<!-- query_runner -->
|
||||
<g id="node6" class="node">
|
||||
<title>query_runner</title>
|
||||
@ -380,18 +362,6 @@
|
||||
<path fill="none" stroke="black" stroke-width="0.6" d="M246.53,-392.46C252.57,-388.58 258.96,-384.49 264.94,-380.73 310.87,-351.8 328.78,-353.23 368.94,-316.73 378.51,-308.02 387.45,-297 394.73,-286.89"/>
|
||||
<polygon fill="#fb8072" stroke="black" stroke-width="0.6" points="398.71,-289.44 402.46,-275.64 390.92,-284.09 398.71,-289.44"/>
|
||||
</g>
|
||||
<!-- replacer->frontend -->
|
||||
<g id="edge16" class="edge">
|
||||
<title>replacer->frontend</title>
|
||||
<path fill="none" stroke="black" stroke-width="0.6" d="M389.2,-428.93C385.79,-450.35 384.7,-488.01 385.93,-519.82"/>
|
||||
<polygon fill="#80b1d3" stroke="black" stroke-width="0.6" points="381.22,-520.25 386.63,-533.49 390.66,-519.77 381.22,-520.25"/>
|
||||
</g>
|
||||
<!-- replacer->gitserver -->
|
||||
<g id="edge17" class="edge">
|
||||
<title>replacer->gitserver</title>
|
||||
<path fill="none" stroke="black" stroke-width="0.6" d="M397.86,-392.67C400.68,-367.59 405.98,-320.38 409.49,-289.17"/>
|
||||
<polygon fill="#80b1d3" stroke="black" stroke-width="0.6" points="414.19,-289.64 411.01,-275.7 404.8,-288.59 414.19,-289.64"/>
|
||||
</g>
|
||||
<!-- query_runner->frontend -->
|
||||
<g id="edge18" class="edge">
|
||||
<title>query_runner->frontend</title>
|
||||
|
||||
|
Before Width: | Height: | Size: 34 KiB After Width: | Height: | Size: 32 KiB |
@ -29,7 +29,6 @@ We use the term "search backend" to describe a function which performs a search
|
||||
- `searchFilesInRepos` for finding `type:file` (text) and `type:path` (filepath) results.
|
||||
- `searchCommitDiffsInRepos` for finding `type:diff` results.
|
||||
- `searchCommitLogInRepos` for finding `type:commit` results.
|
||||
- `performCodemod` for code modification find/replace (read-only) results.
|
||||
|
||||
### "cursor"
|
||||
|
||||
|
||||
@ -2,7 +2,6 @@ gitserver: gitserver
|
||||
query-runner: query-runner
|
||||
repo-updater: repo-updater
|
||||
searcher: searcher
|
||||
replacer: replacer
|
||||
symbols: symbols
|
||||
github-proxy: github-proxy
|
||||
frontend: env CONFIGURATION_MODE=server SITE_CONFIG_ESCAPE_HATCH_PATH=$HOME/.sourcegraph/site-config.json frontend
|
||||
|
||||
@ -14,7 +14,6 @@ var allDockerImages = []string{
|
||||
"github-proxy",
|
||||
"gitserver",
|
||||
"query-runner",
|
||||
"replacer",
|
||||
"repo-updater",
|
||||
"searcher",
|
||||
"server",
|
||||
|
||||
@ -5,7 +5,6 @@
|
||||
{ "Name": "symbols", "Host": "127.0.0.1:6071" },
|
||||
{ "Name": "repo-updater", "Host": "127.0.0.1:6074" },
|
||||
{ "Name": "query-runner", "Host": "127.0.0.1:6067" },
|
||||
{ "Name": "replacer", "Host": "127.0.0.1:6076" },
|
||||
{ "Name": "precise-code-intel-bundle-manager", "Host": "127.0.0.1:6087" },
|
||||
{ "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" }
|
||||
]
|
||||
|
||||
@ -31,7 +31,6 @@ func restartGoremanDev() error {
|
||||
"query-runner",
|
||||
"repo-updater",
|
||||
"searcher",
|
||||
"replacer",
|
||||
"symbols",
|
||||
"github-proxy",
|
||||
"syntect_server",
|
||||
|
||||
@ -35,6 +35,5 @@ var allFields = map[string]struct{}{
|
||||
FieldStable: empty,
|
||||
FieldMax: empty,
|
||||
FieldTimeout: empty,
|
||||
FieldReplace: empty,
|
||||
FieldCombyRule: empty,
|
||||
}
|
||||
|
||||
@ -40,7 +40,6 @@ const (
|
||||
FieldStable = "stable" // Forces search to return a stable result ordering (currently limited to file content matches).
|
||||
FieldMax = "max" // Deprecated alias for count
|
||||
FieldTimeout = "timeout"
|
||||
FieldReplace = "replace"
|
||||
FieldCombyRule = "rule"
|
||||
)
|
||||
|
||||
@ -78,7 +77,6 @@ var (
|
||||
FieldStable: {Literal: types.BoolType, Quoted: types.BoolType, Singular: true},
|
||||
FieldMax: {Literal: types.StringType, Quoted: types.StringType, Singular: true},
|
||||
FieldTimeout: {Literal: types.StringType, Quoted: types.StringType, Singular: true},
|
||||
FieldReplace: {Literal: types.StringType, Quoted: types.StringType, Singular: true},
|
||||
FieldCombyRule: {Literal: types.StringType, Quoted: types.StringType, Singular: true},
|
||||
},
|
||||
FieldAliases: map[string]string{
|
||||
|
||||
@ -251,7 +251,6 @@ func (q AndOrQuery) valueToTypedValue(field, value string, label labels) []*type
|
||||
FieldCount,
|
||||
FieldMax,
|
||||
FieldTimeout,
|
||||
FieldReplace,
|
||||
FieldCombyRule:
|
||||
return []*types.Value{{String: &value}}
|
||||
}
|
||||
|
||||
@ -315,7 +315,6 @@ func validateField(field, value string, negated bool, seen map[string]struct{})
|
||||
case
|
||||
FieldMax,
|
||||
FieldTimeout,
|
||||
FieldReplace,
|
||||
FieldCombyRule:
|
||||
return satisfies(isSingular, isNotNegated)
|
||||
default:
|
||||
|
||||
@ -905,7 +905,6 @@ func main() {
|
||||
PreciseCodeIntelWorker(),
|
||||
PreciseCodeIntelIndexer(),
|
||||
QueryRunner(),
|
||||
Replacer(),
|
||||
RepoUpdater(),
|
||||
Searcher(),
|
||||
Symbols(),
|
||||
|
||||
@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
func Replacer() *Container {
|
||||
return &Container{
|
||||
Name: "replacer",
|
||||
Title: "Replacer",
|
||||
Description: "Backend for find-and-replace operations.",
|
||||
Groups: []Group{
|
||||
{
|
||||
Title: "General",
|
||||
Rows: []Row{
|
||||
{
|
||||
sharedFrontendInternalAPIErrorResponses("replacer"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Title: "Container monitoring (not available on server)",
|
||||
Hidden: true,
|
||||
Rows: []Row{
|
||||
{
|
||||
sharedContainerCPUUsage("replacer"),
|
||||
sharedContainerMemoryUsage("replacer"),
|
||||
},
|
||||
{
|
||||
sharedContainerRestarts("replacer"),
|
||||
sharedContainerFsInodes("replacer"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Title: "Provisioning indicators (not available on server)",
|
||||
Hidden: true,
|
||||
Rows: []Row{
|
||||
{
|
||||
sharedProvisioningCPUUsage7d("replacer"),
|
||||
sharedProvisioningMemoryUsage7d("replacer"),
|
||||
},
|
||||
{
|
||||
sharedProvisioningCPUUsage5m("replacer"),
|
||||
sharedProvisioningMemoryUsage5m("replacer"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Title: "Golang runtime monitoring",
|
||||
Hidden: true,
|
||||
Rows: []Row{
|
||||
{
|
||||
sharedGoGoroutines("replacer"),
|
||||
sharedGoGcDuration("replacer"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Title: "Kubernetes monitoring (ignore if using Docker Compose or server)",
|
||||
Hidden: true,
|
||||
Rows: []Row{
|
||||
{
|
||||
sharedKubernetesPodsAvailable("replacer"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
/**
|
||||
* Whether the experimental code modification feature is enabled.
|
||||
*
|
||||
* To enable this, run `localStorage.codemodExp=true;location.reload()` in your browser's JavaScript
|
||||
* console.
|
||||
*/
|
||||
export const USE_CODEMOD = localStorage.getItem('codemodExp') !== null
|
||||
@ -5,7 +5,6 @@ import * as GQL from '../../../shared/src/graphql/schema'
|
||||
import { asError, createAggregateError, ErrorLike } from '../../../shared/src/util/errors'
|
||||
import { memoizeObservable } from '../../../shared/src/util/memoizeObservable'
|
||||
import { mutateGraphQL, queryGraphQL } from '../backend/graphql'
|
||||
import { USE_CODEMOD } from '../enterprise/codemod'
|
||||
import { SearchSuggestion } from '../../../shared/src/search/suggestions'
|
||||
import { Remote } from 'comlink'
|
||||
import { FlatExtHostAPI } from '../../../shared/src/api/contract'
|
||||
@ -31,7 +30,6 @@ export function search(
|
||||
$query: String!
|
||||
$version: SearchVersion!
|
||||
$patternType: SearchPatternType!
|
||||
$useCodemod: Boolean!
|
||||
$versionContext: String
|
||||
) {
|
||||
search(
|
||||
@ -163,31 +161,6 @@ export function search(
|
||||
}
|
||||
# end of genericSearchResultInterfaceFields inline fragment
|
||||
}
|
||||
... on CodemodResult @include(if: $useCodemod) {
|
||||
# TODO: Make this a proper fragment, blocked by https://github.com/graph-gophers/graphql-go/issues/241.
|
||||
# beginning of genericSearchResultInterfaceFields inline fragment
|
||||
label {
|
||||
html
|
||||
}
|
||||
url
|
||||
icon
|
||||
detail {
|
||||
html
|
||||
}
|
||||
matches {
|
||||
url
|
||||
body {
|
||||
text
|
||||
html
|
||||
}
|
||||
highlights {
|
||||
line
|
||||
character
|
||||
length
|
||||
}
|
||||
}
|
||||
# end of genericSearchResultInterfaceFields inline fragment
|
||||
}
|
||||
}
|
||||
alert {
|
||||
title
|
||||
@ -202,7 +175,7 @@ export function search(
|
||||
}
|
||||
}
|
||||
`,
|
||||
{ query, version, patternType, versionContext, useCodemod: USE_CODEMOD }
|
||||
{ query, version, patternType, versionContext }
|
||||
).pipe(
|
||||
map(({ data, errors }) => {
|
||||
if (!data || !data.search || !data.search.results) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user