diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 931e05d3b6f..037a1a0a2cb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -147,13 +147,12 @@ /internal/campaigns @sourcegraph/campaigns /web/**/campaigns/** @sourcegraph/campaigns -# Search and code mod +# Search */search/**/* @sourcegraph/search /cmd/frontend/internal/pkg/search @sourcegraph/search /cmd/frontend/graphqlbackend/*search* @sourcegraph/search /cmd/frontend/graphqlbackend/*zoekt* @sourcegraph/search /cmd/query-runner/ @sourcegraph/search -/cmd/replacer/ @sourcegraph/search @rvantonder /cmd/searcher/ @sourcegraph/search /cmd/symbols/ @sourcegraph/search /internal/search/ @sourcegraph/search diff --git a/cmd/frontend/graphqlbackend/codemod.go b/cmd/frontend/graphqlbackend/codemod.go deleted file mode 100644 index 11c80e8ec17..00000000000 --- a/cmd/frontend/graphqlbackend/codemod.go +++ /dev/null @@ -1,330 +0,0 @@ -package graphqlbackend - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "strings" - "sync" - - "github.com/inconshreveable/log15" - "github.com/opentracing-contrib/go-stdlib/nethttp" - otlog "github.com/opentracing/opentracing-go/log" - "github.com/pkg/errors" - "github.com/sourcegraph/go-diff/diff" - "github.com/sourcegraph/sourcegraph/internal/env" - "github.com/sourcegraph/sourcegraph/internal/errcode" - "github.com/sourcegraph/sourcegraph/internal/goroutine" - "github.com/sourcegraph/sourcegraph/internal/lazyregexp" - "github.com/sourcegraph/sourcegraph/internal/search" - "github.com/sourcegraph/sourcegraph/internal/search/query" - "github.com/sourcegraph/sourcegraph/internal/trace" - "github.com/sourcegraph/sourcegraph/internal/trace/ot" - "github.com/sourcegraph/sourcegraph/internal/vcs/git" - "golang.org/x/net/context/ctxhttp" -) - -type rawCodemodResult struct { - URI string `json:"uri"` - Diff string -} - -type args struct { - matchTemplate string - rewriteTemplate string - includeFileFilter string - excludeFileFilter string -} - -// codemodResultResolver is a resolver for the GraphQL type `CodemodResult` -type codemodResultResolver struct { - commit *GitCommitResolver - path string - fileURL string - diff string - matches []*searchResultMatchResolver -} - -func (r *codemodResultResolver) ToRepository() (*RepositoryResolver, bool) { return nil, false } -func (r *codemodResultResolver) ToFileMatch() (*FileMatchResolver, bool) { return nil, false } -func (r *codemodResultResolver) ToCommitSearchResult() (*commitSearchResultResolver, bool) { - return nil, false -} - -func (r *codemodResultResolver) ToCodemodResult() (*codemodResultResolver, bool) { - return r, true -} - -func (r *codemodResultResolver) searchResultURIs() (string, string) { - return string(r.commit.repoResolver.repo.Name), r.path -} - -func (r *codemodResultResolver) resultCount() int32 { - return int32(len(r.matches)) -} - -func (r *codemodResultResolver) Icon() string { - return "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' style='width:24px;height:24px' viewBox='0 0 24 24'%3E%3Cpath fill='%23a2b0cd' d='M11,6C12.38,6 13.63,6.56 14.54,7.46L12,10H18V4L15.95,6.05C14.68,4.78 12.93,4 11,4C7.47,4 4.57,6.61 4.08,10H6.1C6.56,7.72 8.58,6 11,6M16.64,15.14C17.3,14.24 17.76,13.17 17.92,12H15.9C15.44,14.28 13.42,16 11,16C9.62,16 8.37,15.44 7.46,14.54L10,12H4V18L6.05,15.95C7.32,17.22 9.07,18 11,18C12.55,18 14,17.5 15.14,16.64L20,21.5L21.5,20L16.64,15.14Z' /%3E%3C/svg%3E" -} - -func (r *codemodResultResolver) Label() (*markdownResolver, error) { - commitURL, err := r.commit.URL() - if err != nil { - return nil, err - } - text := fmt.Sprintf("[%s](%s) › [%s](%s)", r.commit.repoResolver.Name(), commitURL, r.path, r.fileURL) - return &markdownResolver{text: text}, nil -} - -func (r *codemodResultResolver) URL() string { - return r.fileURL -} - -func (r *codemodResultResolver) Detail() (*markdownResolver, error) { - diff, err := diff.ParseFileDiff([]byte(r.diff)) - if err != nil { - return nil, err - } - stat := diff.Stat() - return &markdownResolver{text: stat.String()}, nil -} - -func (r *codemodResultResolver) Matches() []*searchResultMatchResolver { - return r.matches -} - -func (r *codemodResultResolver) Commit() *GitCommitResolver { return r.commit } - -func (r *codemodResultResolver) RawDiff() string { return r.diff } - -func validateQuery(q query.QueryInfo) (*args, error) { - matchValues := q.Values(query.FieldDefault) - var matchTemplates []string - for _, v := range matchValues { - if v.String != nil && *v.String != "" { - matchTemplates = append(matchTemplates, *v.String) - } - if v.Regexp != nil || v.Bool != nil { - return nil, errors.New("this looks like a regex search pattern. Structural search is active because 'replace:' was specified. Please enclose your search string with quotes when using 'replace:'.") - } - } - matchTemplate := strings.Join(matchTemplates, " ") - - replacementValues, _ := q.StringValues(query.FieldReplace) - var rewriteTemplate string - if len(replacementValues) > 0 { - rewriteTemplate = replacementValues[0] - } - - includeFileFilter, excludeFileFilter := q.RegexpPatterns(query.FieldFile) - var includeFileFilterText string - if len(includeFileFilter) > 0 { - includeFileFilterText = includeFileFilter[0] - // only file names or files with extensions in the following characterset are allowed - IsAlphanumericWithPeriod := lazyregexp.New(`^[a-zA-Z0-9_.]+$`).MatchString - if !IsAlphanumericWithPeriod(includeFileFilterText) { - return nil, errors.New("the 'file:' filter cannot contain regex when using the 'replace:' filter currently. Only alphanumeric characters or '.'") - } - } - - var excludeFileFilterText string - if len(excludeFileFilter) > 0 { - excludeFileFilterText = excludeFileFilter[0] - IsAlphanumericWithPeriod := lazyregexp.New(`^[a-zA-Z_.]+$`).MatchString - if !IsAlphanumericWithPeriod(includeFileFilterText) { - return nil, errors.New("the '-file:' filter cannot contain regex when using the 'replace:' filter currently. Only alphanumeric characters or '.'") - } - } - - return &args{matchTemplate, rewriteTemplate, includeFileFilterText, excludeFileFilterText}, nil -} - -// Calls the codemod backend replacer service for a set of repository revisions. -func performCodemod(ctx context.Context, args *search.TextParameters) ([]SearchResultResolver, *searchResultsCommon, error) { - cmodArgs, err := validateQuery(args.Query) - if err != nil { - return nil, nil, err - } - - title := fmt.Sprintf("pattern: %+v, replace: %+v, includeFileFilter: %+v, excludeFileFilter: %+v, numRepoRevs: %d", cmodArgs.matchTemplate, cmodArgs.rewriteTemplate, cmodArgs.includeFileFilter, cmodArgs.excludeFileFilter, len(args.Repos)) - tr, ctx := trace.New(ctx, "callCodemod", title) - defer func() { - tr.SetError(err) - tr.Finish() - }() - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - var ( - wg sync.WaitGroup - mu sync.Mutex - unflattened [][]codemodResultResolver - common = &searchResultsCommon{} - ) - for _, repoRev := range args.Repos { - wg.Add(1) - repoRev := repoRev // shadow variable so it doesn't change while goroutine is running - goroutine.Go(func() { - defer wg.Done() - results, searchErr := callCodemodInRepo(ctx, repoRev, cmodArgs) - if ctx.Err() == context.Canceled { - // Our request has been canceled (either because another one of args.repos had a - // fatal error, or otherwise), so we can just ignore these results. - return - } - repoTimedOut := ctx.Err() == context.DeadlineExceeded - if searchErr != nil { - tr.LogFields(otlog.String("repo", string(repoRev.Repo.Name)), otlog.String("searchErr", searchErr.Error()), otlog.Bool("timeout", errcode.IsTimeout(searchErr)), otlog.Bool("temporary", errcode.IsTemporary(searchErr))) - } - mu.Lock() - defer mu.Unlock() - if fatalErr := handleRepoSearchResult(common, repoRev, false, repoTimedOut, searchErr); fatalErr != nil { - err = errors.Wrapf(searchErr, "failed to call codemod %s", repoRev) - cancel() - } - if len(results) > 0 { - unflattened = append(unflattened, results) - } - }) - } - wg.Wait() - if err != nil { - return nil, nil, err - } - - var results []SearchResultResolver - for _, ur := range unflattened { - for _, resolver := range ur { - v := resolver - results = append(results, &v) - } - } - - return results, common, nil -} - -var ReplacerURL = env.Get("REPLACER_URL", "http://replacer:3185", "replacer server URL") - -func toMatchResolver(fileURL string, raw *rawCodemodResult) ([]*searchResultMatchResolver, error) { - if !strings.Contains(raw.Diff, "@@") { - return nil, errors.Errorf("Invalid diff does not contain expected @@: %v", raw.Diff) - } - strippedDiff := raw.Diff[strings.Index(raw.Diff, "@@"):] - - return []*searchResultMatchResolver{ - { - url: fileURL, - body: "```diff\n" + strippedDiff + "\n```", - highlights: nil, - }, - }, - nil -} - -func callCodemodInRepo(ctx context.Context, repoRevs *search.RepositoryRevisions, args *args) (results []codemodResultResolver, err error) { - tr, ctx := trace.New(ctx, "callCodemodInRepo", fmt.Sprintf("repoRevs: %v, pattern %+v, replace: %+v", repoRevs, args.matchTemplate, args.rewriteTemplate)) - defer func() { - tr.LazyPrintf("%d results", len(results)) - tr.SetError(err) - tr.Finish() - }() - - // For performance, assume repo is cloned in gitserver and do not trigger a repo-updater lookup (this call fails if repo is not on gitserver). - commit, err := git.ResolveRevision(ctx, repoRevs.GitserverRepo(), nil, repoRevs.Revs[0].RevSpec, git.ResolveRevisionOptions{NoEnsureRevision: true}) - if err != nil { - return nil, errors.Wrap(err, "codemod repo lookup failed: it's possible that the repo is not cloned in gitserver. Try force a repo update another way.") - } - - u, err := url.Parse(ReplacerURL) - if err != nil { - return nil, err - } - q := u.Query() - q.Set("repo", string(repoRevs.Repo.Name)) - q.Set("commit", string(commit)) - q.Set("matchtemplate", args.matchTemplate) - q.Set("rewritetemplate", args.rewriteTemplate) - q.Set("fileextension", args.includeFileFilter) - q.Set("directoryexclude", args.excludeFileFilter) - u.RawQuery = q.Encode() - - req, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - - req, ht := nethttp.TraceRequest(ot.GetTracer(ctx), req, - nethttp.OperationName("Codemod client"), - nethttp.ClientTrace(false)) - defer ht.Finish() - - // TODO(RVT): Use a separate HTTP client here dedicated to codemod, - // not doing so means codemod and searcher share the same HTTP limits - // etc. which is fine for now but not if codemod goes in front of users. - // Once separated please fix cmd/frontend/graphqlbackend/textsearch:50 in #6586 - resp, err := ctxhttp.Do(ctx, searchHTTPClient, req) - if err != nil { - // If we failed due to cancellation or timeout (with no partial results in the response body), return just that. - if ctx.Err() != nil { - err = ctx.Err() - } - return nil, errors.Wrap(err, "codemod request failed") - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return nil, errors.WithStack(&searcherError{StatusCode: resp.StatusCode, Message: string(body)}) - } - - scanner := bufio.NewScanner(resp.Body) - // TODO(RVT): Remove buffer inefficiency introduced here. Results are - // line encoded JSON. Malformed JSON can happen, for extremely long - // lines. Unless we know where this and subsequent malformed lines end, - // we can't continue decoding the next one. As an inefficient but robust - // solution, we allow to buffer a very high maximum length line and can - // skip over very long malformed lines. It is set to 10 * 64K. - scanner.Buffer(make([]byte, 100), 10*bufio.MaxScanTokenSize) - - repoResolver := &RepositoryResolver{repo: repoRevs.Repo} - - for scanner.Scan() { - var raw *rawCodemodResult - b := scanner.Bytes() - if err := scanner.Err(); err != nil { - log15.Info(fmt.Sprintf("Skipping codemod scanner error (line too long?): %s", err.Error())) - continue - } - if err := json.Unmarshal(b, &raw); err != nil { - // skip on other decode errors (including e.g., empty - // responses if dependencies are not installed) - continue - } - fileURL := fileMatchURI(repoRevs.Repo.Name, repoRevs.Revs[0].RevSpec, raw.URI) - matches, err := toMatchResolver(fileURL, raw) - if err != nil { - return nil, err - } - results = append(results, codemodResultResolver{ - commit: &GitCommitResolver{ - repoResolver: repoResolver, - inputRev: &repoRevs.Revs[0].RevSpec, - oid: GitObjectID(commit), - }, - path: raw.URI, - fileURL: fileURL, - diff: raw.Diff, - matches: matches, - }) - } - - return results, nil -} diff --git a/cmd/frontend/graphqlbackend/codemod_test.go b/cmd/frontend/graphqlbackend/codemod_test.go deleted file mode 100644 index 7ed365a519d..00000000000 --- a/cmd/frontend/graphqlbackend/codemod_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package graphqlbackend - -import ( - "strings" - "testing" - - "github.com/sourcegraph/sourcegraph/internal/search/query" -) - -func TestCodemod_validateArgsNoRegex(t *testing.T) { - q, _ := query.ParseAndCheck("re.*gex") - _, err := validateQuery(q) - if err == nil { - t.Fatalf("Expected query %v to fail", q) - } - if !strings.HasPrefix(err.Error(), "this looks like a regex search pattern.") { - t.Fatalf("%v expected complaint about regex pattern. Got %s", q, err) - } -} - -func TestCodemod_validateArgsOk(t *testing.T) { - q, _ := query.ParseAndCheck(`"not regex"`) - _, err := validateQuery(q) - if err != nil { - t.Fatalf("Expected query %v to to be OK", q) - } -} - -func TestCodemod_resolver(t *testing.T) { - raw := &rawCodemodResult{ - URI: "", - Diff: "Not a valid diff", - } - _, err := toMatchResolver("", raw) - if err == nil { - t.Fatalf("Expected invalid diff for %v", raw.Diff) - } - if !strings.HasPrefix(err.Error(), "Invalid diff") { - t.Fatalf("Expected error %q", err) - } -} diff --git a/cmd/frontend/graphqlbackend/graphqlbackend_test.go b/cmd/frontend/graphqlbackend/graphqlbackend_test.go index 58efbe13e4c..a80fe59415e 100644 --- a/cmd/frontend/graphqlbackend/graphqlbackend_test.go +++ b/cmd/frontend/graphqlbackend/graphqlbackend_test.go @@ -71,7 +71,6 @@ func TestResolverTo(t *testing.T) { &NamespaceResolver{}, &NodeResolver{}, &RepositoryResolver{}, - &codemodResultResolver{}, &commitSearchResultResolver{}, &gitRevSpec{}, &searchSuggestionResolver{}, diff --git a/cmd/frontend/graphqlbackend/repository.go b/cmd/frontend/graphqlbackend/repository.go index 9941c6f88a0..319315a7096 100644 --- a/cmd/frontend/graphqlbackend/repository.go +++ b/cmd/frontend/graphqlbackend/repository.go @@ -279,9 +279,6 @@ func (r *RepositoryResolver) ToFileMatch() (*FileMatchResolver, bool) { return func (r *RepositoryResolver) ToCommitSearchResult() (*commitSearchResultResolver, bool) { return nil, false } -func (r *RepositoryResolver) ToCodemodResult() (*codemodResultResolver, bool) { - return nil, false -} func (r *RepositoryResolver) searchResultURIs() (string, string) { return string(r.repo.Name), "" diff --git a/cmd/frontend/graphqlbackend/schema.go b/cmd/frontend/graphqlbackend/schema.go index 95903619dd8..182e850b87b 100644 --- a/cmd/frontend/graphqlbackend/schema.go +++ b/cmd/frontend/graphqlbackend/schema.go @@ -1511,7 +1511,7 @@ type SearchFilterSuggestions { } # A search result. -union SearchResult = FileMatch | CommitSearchResult | Repository | CodemodResult +union SearchResult = FileMatch | CommitSearchResult | Repository # An object representing a markdown string. type Markdown { @@ -1741,24 +1741,6 @@ type CommitSearchResult implements GenericSearchResultInterface { diffPreview: HighlightedString } -# The result of a code modification query. -type CodemodResult implements GenericSearchResultInterface { - # URL to an icon that is displayed with every search result. - icon: String! - # A markdown string that is rendered prominently. - label: Markdown! - # The URL of the result. - url: String! - # A markdown string that is rendered less prominently. - detail: Markdown! - # A list of matches in this search result. - matches: [SearchResultMatch!]! - # The commit whose contents the codemod was run against. - commit: GitCommit! - # The raw diff of the modification. - rawDiff: String! -} - # A search result that is a diff between two diffable Git objects. type DiffSearchResult { # The diff that matched the search query. diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql index 95aca527153..6cf633cf440 100755 --- a/cmd/frontend/graphqlbackend/schema.graphql +++ b/cmd/frontend/graphqlbackend/schema.graphql @@ -1518,7 +1518,7 @@ type SearchFilterSuggestions { } # A search result. -union SearchResult = FileMatch | CommitSearchResult | Repository | CodemodResult +union SearchResult = FileMatch | CommitSearchResult | Repository # An object representing a markdown string. type Markdown { @@ -1748,24 +1748,6 @@ type CommitSearchResult implements GenericSearchResultInterface { diffPreview: HighlightedString } -# The result of a code modification query. -type CodemodResult implements GenericSearchResultInterface { - # URL to an icon that is displayed with every search result. - icon: String! - # A markdown string that is rendered prominently. - label: Markdown! - # The URL of the result. - url: String! - # A markdown string that is rendered less prominently. - detail: Markdown! - # A list of matches in this search result. - matches: [SearchResultMatch!]! - # The commit whose contents the codemod was run against. - commit: GitCommit! - # The raw diff of the modification. - rawDiff: String! -} - # A search result that is a diff between two diffable Git objects. type DiffSearchResult { # The diff that matched the search query. diff --git a/cmd/frontend/graphqlbackend/search_commits.go b/cmd/frontend/graphqlbackend/search_commits.go index 5413c3a37bb..a283ff5b22a 100644 --- a/cmd/frontend/graphqlbackend/search_commits.go +++ b/cmd/frontend/graphqlbackend/search_commits.go @@ -69,10 +69,6 @@ func (r *commitSearchResultResolver) ToCommitSearchResult() (*commitSearchResult return r, true } -func (r *commitSearchResultResolver) ToCodemodResult() (*codemodResultResolver, bool) { - return nil, false -} - func (r *commitSearchResultResolver) searchResultURIs() (string, string) { // Diffs aren't going to be returned with other types of results // and are already ordered in the desired order, so we'll just leave them in place. diff --git a/cmd/frontend/graphqlbackend/search_results.go b/cmd/frontend/graphqlbackend/search_results.go index 098c2b9623d..e306492d7c3 100644 --- a/cmd/frontend/graphqlbackend/search_results.go +++ b/cmd/frontend/graphqlbackend/search_results.go @@ -504,8 +504,6 @@ loop: } addPoint(t) }) - case *codemodResultResolver: - continue default: panic("SearchResults.Sparkline unexpected union type state") } @@ -1388,8 +1386,6 @@ func (r *searchResolver) determineResultTypes(args search.TextParameters, forceO // Determine which types of results to return. if forceOnlyResultType != "" { resultTypes = []string{forceOnlyResultType} - } else if len(r.query.Values(query.FieldReplace)) > 0 { - resultTypes = []string{"codemod"} } else { resultTypes, _ = r.query.StringValues(query.FieldType) if len(resultTypes) == 0 { @@ -1767,30 +1763,6 @@ func (r *searchResolver) doResults(ctx context.Context, forceOnlyResultType stri commonMu.Unlock() } }) - case "codemod": - wg := waitGroup(true) - wg.Add(1) - goroutine.Go(func() { - defer wg.Done() - - codemodResults, codemodCommon, err := performCodemod(ctx, &args) - // Timeouts are reported through searchResultsCommon so don't report an error for them - if err != nil && !isContextError(ctx, err) { - multiErrMu.Lock() - multiErr = multierror.Append(multiErr, errors.Wrap(err, "codemod search failed")) - multiErrMu.Unlock() - } - if codemodResults != nil { - resultsMu.Lock() - results = append(results, codemodResults...) - resultsMu.Unlock() - } - if codemodCommon != nil { - commonMu.Lock() - common.update(*codemodCommon) - commonMu.Unlock() - } - }) } } @@ -1866,7 +1838,6 @@ func isContextError(ctx context.Context, err error) bool { // - *RepositoryResolver // repo name match // - *fileMatchResolver // text match // - *commitSearchResultResolver // diff or commit match -// - *codemodResultResolver // code modification // // Note: Any new result types added here also need to be handled properly in search_results.go:301 (sparklines) type SearchResultResolver interface { @@ -1875,7 +1846,6 @@ type SearchResultResolver interface { ToRepository() (*RepositoryResolver, bool) ToFileMatch() (*FileMatchResolver, bool) ToCommitSearchResult() (*commitSearchResultResolver, bool) - ToCodemodResult() (*codemodResultResolver, bool) resultCount() int32 } diff --git a/cmd/frontend/graphqlbackend/textsearch.go b/cmd/frontend/graphqlbackend/textsearch.go index e603f195ff5..316db69944c 100644 --- a/cmd/frontend/graphqlbackend/textsearch.go +++ b/cmd/frontend/graphqlbackend/textsearch.go @@ -49,10 +49,6 @@ var ( // Default is 2, but we can send many concurrent requests MaxIdleConnsPerHost: 500, }, func(u *url.URL) string { - // TODO(uwedeportivo): remove once codemod has its own client - if strings.Contains(u.String(), "replacer") { - return "replace" - } return "search" }), }, @@ -139,10 +135,6 @@ func (fm *FileMatchResolver) ToCommitSearchResult() (*commitSearchResultResolver return nil, false } -func (r *FileMatchResolver) ToCodemodResult() (*codemodResultResolver, bool) { - return nil, false -} - func (fm *FileMatchResolver) searchResultURIs() (string, string) { return fm.Repo.Name(), fm.JPath } diff --git a/cmd/replacer/Dockerfile b/cmd/replacer/Dockerfile deleted file mode 100644 index 11f7895acdb..00000000000 --- a/cmd/replacer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM sourcegraph/alpine:3.10@sha256:4d05cd5669726fc38823e92320659a6d1ef7879e62268adec5df658a0bacf65c - -# hadolint ignore=DL3018 -RUN apk --no-cache add pcre - -# The comby/comby image is a small binary-only distribution. See the bin and src directories -# here: https://github.com/comby-tools/comby/tree/master/dockerfiles/alpine -# hadolint ignore=DL3022 -COPY --from=comby/comby:0.15.0@sha256:b3b57ed810be1489ccd1ef8fa42210d87d2d396e416c62cee345f615a0dcb09b /usr/local/bin/comby /usr/local/bin/comby - -ARG COMMIT_SHA="unknown" -ARG DATE="unknown" -ARG VERSION="unknown" - -LABEL org.opencontainers.image.revision=${COMMIT_SHA} -LABEL org.opencontainers.image.created=${DATE} -LABEL org.opencontainers.image.version=${VERSION} -LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA} - -ENV CACHE_DIR=/mnt/cache/replacer -RUN mkdir -p ${CACHE_DIR} && chown -R sourcegraph:sourcegraph ${CACHE_DIR} -USER sourcegraph -ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/replacer"] -COPY replacer /usr/local/bin/ diff --git a/cmd/replacer/build.sh b/cmd/replacer/build.sh deleted file mode 100755 index 2b90f3c4d9e..00000000000 --- a/cmd/replacer/build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -# We want to build multiple go binaries, so we use a custom build step on CI. -cd "$(dirname "${BASH_SOURCE[0]}")"/../.. -set -ex - -OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX) -cleanup() { - rm -rf "$OUTPUT" -} -trap cleanup EXIT - -# Environment for building linux binaries -export GO111MODULE=on -export GOARCH=amd64 -export GOOS=linux -export CGO_ENABLED=0 - -pkg="github.com/sourcegraph/sourcegraph/cmd/replacer" -go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg" - -docker build -f cmd/replacer/Dockerfile -t "$IMAGE" "$OUTPUT" \ - --progress=plain \ - --build-arg COMMIT_SHA \ - --build-arg DATE \ - --build-arg VERSION diff --git a/cmd/replacer/main.go b/cmd/replacer/main.go deleted file mode 100644 index 0b99836abcb..00000000000 --- a/cmd/replacer/main.go +++ /dev/null @@ -1,104 +0,0 @@ -// Command replacer is an interface to replace and rewrite code. It passes a zipped repo -// to external tools and streams back JSON lines results. -package main - -import ( - "context" - "io" - "log" - "net" - "net/http" - "os" - "os/signal" - "path/filepath" - "strconv" - "time" - - "github.com/inconshreveable/log15" - - "github.com/sourcegraph/sourcegraph/cmd/replacer/replace" - "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/debugserver" - "github.com/sourcegraph/sourcegraph/internal/env" - "github.com/sourcegraph/sourcegraph/internal/gitserver" - "github.com/sourcegraph/sourcegraph/internal/store" - "github.com/sourcegraph/sourcegraph/internal/trace/ot" - - "github.com/sourcegraph/sourcegraph/internal/tracer" -) - -var cacheDir = env.Get("CACHE_DIR", "/tmp", "directory to store cached archives.") -var cacheSizeMB = env.Get("REPLACER_CACHE_SIZE_MB", "100000", "maximum size of the on disk cache in megabytes") - -const port = "3185" - -func main() { - env.Lock() - env.HandleHelpFlag() - log.SetFlags(0) - tracer.Init() - - go debugserver.Start() - - var cacheSizeBytes int64 - if i, err := strconv.ParseInt(cacheSizeMB, 10, 64); err != nil { - log.Fatalf("invalid int %q for REPLACER_CACHE_SIZE_MB: %s", cacheSizeMB, err) - } else { - cacheSizeBytes = i * 1000 * 1000 - } - - store := store.Store{ - FetchTar: func(ctx context.Context, repo gitserver.Repo, commit api.CommitID) (io.ReadCloser, error) { - return gitserver.DefaultClient.Archive(ctx, repo, gitserver.ArchiveOptions{Treeish: string(commit), Format: "tar"}) - }, - Path: filepath.Join(cacheDir, "replacer-archives"), - MaxCacheSizeBytes: cacheSizeBytes, - } - store.SetMaxConcurrentFetchTar(10) - store.Start() - service := &replace.Service{ - Store: &store, - Log: log15.Root(), - } - handler := ot.Middleware(service) - - host := "" - if env.InsecureDev { - host = "127.0.0.1" - } - addr := net.JoinHostPort(host, port) - server := &http.Server{ - Addr: addr, - Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // For cluster liveness and readiness probes - if r.URL.Path == "/healthz" { - w.WriteHeader(200) - _, err := w.Write([]byte("ok")) - if err != nil { - log15.Info("Error checking /healthz: " + err.Error()) - } - return - } - - handler.ServeHTTP(w, r) - }), - } - go shutdownOnSIGINT(server) - - log15.Info("replacer: listening", "addr", server.Addr) - if err := server.ListenAndServe(); err != http.ErrServerClosed { - log.Fatal(err) - } -} - -func shutdownOnSIGINT(s *http.Server) { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - <-c - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - err := s.Shutdown(ctx) - if err != nil { - log.Fatal("graceful server shutdown failed, will exit:", err) - } -} diff --git a/cmd/replacer/protocol/replacer.go b/cmd/replacer/protocol/replacer.go deleted file mode 100644 index 9ed5e440036..00000000000 --- a/cmd/replacer/protocol/replacer.go +++ /dev/null @@ -1,55 +0,0 @@ -// Package protocol contains structures used by the replacer API. -package protocol - -import ( - "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/internal/gitserver" -) - -// Request represents a request to replacer -type Request struct { - // Repo is the name of the repository to search. eg "github.com/gorilla/mux" - Repo api.RepoName - - // URL specifies the repository's Git remote URL (for gitserver). It is optional. See - // (gitserver.ExecRequest).URL for documentation on what it is used for. - URL string - - // Commit is which commit to search. It is required to be resolved, - // not a ref like HEAD or master. eg - // "599cba5e7b6137d46ddf58fb1765f5d928e69604" - Commit api.CommitID - - // The amount of time to wait for a repo archive to fetch. - // It is parsed with time.ParseDuration. - // - // This timeout should be low when searching across many repos - // so that unfetched repos don't delay the search, and because we are likely - // to get results from the repos that have already been fetched. - // - // This timeout should be high when searching across a single repo - // because returning results slowly is better than returning no results at all. - // - // This only times out how long we wait for the fetch request; - // the fetch will still happen in the background so future requests don't have to wait. - FetchTimeout string - - RewriteSpecification -} - -type RewriteSpecification struct { - // A template pattern that expresses what to match. - MatchTemplate string - - // A template pattern that expresses how matches should be rewritten. - RewriteTemplate string - - // A file extension suffix filtering which files to process (e.g., ".go") - FileExtension string - - // A directory prefix to exclude (e.g., vendor) - DirectoryExclude string -} - -// GitserverRepo returns the repository information necessary to perform gitserver requests. -func (r Request) GitserverRepo() gitserver.Repo { return gitserver.Repo{Name: r.Repo, URL: r.URL} } diff --git a/cmd/replacer/replace/replace.go b/cmd/replacer/replace/replace.go deleted file mode 100644 index 76234905c52..00000000000 --- a/cmd/replacer/replace/replace.go +++ /dev/null @@ -1,309 +0,0 @@ -// Package replace is a service exposing an API to replace file contents in a repo. -// It streams back results with JSON lines. -// -// Architecture Notes: -// - The following are the same as cmd/searcher/search.go: -// * Archive is fetched from gitserver -// * Simple HTTP API exposed -// * Currently no concept of authorization -// * On disk cache of fetched archives to reduce load on gitserver -// -// - Here is where replacer.go differs -// * Pass the zip file path to external replacer tool(s) after validating -// * Read tool stdout and write it out on the HTTP connection -// * Input from stdout is expected to use JSON lines format, but the format isn't checked here: line-buffering is done on the frontend - -package replace - -import ( - "context" - "fmt" - "io" - "log" - "net/http" - "os/exec" - "strings" - "time" - - nettrace "golang.org/x/net/trace" - - "github.com/inconshreveable/log15" - "github.com/opentracing/opentracing-go/ext" - otlog "github.com/opentracing/opentracing-go/log" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/sourcegraph/sourcegraph/cmd/replacer/protocol" - "github.com/sourcegraph/sourcegraph/internal/store" - "github.com/sourcegraph/sourcegraph/internal/trace/ot" - - "github.com/gorilla/schema" -) - -type Service struct { - Store *store.Store - Log log15.Logger -} - -type ExternalTool struct { - Name string - BinaryPath string -} - -// Configure the command line options and return the command to execute using an external tool -func (t *ExternalTool) command(ctx context.Context, spec *protocol.RewriteSpecification, zipPath string) (cmd *exec.Cmd, err error) { - switch t.Name { - case "comby": - _, err = exec.LookPath("comby") - if err != nil { - return nil, errors.New("comby is not installed on the PATH. Try running 'bash <(curl -sL get.comby.dev)'.") - } - - var args []string - args = append(args, spec.MatchTemplate, spec.RewriteTemplate) - - if spec.FileExtension != "" { - args = append(args, spec.FileExtension) - } - - args = append(args, "-zip", zipPath, "-json-lines", "-json-only-diff") - - if spec.DirectoryExclude != "" { - args = append(args, "-exclude-dir", spec.DirectoryExclude) - } - - log15.Info(fmt.Sprintf("running command: comby %q", strings.Join(args[:], " "))) - return exec.CommandContext(ctx, t.BinaryPath, args...), nil - - default: - return nil, errors.Errorf("Unknown external replace tool %q.", t.Name) - } -} - -var decoder = schema.NewDecoder() - -func init() { - decoder.IgnoreUnknownKeys(true) -} - -// ServeHTTP handles HTTP based replace requests -func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - running.Inc() - defer running.Dec() - - err := r.ParseForm() - if err != nil { - log15.Info("Didn't parse" + err.Error()) - http.Error(w, "failed to parse form: "+err.Error(), http.StatusBadRequest) - return - } - - var p protocol.Request - err = decoder.Decode(&p, r.Form) - if err != nil { - http.Error(w, "failed to decode form: "+err.Error(), http.StatusBadRequest) - return - } - - if err = validateParams(&p); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - deadlineHit, err := s.replace(ctx, &p, w, r) - if err != nil { - code := http.StatusInternalServerError - if isBadRequest(err) || ctx.Err() == context.Canceled { - code = http.StatusBadRequest - } else if isTemporary(err) { - code = http.StatusServiceUnavailable - } else { - log.Printf("internal error serving %#+v: %s", p, err) - } - http.Error(w, err.Error(), code) - return - } - - if deadlineHit { - log15.Info("Deadline hit") - http.Error(w, "Deadline hit", http.StatusRequestTimeout) - return - } -} - -func (s *Service) replace(ctx context.Context, p *protocol.Request, w http.ResponseWriter, r *http.Request) (deadlineHit bool, err error) { - tr := nettrace.New("replace", fmt.Sprintf("%s@%s", p.Repo, p.Commit)) - tr.LazyPrintf("%s", p.RewriteSpecification) - - span, ctx := ot.StartSpanFromContext(ctx, "Replace") - ext.Component.Set(span, "service") - span.SetTag("repo", p.Repo) - span.SetTag("url", p.URL) - span.SetTag("commit", p.Commit) - span.SetTag("rewriteSpecification", p.RewriteSpecification) - defer func(start time.Time) { - code := "200" - // We often have canceled and timed out requests. We do not want to - // record them as errors to avoid noise - if ctx.Err() == context.Canceled { - code = "canceled" - span.SetTag("err", err) - } else if ctx.Err() == context.DeadlineExceeded { - code = "timedout" - span.SetTag("err", err) - deadlineHit = true - err = nil // error is fully described by deadlineHit=true return value - } else if err != nil { - tr.LazyPrintf("error: %v", err) - tr.SetError() - ext.Error.Set(span, true) - span.SetTag("err", err.Error()) - if isBadRequest(err) { - code = "400" - } else if isTemporary(err) { - code = "503" - } else { - code = "500" - } - } - tr.LazyPrintf("code=%s deadlineHit=%v", code, deadlineHit) - tr.Finish() - requestTotal.WithLabelValues(code).Inc() - span.SetTag("deadlineHit", deadlineHit) - span.Finish() - - if s.Log != nil { - s.Log.Debug("replace request", "repo", p.Repo, "commit", p.Commit, "rewriteSpecification", p.RewriteSpecification, "code", code, "duration", time.Since(start), "err", err) - } - }(time.Now()) - - if p.FetchTimeout == "" { - p.FetchTimeout = "500ms" - } - fetchTimeout, err := time.ParseDuration(p.FetchTimeout) - if err != nil { - return false, err - } - prepareCtx, cancel := context.WithTimeout(ctx, fetchTimeout) - defer cancel() - - getZf := func() (string, *store.ZipFile, error) { - path, err := s.Store.PrepareZip(prepareCtx, p.GitserverRepo(), p.Commit) - if err != nil { - return "", nil, err - } - zf, err := s.Store.ZipCache.Get(path) - return path, zf, err - } - - zipPath, zf, err := store.GetZipFileWithRetry(getZf) - if err != nil { - return false, errors.Wrap(err, "failed to get archive") - } - defer zf.Close() - - nFiles := uint64(len(zf.Files)) - bytes := int64(len(zf.Data)) - tr.LazyPrintf("files=%d bytes=%d", nFiles, bytes) - span.LogFields( - otlog.Uint64("archive.files", nFiles), - otlog.Int64("archive.size", bytes)) - archiveFiles.Observe(float64(nFiles)) - archiveSize.Observe(float64(bytes)) - - w.Header().Set("Transfer-Encoding", "chunked") - w.WriteHeader(http.StatusOK) - - t := &ExternalTool{ - Name: "comby", - BinaryPath: "comby", - } - - cmd, err := t.command(ctx, &p.RewriteSpecification, zipPath) - if err != nil { - log15.Info("Invalid command: " + err.Error()) - return false, errors.Wrap(err, "invalid command") - } - - stdout, err := cmd.StdoutPipe() - if err != nil { - log15.Info("Could not connect to command stdout: " + err.Error()) - return false, errors.Wrap(err, "failed to connect to command stdout") - } - - if err := cmd.Start(); err != nil { - log15.Info("Error starting command: " + err.Error()) - return false, errors.Wrap(err, "failed to start command") - } - - _, err = io.Copy(w, stdout) - if err != nil { - log15.Info("Error copying external command output to HTTP writer: " + err.Error()) - return false, errors.Wrap(err, "failed while copying command output to HTTP") - } - - if err := cmd.Wait(); err != nil { - log15.Info("Error after executing command: " + string(err.(*exec.ExitError).Stderr)) - } - - return false, nil -} - -func validateParams(p *protocol.Request) error { - if p.Repo == "" { - return errors.New("Repo must be non-empty") - } - // Surprisingly this is the same sanity check used in the git source. - if len(p.Commit) != 40 { - return errors.Errorf("Commit must be resolved (Commit=%q)", p.Commit) - } - - if p.RewriteSpecification.MatchTemplate == "" { - return errors.New("MatchTemplate must be non-empty") - } - return nil -} - -const megabyte = float64(1000 * 1000) - -var ( - running = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "replacer_service_running", - Help: "Number of running search requests.", - }) - archiveSize = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "replacer_service_archive_size_bytes", - Help: "Observes the size when an archive is searched.", - Buckets: []float64{1 * megabyte, 10 * megabyte, 100 * megabyte, 500 * megabyte, 1000 * megabyte, 5000 * megabyte}, - }) - archiveFiles = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "replacer_service_archive_files", - Help: "Observes the number of files when an archive is searched.", - Buckets: []float64{100, 1000, 10000, 50000, 100000}, - }) - requestTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "replacer_service_request_total", - Help: "Number of returned replace requests.", - }, []string{"code"}) -) - -func init() { - prometheus.MustRegister(running) - prometheus.MustRegister(archiveSize) - prometheus.MustRegister(archiveFiles) - prometheus.MustRegister(requestTotal) -} - -func isBadRequest(err error) bool { - e, ok := errors.Cause(err).(interface { - BadRequest() bool - }) - return ok && e.BadRequest() -} - -func isTemporary(err error) bool { - e, ok := errors.Cause(err).(interface { - Temporary() bool - }) - return ok && e.Temporary() -} diff --git a/cmd/replacer/replace/replace_test.go b/cmd/replacer/replace/replace_test.go deleted file mode 100644 index 65ce319d8ed..00000000000 --- a/cmd/replacer/replace/replace_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package replace_test - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "os" - "os/exec" - "strings" - "testing" - - "github.com/sourcegraph/sourcegraph/cmd/replacer/protocol" - "github.com/sourcegraph/sourcegraph/cmd/replacer/replace" - "github.com/sourcegraph/sourcegraph/internal/testutil" -) - -func TestReplace(t *testing.T) { - // If we are not on CI skip the test if comby is not installed. - if _, err := exec.LookPath("comby"); os.Getenv("CI") == "" && err != nil { - t.Skip("comby is not installed on the PATH. Try running 'bash <(curl -sL get.comby.dev)'.") - } - - files := map[string]string{ - - "README.md": `# Hello World - -Hello world example in go`, - "main.go": `package main - -import "fmt" - -func main() { - fmt.Println("Hello foo") -} -`, - } - - cases := []struct { - arg protocol.RewriteSpecification - want string - }{ - {protocol.RewriteSpecification{ - MatchTemplate: "func", - RewriteTemplate: "derp", - FileExtension: ".go", - }, ` -{"uri":"main.go","diff":"--- main.go\n+++ main.go\n@@ -2,6 +2,6 @@\n \n import \"fmt\"\n \n-func main() {\n+derp main() {\n \tfmt.Println(\"Hello foo\")\n }"} -`}, - } - - store, cleanup, err := testutil.NewStore(files) - if err != nil { - t.Fatal(err) - } - defer cleanup() - ts := httptest.NewServer(&replace.Service{Store: store}) - defer ts.Close() - - for _, test := range cases { - req := protocol.Request{ - Repo: "foo", - URL: "u", - Commit: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - RewriteSpecification: test.arg, - FetchTimeout: "5000ms", - } - got, err := doReplace(ts.URL, &req) - if err != nil { - t.Errorf("%v failed: %s", test.arg, err) - continue - } - - // We have an extra newline to make expected readable - if len(test.want) > 0 { - test.want = test.want[1:] - } - - if got != test.want { - d, err := testutil.Diff(test.want, got) - if err != nil { - t.Fatal(err) - } - t.Errorf("%v unexpected response:\n%s", test.arg, d) - continue - } - - } -} - -func TestReplace_badrequest(t *testing.T) { - cases := []protocol.Request{ - { - Repo: "foo", - URL: "u", - Commit: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - // No MatchTemplate - }, - } - - store, cleanup, err := testutil.NewStore(nil) - if err != nil { - t.Fatal(err) - } - defer cleanup() - ts := httptest.NewServer(&replace.Service{Store: store}) - defer ts.Close() - - for _, p := range cases { - _, err := doReplace(ts.URL, &p) - if err == nil { - t.Fatalf("%v expected to fail", p) - } - if !strings.HasPrefix(err.Error(), "non-200 response: code=400 ") { - t.Fatalf("%v expected to have HTTP 400 response. Got %s", p, err) - } - } -} - -func doReplace(u string, p *protocol.Request) (string, error) { - form := url.Values{ - "Repo": []string{string(p.Repo)}, - "URL": []string{string(p.URL)}, - "Commit": []string{string(p.Commit)}, - "FetchTimeout": []string{p.FetchTimeout}, - "MatchTemplate": []string{p.RewriteSpecification.MatchTemplate}, - "RewriteTemplate": []string{p.RewriteSpecification.RewriteTemplate}, - "FileExtension": []string{p.RewriteSpecification.FileExtension}, - } - resp, err := http.PostForm(u, form) - if err != nil { - return "", err - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if resp.StatusCode != 200 { - return "", fmt.Errorf("non-200 response: code=%d body=%s", resp.StatusCode, string(body)) - } - - return string(body), err -} diff --git a/cmd/server/build.sh b/cmd/server/build.sh index 9b8f89fe97c..ab42b18b0cb 100755 --- a/cmd/server/build.sh +++ b/cmd/server/build.sh @@ -61,7 +61,6 @@ PACKAGES=( github.com/sourcegraph/sourcegraph/cmd/github-proxy github.com/sourcegraph/sourcegraph/cmd/gitserver github.com/sourcegraph/sourcegraph/cmd/query-runner - github.com/sourcegraph/sourcegraph/cmd/replacer github.com/sourcegraph/sourcegraph/cmd/searcher github.com/sourcegraph/sourcegraph/cmd/symbols github.com/google/zoekt/cmd/zoekt-archive-index diff --git a/cmd/server/shared/globals.go b/cmd/server/shared/globals.go index d9dc7ed026f..5511f516dd6 100644 --- a/cmd/server/shared/globals.go +++ b/cmd/server/shared/globals.go @@ -16,7 +16,6 @@ var SrcProfServices = []map[string]string{ {"Name": "symbols", "Host": "127.0.0.1:6071"}, {"Name": "repo-updater", "Host": "127.0.0.1:6074"}, {"Name": "query-runner", "Host": "127.0.0.1:6067"}, - {"Name": "replacer", "Host": "127.0.0.1:6076"}, {"Name": "zoekt-indexserver", "Host": "127.0.0.1:6072"}, {"Name": "zoekt-webserver", "Host": "127.0.0.1:3070", "DefaultPath": "/debug/requests/"}, } diff --git a/cmd/server/shared/shared.go b/cmd/server/shared/shared.go index 15c73a1ceca..60aeaf5cb15 100644 --- a/cmd/server/shared/shared.go +++ b/cmd/server/shared/shared.go @@ -33,7 +33,6 @@ var DefaultEnv = map[string]string{ "QUERY_RUNNER_URL": "http://127.0.0.1:3183", "SRC_SYNTECT_SERVER": "http://127.0.0.1:9238", "SYMBOLS_URL": "http://127.0.0.1:3184", - "REPLACER_URL": "http://127.0.0.1:3185", "SRC_HTTP_ADDR": ":8080", "SRC_HTTPS_ADDR": ":8443", "SRC_FRONTEND_INTERNAL": FrontendInternalHost, @@ -46,7 +45,6 @@ var DefaultEnv = map[string]string{ // searcher/symbols to ensure this value isn't larger than the volume for // CACHE_DIR. "SEARCHER_CACHE_SIZE_MB": "50000", - "REPLACER_CACHE_SIZE_MB": "50000", "SYMBOLS_CACHE_SIZE_MB": "50000", // Used to differentiate between deployments on dev, Docker, and Kubernetes. @@ -138,7 +136,6 @@ func Main() { `query-runner: query-runner`, `symbols: symbols`, `searcher: searcher`, - `replacer: replacer`, `github-proxy: github-proxy`, `repo-updater: repo-updater`, `syntect_server: sh -c 'env QUIET=true ROCKET_ENV=production ROCKET_PORT=9238 ROCKET_LIMITS='"'"'{json=10485760}'"'"' ROCKET_SECRET_KEY='"'"'SeerutKeyIsI7releuantAndknvsuZPluaseIgnorYA='"'"' ROCKET_KEEP_ALIVE=0 ROCKET_ADDRESS='"'"'"127.0.0.1"'"'"' syntect_server | grep -v "Rocket has launched" | grep -v "Warning: environment is"' | grep -v 'Configured for production'`, diff --git a/dev/Procfile b/dev/Procfile index eb83acdc998..777b9bf549c 100644 --- a/dev/Procfile +++ b/dev/Procfile @@ -2,7 +2,6 @@ gitserver: gitserver query-runner: query-runner repo-updater: repo-updater searcher: searcher -replacer: replacer symbols: symbols github-proxy: github-proxy frontend: env CONFIGURATION_MODE=server SITE_CONFIG_ESCAPE_HATCH_PATH=$HOME/.sourcegraph/site-config.json frontend diff --git a/dev/ci/go-test.sh b/dev/ci/go-test.sh index b576c7a03b8..cdb41eeeeb9 100755 --- a/dev/ci/go-test.sh +++ b/dev/ci/go-test.sh @@ -6,7 +6,7 @@ set -e echo "--- build libsqlite" ./dev/libsqlite3-pcre/build.sh -# For searcher and replacer tests +# For searcher echo "--- comby install" ./dev/comby-install-or-upgrade.sh diff --git a/dev/comby-install-or-upgrade.sh b/dev/comby-install-or-upgrade.sh index 8b07ed1cc1f..a6bca979f13 100755 --- a/dev/comby-install-or-upgrade.sh +++ b/dev/comby-install-or-upgrade.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# This function installs the comby dependency for cmd/searcher and -# cmd/replacer. The CI pipeline calls this script to install or upgrade comby +# This function installs the comby dependency for cmd/searcher. +# The CI pipeline calls this script to install or upgrade comby # for tests or development environments. REQUIRE_VERSION="0.14.1" diff --git a/dev/go-install.sh b/dev/go-install.sh index 684dd39e4cc..22593d57f48 100755 --- a/dev/go-install.sh +++ b/dev/go-install.sh @@ -6,7 +6,7 @@ # This will install binaries into the `.bin` directory under the repository root. # -all_oss_commands=" gitserver query-runner github-proxy searcher replacer frontend repo-updater symbols " +all_oss_commands=" gitserver query-runner github-proxy searcher frontend repo-updater symbols " all_commands="${all_oss_commands}${ENTERPRISE_ONLY_COMMANDS}" # handle options diff --git a/dev/prometheus/all/prometheus_targets.yml b/dev/prometheus/all/prometheus_targets.yml index 2d448940b89..cf8ac45e74a 100644 --- a/dev/prometheus/all/prometheus_targets.yml +++ b/dev/prometheus/all/prometheus_targets.yml @@ -54,8 +54,3 @@ targets: # postgres exporter - host.docker.internal:9187 -- labels: - job: replacer - targets: - # replacer - - host.docker.internal:6076 diff --git a/dev/prometheus/linux/prometheus_targets.yml b/dev/prometheus/linux/prometheus_targets.yml index 33465c32a0f..c706037bb33 100644 --- a/dev/prometheus/linux/prometheus_targets.yml +++ b/dev/prometheus/linux/prometheus_targets.yml @@ -54,8 +54,3 @@ targets: # postgres exporter - 127.0.0.1:9187 -- labels: - job: replacer - targets: - # replacer - - 127.0.0.1:6076 diff --git a/dev/src-prof-services.json b/dev/src-prof-services.json index 6f53a14f5b7..4e4ba1632f9 100644 --- a/dev/src-prof-services.json +++ b/dev/src-prof-services.json @@ -5,7 +5,6 @@ { "Name": "symbols", "Host": "127.0.0.1:6071" }, { "Name": "repo-updater", "Host": "127.0.0.1:6074" }, { "Name": "query-runner", "Host": "127.0.0.1:6067" }, - { "Name": "replacer", "Host": "127.0.0.1:6076" }, { "Name": "precise-code-intel-bundle-manager", "Host": "127.0.0.1:6087" }, { "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" }, { "Name": "precise-code-intel-indexer", "Host": "127.0.0.1:6089" }, diff --git a/dev/start.sh b/dev/start.sh index 25e784784a6..8542b2dffcc 100755 --- a/dev/start.sh +++ b/dev/start.sh @@ -63,7 +63,6 @@ export INSECURE_DEV=1 export SRC_GIT_SERVERS=127.0.0.1:3178 export GOLANGSERVER_SRC_GIT_SERVERS=host.docker.internal:3178 export SEARCHER_URL=http://127.0.0.1:3181 -export REPLACER_URL=http://127.0.0.1:3185 export REPO_UPDATER_URL=http://127.0.0.1:3182 export REDIS_ENDPOINT=127.0.0.1:6379 export QUERY_RUNNER_URL=http://localhost:3183 diff --git a/doc/admin/observability/alert_solutions.md b/doc/admin/observability/alert_solutions.md index 677700f2be8..c08a533b6ee 100644 --- a/doc/admin/observability/alert_solutions.md +++ b/doc/admin/observability/alert_solutions.md @@ -2850,229 +2850,6 @@ To learn more about Sourcegraph's alerting, see [our alerting documentation](htt ] ``` -## replacer: frontend_internal_api_error_responses - -**Descriptions:** - -- _replacer: 5+ frontend-internal API error responses every 5m by route_ - -**Possible solutions:** - -- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing. -- **Kubernetes:** - - Confirm that `kubectl get pods` shows the `frontend` pods are healthy. - - Check `kubectl logs replacer` for logs indicate request failures to `frontend` or `frontend-internal`. -- **Docker Compose:** - - Confirm that `docker ps` shows the `frontend-internal` container is healthy. - - Check `docker logs replacer` for logs indicating request failures to `frontend` or `frontend-internal`. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_frontend_internal_api_error_responses" -] -``` - -## replacer: container_cpu_usage - -**Descriptions:** - -- _replacer: 99%+ container cpu usage total (1m average) across all cores by instance_ - -**Possible solutions:** - -- **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`. -- **Docker Compose:** Consider increasing `cpus:` of the replacer container in `docker-compose.yml`. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_container_cpu_usage" -] -``` - -## replacer: container_memory_usage - -**Descriptions:** - -- _replacer: 99%+ container memory usage by instance_ - -**Possible solutions:** - -- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. -- **Docker Compose:** Consider increasing `memory:` of replacer container in `docker-compose.yml`. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_container_memory_usage" -] -``` - -## replacer: container_restarts - -**Descriptions:** - -- _replacer: 1+ container restarts every 5m by instance_ - -**Possible solutions:** - -- **Kubernetes:** - - Determine if the pod was OOM killed using `kubectl describe pod replacer` (look for `OOMKilled: true`) and, if so, consider increasing the memory limit in the relevant `Deployment.yaml`. - - Check the logs before the container restarted to see if there are `panic:` messages or similar using `kubectl logs -p replacer`. -- **Docker Compose:** - - Determine if the pod was OOM killed using `docker inspect -f '{{json .State}}' replacer` (look for `"OOMKilled":true`) and, if so, consider increasing the memory limit of the replacer container in `docker-compose.yml`. - - Check the logs before the container restarted to see if there are `panic:` messages or similar using `docker logs replacer` (note this will include logs from the previous and currently running container). -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_container_restarts" -] -``` - -## replacer: fs_inodes_used - -**Descriptions:** - -- _replacer: 3e+06+ fs inodes in use by instance_ - -**Possible solutions:** - -- - Refer to your OS or cloud provider`s documentation for how to increase inodes. - - **Kubernetes:** consider provisioning more machines with less resources. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_fs_inodes_used" -] -``` - -## replacer: provisioning_container_cpu_usage_7d - -**Descriptions:** - -- _replacer: 80%+ or less than 30% container cpu usage total (7d maximum) across all cores by instance_ - -**Possible solutions:** - -- If usage is high: - - **Kubernetes:** Consider decreasing CPU limits in the the relevant `Deployment.yaml`. - - **Docker Compose:** Consider descreasing `cpus:` of the replacer container in `docker-compose.yml`. -- If usage is low, consider decreasing the above values. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_provisioning_container_cpu_usage_7d" -] -``` - -## replacer: provisioning_container_memory_usage_7d - -**Descriptions:** - -- _replacer: 80%+ or less than 30% container memory usage (7d maximum) by instance_ - -**Possible solutions:** - -- If usage is high: - - **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. - - **Docker Compose:** Consider increasing `memory:` of replacer container in `docker-compose.yml`. -- If usage is low, consider decreasing the above values. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_provisioning_container_memory_usage_7d" -] -``` - -## replacer: provisioning_container_cpu_usage_5m - -**Descriptions:** - -- _replacer: 90%+ container cpu usage total (5m maximum) across all cores by instance_ - -**Possible solutions:** - -- **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`. -- **Docker Compose:** Consider increasing `cpus:` of the replacer container in `docker-compose.yml`. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_provisioning_container_cpu_usage_5m" -] -``` - -## replacer: provisioning_container_memory_usage_5m - -**Descriptions:** - -- _replacer: 90%+ container memory usage (5m maximum) by instance_ - -**Possible solutions:** - -- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`. -- **Docker Compose:** Consider increasing `memory:` of replacer container in `docker-compose.yml`. -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_provisioning_container_memory_usage_5m" -] -``` - -## replacer: go_goroutines - -**Descriptions:** - -- _replacer: 10000+ maximum active goroutines for 10m_ - -**Possible solutions:** - -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_go_goroutines" -] -``` - -## replacer: go_gc_duration_seconds - -**Descriptions:** - -- _replacer: 2s+ maximum go garbage collection duration_ - -**Possible solutions:** - -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "warning_replacer_go_gc_duration_seconds" -] -``` - -## replacer: pods_available_percentage - -**Descriptions:** - -- _replacer: less than 90% percentage pods available for 10m_ - -**Possible solutions:** - -- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: - -```json -"observability.silenceAlerts": [ - "critical_replacer_pods_available_percentage" -] -``` - ## repo-updater: frontend_internal_api_error_responses **Descriptions:** diff --git a/doc/admin/observability/metrics_guide.md b/doc/admin/observability/metrics_guide.md index c9263e6c537..e560a839f78 100644 --- a/doc/admin/observability/metrics_guide.md +++ b/doc/admin/observability/metrics_guide.md @@ -32,7 +32,6 @@ To set up notifications for these alerts, see: [alerting](alerting.md). - `"gitserver"` - `"precise-code-intel"` - `"query-runner"` - - `"replacer"` - `"repo-updater"` - `"searcher"` - `"symbols"` diff --git a/doc/admin/pprof.md b/doc/admin/pprof.md index b154fd56181..d31d41f732f 100644 --- a/doc/admin/pprof.md +++ b/doc/admin/pprof.md @@ -75,7 +75,6 @@ This is a table of Sourcegraph backend debug ports in the two deployment context | query-runner | 6060 | 6067 | | zoekt-indexserver | 6060 | 6072 | | zoekt-webserver | 6060 | 3070 | -| replacer | 6060 | 6076 | ## Profiling kinds diff --git a/doc/dev/architecture/architecture.dot b/doc/dev/architecture/architecture.dot index abd926ce8fa..c7a2cd827b8 100644 --- a/doc/dev/architecture/architecture.dot +++ b/doc/dev/architecture/architecture.dot @@ -61,12 +61,6 @@ digraph architecture { shape="box3d" URL="https://github.com/sourcegraph/sourcegraph/tree/master/cmd/searcher" ] - replacer [ - label="replacer" - fillcolor="5" - shape="box3d" - URL="https://github.com/sourcegraph/sourcegraph/tree/master/cmd/replacer" - ] query_runner [ label="query\nrunner" fillcolor="6" @@ -152,7 +146,6 @@ digraph architecture { gitserver query_runner searcher - replacer repo_updater github_proxy zoekt_webserver @@ -173,13 +166,6 @@ digraph architecture { fillcolor="4" ] - replacer -> { - frontend - gitserver - } [ - fillcolor="5" - ] - query_runner -> frontend [fillcolor="6"] symbols -> { diff --git a/doc/dev/architecture/architecture.svg b/doc/dev/architecture/architecture.svg index f47e8ebc8d1..6e87fb6752f 100644 --- a/doc/dev/architecture/architecture.svg +++ b/doc/dev/architecture/architecture.svg @@ -99,24 +99,6 @@ - - -replacer - - - - - -replacer - - - - - -frontend->replacer - - - query_runner @@ -380,18 +362,6 @@ - - -replacer->frontend - - - - - -replacer->gitserver - - - query_runner->frontend diff --git a/doc/dev/architecture/search-pagination.md b/doc/dev/architecture/search-pagination.md index 624a2292f8f..4f1edfa0fdd 100644 --- a/doc/dev/architecture/search-pagination.md +++ b/doc/dev/architecture/search-pagination.md @@ -29,7 +29,6 @@ We use the term "search backend" to describe a function which performs a search - `searchFilesInRepos` for finding `type:file` (text) and `type:path` (filepath) results. - `searchCommitDiffsInRepos` for finding `type:diff` results. - `searchCommitLogInRepos` for finding `type:commit` results. -- `performCodemod` for code modification find/replace (read-only) results. ### "cursor" diff --git a/enterprise/dev/Procfile b/enterprise/dev/Procfile index 4bbc090ce20..c10faa92e0b 100644 --- a/enterprise/dev/Procfile +++ b/enterprise/dev/Procfile @@ -2,7 +2,6 @@ gitserver: gitserver query-runner: query-runner repo-updater: repo-updater searcher: searcher -replacer: replacer symbols: symbols github-proxy: github-proxy frontend: env CONFIGURATION_MODE=server SITE_CONFIG_ESCAPE_HATCH_PATH=$HOME/.sourcegraph/site-config.json frontend diff --git a/enterprise/dev/ci/ci/pipeline-steps.go b/enterprise/dev/ci/ci/pipeline-steps.go index cf237e06ce4..9bea4df5ab4 100644 --- a/enterprise/dev/ci/ci/pipeline-steps.go +++ b/enterprise/dev/ci/ci/pipeline-steps.go @@ -14,7 +14,6 @@ var allDockerImages = []string{ "github-proxy", "gitserver", "query-runner", - "replacer", "repo-updater", "searcher", "server", diff --git a/enterprise/dev/src-prof-services.json b/enterprise/dev/src-prof-services.json index 3119072bcc5..1470bc40963 100644 --- a/enterprise/dev/src-prof-services.json +++ b/enterprise/dev/src-prof-services.json @@ -5,7 +5,6 @@ { "Name": "symbols", "Host": "127.0.0.1:6071" }, { "Name": "repo-updater", "Host": "127.0.0.1:6074" }, { "Name": "query-runner", "Host": "127.0.0.1:6067" }, - { "Name": "replacer", "Host": "127.0.0.1:6076" }, { "Name": "precise-code-intel-bundle-manager", "Host": "127.0.0.1:6087" }, { "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" } ] diff --git a/internal/processrestart/goreman_dev.go b/internal/processrestart/goreman_dev.go index 70d88f460e0..048754ce7f6 100644 --- a/internal/processrestart/goreman_dev.go +++ b/internal/processrestart/goreman_dev.go @@ -31,7 +31,6 @@ func restartGoremanDev() error { "query-runner", "repo-updater", "searcher", - "replacer", "symbols", "github-proxy", "syntect_server", diff --git a/internal/search/query/fields.go b/internal/search/query/fields.go index f82e70348b0..f12fd763b8c 100644 --- a/internal/search/query/fields.go +++ b/internal/search/query/fields.go @@ -35,6 +35,5 @@ var allFields = map[string]struct{}{ FieldStable: empty, FieldMax: empty, FieldTimeout: empty, - FieldReplace: empty, FieldCombyRule: empty, } diff --git a/internal/search/query/searchquery.go b/internal/search/query/searchquery.go index acd6be63a3d..ce2a1302507 100644 --- a/internal/search/query/searchquery.go +++ b/internal/search/query/searchquery.go @@ -40,7 +40,6 @@ const ( FieldStable = "stable" // Forces search to return a stable result ordering (currently limited to file content matches). FieldMax = "max" // Deprecated alias for count FieldTimeout = "timeout" - FieldReplace = "replace" FieldCombyRule = "rule" ) @@ -78,7 +77,6 @@ var ( FieldStable: {Literal: types.BoolType, Quoted: types.BoolType, Singular: true}, FieldMax: {Literal: types.StringType, Quoted: types.StringType, Singular: true}, FieldTimeout: {Literal: types.StringType, Quoted: types.StringType, Singular: true}, - FieldReplace: {Literal: types.StringType, Quoted: types.StringType, Singular: true}, FieldCombyRule: {Literal: types.StringType, Quoted: types.StringType, Singular: true}, }, FieldAliases: map[string]string{ diff --git a/internal/search/query/types.go b/internal/search/query/types.go index 0f3f66bd0ef..286644e641b 100644 --- a/internal/search/query/types.go +++ b/internal/search/query/types.go @@ -251,7 +251,6 @@ func (q AndOrQuery) valueToTypedValue(field, value string, label labels) []*type FieldCount, FieldMax, FieldTimeout, - FieldReplace, FieldCombyRule: return []*types.Value{{String: &value}} } diff --git a/internal/search/query/validate.go b/internal/search/query/validate.go index d262bc34678..9e6601f89b8 100644 --- a/internal/search/query/validate.go +++ b/internal/search/query/validate.go @@ -315,7 +315,6 @@ func validateField(field, value string, negated bool, seen map[string]struct{}) case FieldMax, FieldTimeout, - FieldReplace, FieldCombyRule: return satisfies(isSingular, isNotNegated) default: diff --git a/monitoring/generator.go b/monitoring/generator.go index 4379526d5ce..d9591be1571 100644 --- a/monitoring/generator.go +++ b/monitoring/generator.go @@ -905,7 +905,6 @@ func main() { PreciseCodeIntelWorker(), PreciseCodeIntelIndexer(), QueryRunner(), - Replacer(), RepoUpdater(), Searcher(), Symbols(), diff --git a/monitoring/replacer.go b/monitoring/replacer.go deleted file mode 100644 index 630d57a895b..00000000000 --- a/monitoring/replacer.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -func Replacer() *Container { - return &Container{ - Name: "replacer", - Title: "Replacer", - Description: "Backend for find-and-replace operations.", - Groups: []Group{ - { - Title: "General", - Rows: []Row{ - { - sharedFrontendInternalAPIErrorResponses("replacer"), - }, - }, - }, - { - Title: "Container monitoring (not available on server)", - Hidden: true, - Rows: []Row{ - { - sharedContainerCPUUsage("replacer"), - sharedContainerMemoryUsage("replacer"), - }, - { - sharedContainerRestarts("replacer"), - sharedContainerFsInodes("replacer"), - }, - }, - }, - { - Title: "Provisioning indicators (not available on server)", - Hidden: true, - Rows: []Row{ - { - sharedProvisioningCPUUsage7d("replacer"), - sharedProvisioningMemoryUsage7d("replacer"), - }, - { - sharedProvisioningCPUUsage5m("replacer"), - sharedProvisioningMemoryUsage5m("replacer"), - }, - }, - }, - { - Title: "Golang runtime monitoring", - Hidden: true, - Rows: []Row{ - { - sharedGoGoroutines("replacer"), - sharedGoGcDuration("replacer"), - }, - }, - }, - { - Title: "Kubernetes monitoring (ignore if using Docker Compose or server)", - Hidden: true, - Rows: []Row{ - { - sharedKubernetesPodsAvailable("replacer"), - }, - }, - }, - }, - } -} diff --git a/web/src/enterprise/codemod/index.ts b/web/src/enterprise/codemod/index.ts deleted file mode 100644 index f69efeac617..00000000000 --- a/web/src/enterprise/codemod/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Whether the experimental code modification feature is enabled. - * - * To enable this, run `localStorage.codemodExp=true;location.reload()` in your browser's JavaScript - * console. - */ -export const USE_CODEMOD = localStorage.getItem('codemodExp') !== null diff --git a/web/src/search/backend.tsx b/web/src/search/backend.tsx index a6ef59bdfd3..25cfd28dd71 100644 --- a/web/src/search/backend.tsx +++ b/web/src/search/backend.tsx @@ -5,7 +5,6 @@ import * as GQL from '../../../shared/src/graphql/schema' import { asError, createAggregateError, ErrorLike } from '../../../shared/src/util/errors' import { memoizeObservable } from '../../../shared/src/util/memoizeObservable' import { mutateGraphQL, queryGraphQL } from '../backend/graphql' -import { USE_CODEMOD } from '../enterprise/codemod' import { SearchSuggestion } from '../../../shared/src/search/suggestions' import { Remote } from 'comlink' import { FlatExtHostAPI } from '../../../shared/src/api/contract' @@ -31,7 +30,6 @@ export function search( $query: String! $version: SearchVersion! $patternType: SearchPatternType! - $useCodemod: Boolean! $versionContext: String ) { search( @@ -163,31 +161,6 @@ export function search( } # end of genericSearchResultInterfaceFields inline fragment } - ... on CodemodResult @include(if: $useCodemod) { - # TODO: Make this a proper fragment, blocked by https://github.com/graph-gophers/graphql-go/issues/241. - # beginning of genericSearchResultInterfaceFields inline fragment - label { - html - } - url - icon - detail { - html - } - matches { - url - body { - text - html - } - highlights { - line - character - length - } - } - # end of genericSearchResultInterfaceFields inline fragment - } } alert { title @@ -202,7 +175,7 @@ export function search( } } `, - { query, version, patternType, versionContext, useCodemod: USE_CODEMOD } + { query, version, patternType, versionContext } ).pipe( map(({ data, errors }) => { if (!data || !data.search || !data.search.results) {