diff --git a/.prettierignore b/.prettierignore index 560460d78b0..4b868a39979 100644 --- a/.prettierignore +++ b/.prettierignore @@ -31,3 +31,4 @@ storybook-static/ browser/code-intel-extensions/ .buildkite-cache/ lib/codeintel/reprolang/ +cmd/symbols/squirrel/language-file-extensions.json diff --git a/cmd/frontend/graphqlbackend/codeintel.graphql b/cmd/frontend/graphqlbackend/codeintel.graphql index b8bcc20eb5e..dd29eb3e939 100644 --- a/cmd/frontend/graphqlbackend/codeintel.graphql +++ b/cmd/frontend/graphqlbackend/codeintel.graphql @@ -558,6 +558,13 @@ extend type GitBlob { Provides info on the level of code-intel support for this git blob. """ codeIntelSupport: CodeIntelSupport! + + """ + Provides code intelligence within the file. + + Experimental: This API is likely to change in the future. + """ + localCodeIntel: JSONValue } """ diff --git a/cmd/frontend/graphqlbackend/git_tree_entry.go b/cmd/frontend/graphqlbackend/git_tree_entry.go index 20eed1e70c1..f3e258954d8 100644 --- a/cmd/frontend/graphqlbackend/git_tree_entry.go +++ b/cmd/frontend/graphqlbackend/git_tree_entry.go @@ -2,6 +2,7 @@ package graphqlbackend import ( "context" + "encoding/json" "io/fs" "net/url" "os" @@ -21,8 +22,10 @@ import ( "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/authz" "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/symbols" "github.com/sourcegraph/sourcegraph/internal/trace" "github.com/sourcegraph/sourcegraph/internal/trace/ot" + "github.com/sourcegraph/sourcegraph/internal/types" "github.com/sourcegraph/sourcegraph/internal/vcs/git" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -283,6 +286,29 @@ func (r *GitTreeEntryResolver) CodeIntelInfo(ctx context.Context) (GitTreeCodeIn }) } +func (r *GitTreeEntryResolver) LocalCodeIntel(ctx context.Context) (*JSONValue, error) { + repo, err := r.commit.repoResolver.repo(ctx) + if err != nil { + return nil, err + } + + payload, err := symbols.DefaultClient.LocalCodeIntel(ctx, types.RepoCommitPath{ + Repo: string(repo.Name), + Commit: string(r.commit.oid), + Path: r.Path(), + }) + if err != nil { + return nil, err + } + + jsonValue, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + return &JSONValue{Value: string(jsonValue)}, nil +} + type fileInfo struct { path string size int64 diff --git a/cmd/symbols/internal/api/handler.go b/cmd/symbols/internal/api/handler.go index f5133256b1c..9e234f1b84b 100644 --- a/cmd/symbols/internal/api/handler.go +++ b/cmd/symbols/internal/api/handler.go @@ -9,6 +9,7 @@ import ( "github.com/sourcegraph/go-ctags" + "github.com/sourcegraph/sourcegraph/cmd/symbols/squirrel" "github.com/sourcegraph/sourcegraph/cmd/symbols/types" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -22,6 +23,8 @@ func NewHandler( mux.HandleFunc("/search", handleSearchWith(searchFunc)) mux.HandleFunc("/healthz", handleHealthCheck) mux.HandleFunc("/list-languages", handleListLanguages(ctagsBinary)) + mux.HandleFunc("/localCodeIntel", squirrel.LocalCodeIntelHandler) + mux.HandleFunc("/debugLocalCodeIntel", squirrel.DebugLocalCodeIntelHandler) if handleStatus != nil { mux.HandleFunc("/status", handleStatus) } diff --git a/cmd/symbols/squirrel/README.md b/cmd/symbols/squirrel/README.md new file mode 100644 index 00000000000..676ca44041e --- /dev/null +++ b/cmd/symbols/squirrel/README.md @@ -0,0 +1,3 @@ +# Squirrel + +Squirrel is an HTTP server for fast and precise local code intelligence using tree-sitter. diff --git a/cmd/symbols/squirrel/breadcrumbs.go b/cmd/symbols/squirrel/breadcrumbs.go new file mode 100644 index 00000000000..0a80379cbe0 --- /dev/null +++ b/cmd/symbols/squirrel/breadcrumbs.go @@ -0,0 +1,134 @@ +package squirrel + +import ( + "context" + "fmt" + "strings" + + "github.com/fatih/color" + + "github.com/sourcegraph/sourcegraph/internal/types" +) + +// Breadcrumb is an arbitrary annotation on a token in a file. It's used as a way to log where Squirrel +// has been traversing through trees and files for debugging. +type Breadcrumb struct { + types.RepoCommitPathRange + length int + message string +} + +// addBreadcrumb adds a breadcrumb to the given slice. +func addBreadcrumb(breadcrumbs *[]Breadcrumb, node Node, message string) { + *breadcrumbs = append(*breadcrumbs, Breadcrumb{ + RepoCommitPathRange: types.RepoCommitPathRange{ + RepoCommitPath: node.RepoCommitPath, + Range: nodeToRange(node.Node), + }, + length: nodeLength(node.Node), + message: message, + }) +} + +// Prints breadcrumbs like this: +// +// v some breadcrumb +// vvv other breadcrumb +// 78 | func f(f Foo) { +func prettyPrintBreadcrumbs(w *strings.Builder, breadcrumbs []Breadcrumb, readFile ReadFileFunc) { + // First collect all the breadcrumbs in a map (path -> line -> breadcrumb) for easier printing. + pathToLineToBreadcrumbs := map[types.RepoCommitPath]map[int][]Breadcrumb{} + for _, breadcrumb := range breadcrumbs { + path := breadcrumb.RepoCommitPath + + if _, ok := pathToLineToBreadcrumbs[path]; !ok { + pathToLineToBreadcrumbs[path] = map[int][]Breadcrumb{} + } + + pathToLineToBreadcrumbs[path][int(breadcrumb.Row)] = append(pathToLineToBreadcrumbs[path][int(breadcrumb.Row)], breadcrumb) + } + + // Loop over each path, printing the breadcrumbs for each line. + for repoCommitPath, lineToBreadcrumb := range pathToLineToBreadcrumbs { + // Print the path header. + blue := color.New(color.FgBlue).SprintFunc() + grey := color.New(color.FgBlack).SprintFunc() + fmt.Fprintf(w, blue("repo %s, commit %s, path %s"), repoCommitPath.Repo, repoCommitPath.Commit, repoCommitPath.Path) + fmt.Fprintln(w) + + // Read the file. + contents, err := readFile(context.Background(), repoCommitPath) + if err != nil { + fmt.Println("Error reading file: ", err) + return + } + + // Print the breadcrumbs for each line. + for lineNumber, line := range strings.Split(string(contents), "\n") { + breadcrumbs, ok := lineToBreadcrumb[lineNumber] + if !ok { + // No breadcrumbs on this line. + continue + } + + fmt.Fprintln(w) + + gutter := fmt.Sprintf("%5d | ", lineNumber) + + columnToMessage := map[int]string{} + for _, breadcrumb := range breadcrumbs { + for column := int(breadcrumb.Column); column < int(breadcrumb.Column)+breadcrumb.length; column++ { + columnToMessage[lengthInSpaces(line[:column])] = breadcrumb.message + } + + gutterPadding := strings.Repeat(" ", len(gutter)) + + space := strings.Repeat(" ", lengthInSpaces(line[:breadcrumb.Column])) + + arrows := messageColor(breadcrumb.message)(strings.Repeat("v", breadcrumb.length)) + + fmt.Fprintf(w, "%s%s%s %s\n", gutterPadding, space, arrows, messageColor(breadcrumb.message)(breadcrumb.message)) + } + + fmt.Fprint(w, grey(gutter)) + lineWithSpaces := strings.ReplaceAll(line, "\t", " ") + for c := 0; c < len(lineWithSpaces); c++ { + if message, ok := columnToMessage[c]; ok { + fmt.Fprint(w, messageColor(message)(string(lineWithSpaces[c]))) + } else { + fmt.Fprint(w, grey(string(lineWithSpaces[c]))) + } + } + fmt.Fprintln(w) + } + } +} + +// Returns breadcrumbs that have one of the given messages. +func pickBreadcrumbs(breadcrumbs []Breadcrumb, messages []string) []Breadcrumb { + var picked []Breadcrumb + for _, breadcrumb := range breadcrumbs { + for _, message := range messages { + if strings.Contains(breadcrumb.message, message) { + picked = append(picked, breadcrumb) + break + } + } + } + return picked +} + +// Returns the color to be used to print a message. +func messageColor(message string) colorSprintfFunc { + if message == "start" { + return color.New(color.FgHiCyan).SprintFunc() + } else if message == "found" { + return color.New(color.FgRed).SprintFunc() + } else if message == "correct" { + return color.New(color.FgGreen).SprintFunc() + } else if strings.Contains(message, "scope") { + return color.New(color.FgHiYellow).SprintFunc() + } else { + return color.New(color.FgHiMagenta).SprintFunc() + } +} diff --git a/cmd/symbols/squirrel/hover.go b/cmd/symbols/squirrel/hover.go new file mode 100644 index 00000000000..a0c5311da41 --- /dev/null +++ b/cmd/symbols/squirrel/hover.go @@ -0,0 +1,82 @@ +package squirrel + +import ( + "math" + "strings" +) + +// Returns the markdown hover message for the given node if it exists. +func findHover(node Node) string { + style := node.LangSpec.commentStyle + + hover := "" + hover += "```" + style.codeFenceName + "\n" + hover += strings.Split(string(node.Contents), "\n")[node.StartPoint().Row] + "\n" + hover += "```" + + for cur := node.Node; cur != nil && cur.StartPoint().Row == node.StartPoint().Row; cur = cur.Parent() { + prev := cur.PrevNamedSibling() + + // Skip over Java annotations and the like. + for ; prev != nil; prev = prev.PrevNamedSibling() { + if !contains(style.skipNodeTypes, prev.Type()) { + break + } + } + + // Collect comments backwards. + comments := []string{} + lastStartRow := -1 + for ; prev != nil && contains(style.nodeTypes, prev.Type()); prev = prev.PrevNamedSibling() { + if lastStartRow == -1 { + lastStartRow = int(prev.StartPoint().Row) + } else if lastStartRow != int(prev.EndPoint().Row+1) { + break + } else { + lastStartRow = int(prev.StartPoint().Row) + } + + comment := prev.Content(node.Contents) + + // Strip line noise and delete garbage lines. + lines := []string{} + allLines := strings.Split(comment, "\n") + for _, line := range allLines { + if style.ignoreRegex != nil && style.ignoreRegex.MatchString(line) { + continue + } + + if style.stripRegex != nil { + line = style.stripRegex.ReplaceAllString(line, "") + } + + lines = append(lines, line) + } + + // Remove shared leading spaces. + spaces := math.MaxInt32 + for _, line := range lines { + spaces = min(spaces, len(line)-len(strings.TrimLeft(line, " "))) + } + for i := range lines { + lines[i] = strings.TrimLeft(lines[i], " ") + } + + // Join lines. + comments = append(comments, strings.Join(lines, "\n")) + } + + if len(comments) == 0 { + continue + } + + // Reverse comments + for i, j := 0, len(comments)-1; i < j; i, j = i+1, j-1 { + comments[i], comments[j] = comments[j], comments[i] + } + + hover = hover + "\n\n---\n\n" + strings.Join(comments, "\n") + "\n" + } + + return hover +} diff --git a/cmd/symbols/squirrel/hover_test.go b/cmd/symbols/squirrel/hover_test.go new file mode 100644 index 00000000000..c482eee0781 --- /dev/null +++ b/cmd/symbols/squirrel/hover_test.go @@ -0,0 +1,79 @@ +package squirrel + +import ( + "context" + "strings" + "testing" + + "github.com/sourcegraph/sourcegraph/internal/types" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +func TestHover(t *testing.T) { + java := ` +class C { + void m() { + // not a comment line + + // comment line 1 + // comment line 2 + int x = 5; + } +} +` + + tests := []struct { + path string + contents string + want string + }{ + {"test.java", java, "comment line 1\ncomment line 2\n"}, + } + + readFile := func(ctx context.Context, path types.RepoCommitPath) ([]byte, error) { + for _, test := range tests { + if test.path == path.Path { + return []byte(test.contents), nil + } + } + return nil, errors.Newf("path %s not found", path.Path) + } + + squirrel := NewSquirrelService(readFile) + defer squirrel.Close() + + for _, test := range tests { + payload, err := squirrel.localCodeIntel(context.Background(), types.RepoCommitPath{Repo: "foo", Commit: "bar", Path: test.path}) + fatalIfError(t, err) + + ok := false + for _, symbol := range payload.Symbols { + got := symbol.Hover + + if !strings.Contains(got, test.want) { + continue + } else { + ok = true + break + } + } + + if !ok { + comments := []string{} + for _, symbol := range payload.Symbols { + comments = append(comments, symbol.Hover) + } + t.Logf("did not find comment %q. All comments:\n", test.want) + for _, comment := range comments { + t.Logf("%q\n", comment) + } + t.FailNow() + } + } +} + +func fatalIfError(t *testing.T, err error) { + if err != nil { + t.Fatal(err) + } +} diff --git a/cmd/symbols/squirrel/http_handlers.go b/cmd/symbols/squirrel/http_handlers.go new file mode 100644 index 00000000000..63d51ea46bd --- /dev/null +++ b/cmd/symbols/squirrel/http_handlers.go @@ -0,0 +1,161 @@ +package squirrel + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "html" + "io" + "math/rand" + "net/http" + "os" + "strings" + + "github.com/inconshreveable/log15" + sitter "github.com/smacker/go-tree-sitter" + + "github.com/sourcegraph/sourcegraph/internal/types" +) + +// Responds to /localCodeIntel +func LocalCodeIntelHandler(w http.ResponseWriter, r *http.Request) { + // Read the args from the request body. + body, err := io.ReadAll(r.Body) + if err != nil { + log15.Error("failed to read request body", "err", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + var args types.RepoCommitPath + if err := json.NewDecoder(bytes.NewReader(body)).Decode(&args); err != nil { + log15.Error("failed to decode request body", "err", err, "body", string(body)) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + squirrel := NewSquirrelService(readFileFromGitserver) + defer squirrel.Close() + + // Compute the local code intel payload. + payload, err := squirrel.localCodeIntel(r.Context(), args) + if payload != nil && os.Getenv("SQUIRREL_DEBUG") == "true" { + debugStringBuilder := &strings.Builder{} + fmt.Fprintln(debugStringBuilder, "👉 /localCodeIntel repo:", args.Repo, "commit:", args.Commit, "path:", args.Path) + contents, err := readFileFromGitserver(r.Context(), args) + if err != nil { + log15.Error("failed to read file from gitserver", "err", err) + } else { + prettyPrintLocalCodeIntelPayload(debugStringBuilder, *payload, string(contents)) + fmt.Fprintln(debugStringBuilder, "✅ /localCodeIntel repo:", args.Repo, "commit:", args.Commit, "path:", args.Path) + + fmt.Println(" ") + fmt.Println(bracket(debugStringBuilder.String())) + fmt.Println(" ") + } + } + if err != nil { + _ = json.NewEncoder(w).Encode(nil) + log15.Error("failed to generate local code intel payload", "err", err) + return + } + + // Write the response. + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(payload) + if err != nil { + log15.Error("failed to write response: %s", "error", err) + http.Error(w, fmt.Sprintf("failed to generate local code intel payload: %s", err), http.StatusInternalServerError) + return + } +} + +// Response to /debugLocalCodeIntel. +func DebugLocalCodeIntelHandler(w http.ResponseWriter, r *http.Request) { + // Read ?ext= from the request. + ext := r.URL.Query().Get("ext") + if ext == "" { + http.Error(w, "missing ?ext= query parameter", http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "text/html") + + path := types.RepoCommitPath{ + Repo: "foo", + Commit: "bar", + Path: "example." + ext, + } + + fileToRead := "/tmp/squirrel-example.txt" + readFile := func(ctx context.Context, args types.RepoCommitPath) ([]byte, error) { + return os.ReadFile("/tmp/squirrel-example.txt") + } + + squirrel := NewSquirrelService(readFile) + defer squirrel.Close() + + rangeToSymbolIx := map[types.Range]int{} + symbolIxToColor := map[int]string{} + payload, err := squirrel.localCodeIntel(r.Context(), path) + if err != nil { + fmt.Fprintf(w, "failed to generate local code intel payload: %s\n\n", err) + } else { + for ix := range payload.Symbols { + symbolIxToColor[ix] = fmt.Sprintf("hsla(%d, 100%%, 50%%, 0.5)", rand.Intn(360)) + } + + for ix, symbol := range payload.Symbols { + rangeToSymbolIx[symbol.Def] = ix + for _, ref := range symbol.Refs { + rangeToSymbolIx[ref] = ix + } + } + } + + node, err := squirrel.parse(r.Context(), path) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + fmt.Fprintf(w, "

Parsing as %s from file on disk %s

\n", ext, fileToRead) + + nodeToHtml := func(n *sitter.Node, contents []byte) string { + color := "hsla(0, 0%, 0%, 0.1)" + if n.Type() == "ERROR" { + color = "hsla(0, 100%, 50%, 0.2)" + } + if ix, ok := rangeToSymbolIx[nodeToRange(n)]; ok { + if c, ok := symbolIxToColor[ix]; ok { + color = c + } + } + title := fmt.Sprintf("%s %d:%d-%d:%d", html.EscapeString(n.Type()), n.StartPoint().Row, n.StartPoint().Column, n.EndPoint().Row, n.EndPoint().Column) + + return fmt.Sprintf( + `%s`, + color, + title, + html.EscapeString(string(contents[n.StartByte():n.EndByte()])), + ) + } + + fmt.Fprint(w, "
")
+
+	prev := uint32(0)
+	walkFilter(node.Node, func(n *sitter.Node) bool {
+		if n.Type() != "ERROR" && n.ChildCount() > 0 {
+			return true
+		}
+
+		fmt.Fprint(w, html.EscapeString(string(node.Contents[prev:n.StartByte()])))
+		fmt.Fprint(w, nodeToHtml(n, node.Contents))
+
+		prev = n.EndByte()
+
+		return false
+	})
+
+	fmt.Fprint(w, "
") +} diff --git a/cmd/symbols/squirrel/language-file-extensions.json b/cmd/symbols/squirrel/language-file-extensions.json new file mode 100644 index 00000000000..94d59f7bb98 --- /dev/null +++ b/cmd/symbols/squirrel/language-file-extensions.json @@ -0,0 +1,472 @@ +{ + "actionscript": [ + "as" + ], + "ada": [ + "adb", + "ada", + "ads" + ], + "apache": [ + "apacheconf" + ], + "apex": [ + "cls", + "apex", + "trigger" + ], + "applescript": [ + "applescript", + "scpt" + ], + "beancount": [ + "beancount" + ], + "bibtex": [ + "bib" + ], + "clojure": [ + "clj", + "cljs", + "cljx" + ], + "cmake": [ + "cmake", + "cmake.in", + "in" + ], + "coffescript": [ + "coffee", + "cake", + "cson", + "cjsx", + "iced" + ], + "cpp": [ + "c", + "cc", + "cpp", + "cxx", + "c++", + "h++", + "hh", + "h", + "hpp", + "pc", + "pcc" + ], + "csharp": [ + "cs", + "csx" + ], + "css": [ + "css" + ], + "cuda": [ + "cu", + "cuh" + ], + "d": [ + "d" + ], + "dot": [ + "dot" + ], + "dart": [ + "dart" + ], + "diff": [ + "diff", + "patch" + ], + "dockerfile": [ + "Dockerfile" + ], + "django": [ + "jinja" + ], + "dos": [ + "bat", + "cmd" + ], + "elixir": [ + "ex", + "exs" + ], + "elm": [ + "elm" + ], + "erlang": [ + "erl" + ], + "fortran": [ + "f", + "for", + "frt", + "fr", + "forth", + "4th", + "fth" + ], + "fsharp": [ + "fs" + ], + "go": [ + "go" + ], + "graphql": [ + "graphql" + ], + "groovy": [ + "groovy" + ], + "haml": [ + "haml" + ], + "handlebars": [ + "hbs", + "handlebars" + ], + "haskell": [ + "hs", + "hsc" + ], + "hcl": [ + "hcl", + "nomad", + "tf", + "tfvars", + "workflow" + ], + "html": [ + "htm", + "html", + "xhtml" + ], + "ini": [ + "ini", + "cfg", + "prefs", + "pro", + "properties" + ], + "java": [ + "java" + ], + "javascript": [ + "js", + "jsx", + "es", + "es6", + "jss", + "jsm" + ], + "json": [ + "json", + "sublime_metrics", + "sublime_session", + "sublime-keymap", + "sublime-mousemap", + "sublime-project", + "sublime-settings", + "sublime-workspace" + ], + "jsonnet": [ + "jsonnet", + "libsonnet" + ], + "julia": [ + "jl" + ], + "kotlin": [ + "kt", + "ktm", + "kts" + ], + "less": [ + "less" + ], + "lisp": [ + "lisp", + "asd", + "cl", + "lsp", + "l", + "ny", + "podsl", + "sexp", + "el" + ], + "lua": [ + "lua", + "fcgi", + "nse", + "pd_lua", + "rbxs", + "wlua" + ], + "makefile": [ + "mk", + "mak" + ], + "markdown": [ + "md", + "mkdown", + "mkd" + ], + "nginx": [ + "nginxconf" + ], + "objectivec": [ + "m", + "mm" + ], + "ocaml": [ + "ml", + "eliom", + "eliomi", + "ml4", + "mli", + "mll", + "mly", + "re" + ], + "pascal": [ + "p", + "pas", + "pp" + ], + "perl": [ + "pl", + "al", + "cgi", + "perl", + "ph", + "plx", + "pm", + "pod", + "psgi", + "t" + ], + "php": [ + "php", + "phtml", + "php3", + "php4", + "php5", + "php6", + "php7", + "phps" + ], + "powershell": [ + "ps1", + "psd1", + "psm1" + ], + "protobuf": [ + "proto" + ], + "python": [ + "py", + "pyc", + "pyd", + "pyo", + "pyw", + "pyz" + ], + "r": [ + "r", + "rd", + "rsx" + ], + "repro": [ + "repro" + ], + "ruby": [ + "rb", + "builder", + "eye", + "gemspec", + "god", + "jbuilder", + "mspec", + "pluginspec", + "podspec", + "rabl", + "rake", + "rbuild", + "rbw", + "rbx", + "ru", + "ruby", + "spec", + "thor", + "watchr" + ], + "rust": [ + "rs", + "rs.in" + ], + "scala": [ + "sbt", + "sc", + "scala" + ], + "scheme": [ + "scm", + "sch", + "sls", + "sps", + "ss" + ], + "scss": [ + "sass", + "scss" + ], + "shell": [ + "sh", + "bash", + "zsh" + ], + "smalltalk": [ + "st" + ], + "sql": [ + "sql" + ], + "stylus": [ + "styl" + ], + "svelte": [ + "svelte" + ], + "swift": [ + "swift" + ], + "thrift": [ + "thrift" + ], + "toml": [ + "toml" + ], + "twig": [ + "twig" + ], + "typescript": [ + "ts", + "tsx" + ], + "vbnet": [ + "vb" + ], + "vbscrip": [ + "vbs" + ], + "verilog": [ + "v", + "veo", + "sv", + "svh", + "svi" + ], + "vhdl": [ + "vhd", + "vhdl" + ], + "vim": [ + "vim" + ], + "xml": [ + "xml", + "adml", + "admx", + "ant", + "axml", + "builds", + "ccxml", + "clixml", + "cproject", + "csl", + "csproj", + "ct", + "dita", + "ditamap", + "ditaval", + "dll.config", + "dotsettings", + "filters", + "fsproj", + "fxml", + "glade", + "gml", + "grxml", + "iml", + "ivy", + "jelly", + "jsproj", + "kml", + "launch", + "mdpolicy", + "mjml", + "mod", + "mxml", + "nproj", + "nuspec", + "odd", + "osm", + "pkgproj", + "plist", + "props", + "ps1xml", + "psc1", + "pt", + "rdf", + "resx", + "rss", + "scxml", + "sfproj", + "srdf", + "storyboard", + "stTheme", + "sublime-snippet", + "targets", + "tmCommand", + "tml", + "tmLanguage", + "tmPreferences", + "tmSnippet", + "tmTheme", + "ui", + "urdf", + "ux", + "vbproj", + "vcxproj", + "vsixmanifest", + "vssettings", + "vstemplate", + "vxml", + "wixproj", + "wsdl", + "wsf", + "wxi", + "wxl", + "wxs", + "x3d", + "xacro", + "xaml", + "xib", + "xlf", + "xliff", + "xmi", + "xml.dist", + "xproj", + "xsd", + "xspec", + "xul", + "zcml" + ], + "yaml": [ + "yml", + "yaml" + ] +} diff --git a/cmd/symbols/squirrel/languages.go b/cmd/symbols/squirrel/languages.go new file mode 100644 index 00000000000..36b9e5511ec --- /dev/null +++ b/cmd/symbols/squirrel/languages.go @@ -0,0 +1,86 @@ +package squirrel + +import ( + _ "embed" + "encoding/json" + "fmt" + + "github.com/grafana/regexp" + sitter "github.com/smacker/go-tree-sitter" + "github.com/smacker/go-tree-sitter/java" +) + +//go:embed language-file-extensions.json +var languageFileExtensionsJson string + +// Mapping from langauge name to file extensions. +var langToExts = func() map[string][]string { + var m map[string][]string + err := json.Unmarshal([]byte(languageFileExtensionsJson), &m) + if err != nil { + panic(err) + } + return m +}() + +// Mapping from file extension to language name. +var extToLang = func() map[string]string { + m := map[string]string{} + for lang, exts := range langToExts { + for _, ext := range exts { + if _, ok := m[ext]; ok { + panic(fmt.Sprintf("duplicate file extension %s", ext)) + } + m[ext] = lang + } + } + return m +}() + +// Info about a language. +type LangSpec struct { + language *sitter.Language + commentStyle CommentStyle + // localsQuery is a tree-sitter localsQuery that finds scopes and defs. + localsQuery string +} + +// Info about comments in a language. +type CommentStyle struct { + ignoreRegex *regexp.Regexp + stripRegex *regexp.Regexp + skipNodeTypes []string + nodeTypes []string + codeFenceName string +} + +// Mapping from language name to language specification. Queries were copied from +// nvim-treesitter@5b6f6ae30c1cf8fceefe08a9bcf799870558a878 as a starting point. +var langToLangSpec = map[string]LangSpec{ + "java": { + language: java.GetLanguage(), + commentStyle: CommentStyle{ + nodeTypes: []string{"comment"}, + stripRegex: regexp.MustCompile(`(^//|^\s*\*|^/\*\*|\*/$)`), + ignoreRegex: regexp.MustCompile(`^\s*(/\*\*|\*/)\s*$`), + codeFenceName: "java", + skipNodeTypes: []string{"modifiers"}, + }, + localsQuery: ` +(block) @scope ; { ... } +(lambda_expression) @scope ; (x, y) -> ... +(catch_clause) @scope ; try { ... } catch (Exception e) { ... } +(enhanced_for_statement) @scope ; for (var item : items) ... +(for_statement) @scope ; for (var i = 0; i < 5; i++) ... +(constructor_declaration) @scope ; public Foo() { ... } +(method_declaration) @scope ; public void f() { ... } + +(local_variable_declaration declarator: (variable_declarator name: (identifier) @definition)) ; int x = ...; +(formal_parameter name: (identifier) @definition) ; public void f(int x) { ... } +(catch_formal_parameter name: (identifier) @definition) ; try { ... } catch (Exception e) { ... } +(lambda_expression parameters: (inferred_parameters (identifier) @definition)) ; (x, y) -> ... +(lambda_expression parameters: (identifier) @definition) ; x -> ... +(enhanced_for_statement name: (identifier) @definition) ; for (var item : items) ... +`, + }, +} diff --git a/cmd/symbols/squirrel/local_code_intel.go b/cmd/symbols/squirrel/local_code_intel.go new file mode 100644 index 00000000000..2f0dce6117d --- /dev/null +++ b/cmd/symbols/squirrel/local_code_intel.go @@ -0,0 +1,193 @@ +package squirrel + +import ( + "context" + "fmt" + "io" + "sort" + "strings" + + "github.com/fatih/color" + sitter "github.com/smacker/go-tree-sitter" + + "github.com/sourcegraph/sourcegraph/internal/types" +) + +// Nominal type for symbol names. +type SymbolName string + +// Scope is a mapping from symbol name to symbol. +type Scope = map[SymbolName]*PartialSymbol // pointer for mutability + +// PartialSymbol is the same as types.Symbol, but with the refs stored in a map to deduplicate. +type PartialSymbol struct { + Name string + Hover string + Def types.Range + // Store refs as a set to avoid duplicates from some tree-sitter queries. + Refs map[types.Range]struct{} +} + +// Computes the local code intel payload, which is a list of symbols. +func (squirrel *SquirrelService) localCodeIntel(ctx context.Context, repoCommitPath types.RepoCommitPath) (*types.LocalCodeIntelPayload, error) { + // Parse the file. + root, err := squirrel.parse(ctx, repoCommitPath) + if err != nil { + return nil, err + } + + // Collect scopes + rootScopeId := nodeId(root.Node) + scopes := map[NodeId]Scope{ + rootScopeId: {}, + } + err = forEachCapture(root.LangSpec.localsQuery, *root, func(captureName string, node Node) { + if captureName == "scope" { + scopes[nodeId(node.Node)] = map[SymbolName]*PartialSymbol{} + return + } + }) + if err != nil { + return nil, err + } + + // Collect defs + err = forEachCapture(root.LangSpec.localsQuery, *root, func(captureName string, node Node) { + // Only collect "definition*" captures. + if strings.HasPrefix(captureName, "definition") { + // Find the nearest scope (if it exists). + for cur := node.Node; cur != nil; cur = cur.Parent() { + // Found the scope. + if scope, ok := scopes[nodeId(cur)]; ok { + // Get the symbol name. + symbolName := SymbolName(node.Content(node.Contents)) + + // Skip the symbol if it's already defined. + if _, ok := scope[symbolName]; ok { + break + } + + // Put the symbol in the scope. + scope[symbolName] = &PartialSymbol{ + Name: string(symbolName), + Hover: findHover(node), + Def: nodeToRange(node.Node), + Refs: map[types.Range]struct{}{}, + } + + // Stop walking up the tree. + break + } + } + } + }) + if err != nil { + return nil, err + } + + // Collect refs by walking the entire tree. + walk(root.Node, func(node *sitter.Node) { + // Only collect identifiers. + if !strings.Contains(node.Type(), "identifier") { + return + } + + // Get the symbol name. + symbolName := SymbolName(node.Content(root.Contents)) + + // Find the nearest scope (if it exists). + for cur := node; cur != nil; cur = cur.Parent() { + if scope, ok := scopes[nodeId(cur)]; ok { + // Check if it's in the scope. + if _, ok := scope[symbolName]; !ok { + // It's not in this scope, so keep walking up the tree. + continue + } + + // Put the ref in the scope. + scope[symbolName].Refs[nodeToRange(node)] = struct{}{} + + // Done. + return + } + } + + // Did not find the symbol in this file, so ignore it. + }) + + // Collect the symbols. + symbols := []types.Symbol{} + for _, scope := range scopes { + for _, partialSymbol := range scope { + refs := []types.Range{} + for ref := range partialSymbol.Refs { + refs = append(refs, ref) + } + symbols = append(symbols, types.Symbol{ + Name: string(partialSymbol.Name), + Hover: partialSymbol.Hover, + Def: partialSymbol.Def, + Refs: refs, + }) + } + } + + return &types.LocalCodeIntelPayload{Symbols: symbols}, nil +} + +// Pretty prints the local code intel payload for debugging. +func prettyPrintLocalCodeIntelPayload(w io.Writer, payload types.LocalCodeIntelPayload, contents string) { + lines := strings.Split(contents, "\n") + + // Sort payload.Symbols by Def Row then Column. + sort.Slice(payload.Symbols, func(i, j int) bool { + return isLessRange(payload.Symbols[i].Def, payload.Symbols[j].Def) + }) + + // Print all symbols. + for _, symbol := range payload.Symbols { + defColor := color.New(color.FgMagenta) + refColor := color.New(color.FgCyan) + fmt.Fprintf(w, "Hover %q, %s, %s\n", symbol.Hover, defColor.Sprint("def"), refColor.Sprint("refs")) + + // Convert each def and ref into a rangeColor. + type rangeColor struct { + rnge types.Range + color_ *color.Color + } + + rnges := []rangeColor{} + rnges = append(rnges, rangeColor{rnge: symbol.Def, color_: defColor}) + for _, ref := range symbol.Refs { + rnges = append(rnges, rangeColor{rnge: ref, color_: refColor}) + } + + // How to print a range in color. + printRange := func(rnge types.Range, c *color.Color) { + line := lines[rnge.Row] + lineWithSpaces := tabsToSpaces(line) + column := lengthInSpaces(line[:rnge.Column]) + length := lengthInSpaces(line[rnge.Column : rnge.Column+rnge.Length]) + fmt.Fprint(w, color.New(color.FgBlack).Sprintf("%4d | ", rnge.Row)) + fmt.Fprint(w, color.New(color.FgBlack).Sprint(lineWithSpaces[:column])) + fmt.Fprint(w, c.Sprint(lineWithSpaces[column:column+length])) + fmt.Fprint(w, color.New(color.FgBlack).Sprint(lineWithSpaces[column+length:])) + fmt.Fprintln(w) + } + + // Sort ranges by row, then column. + sort.Slice(rnges, func(i, j int) bool { + if rnges[i].rnge.Row == rnges[j].rnge.Row { + return rnges[i].rnge.Column < rnges[j].rnge.Column + } + return rnges[i].rnge.Row < rnges[j].rnge.Row + }) + + // Print each range. + for _, rnge := range rnges { + printRange(rnge.rnge, rnge.color_) + } + + fmt.Fprintln(w) + } +} diff --git a/cmd/symbols/squirrel/local_code_intel_test.go b/cmd/symbols/squirrel/local_code_intel_test.go new file mode 100644 index 00000000000..db48b168388 --- /dev/null +++ b/cmd/symbols/squirrel/local_code_intel_test.go @@ -0,0 +1,241 @@ +package squirrel + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/grafana/regexp" + + "github.com/sourcegraph/sourcegraph/internal/types" +) + +func TestLocalCodeIntel(t *testing.T) { + path := types.RepoCommitPath{Repo: "foo", Commit: "bar", Path: "test.java"} + contents := ` +class Foo { + + // v f1.p def + // v f1.p ref + void f1(String p) { + + // v f1.x def + // v f1.x ref + // v f1.p ref + String x = p; + } + + // v f2.p def + // v f2.p ref + void f2(String p) { + + // v f2.x def + // v f2.x ref + // v f2.p ref + String x = p; + } +} +` + + want := collectAnnotations(path, contents) + + payload := getLocalCodeIntel(t, path, contents) + got := []annotation{} + for _, symbol := range payload.Symbols { + got = append(got, annotation{ + repoCommitPathPoint: types.RepoCommitPathPoint{ + RepoCommitPath: path, + Point: types.Point{ + Row: symbol.Def.Row, + Column: symbol.Def.Column, + }, + }, + symbol: "(unused)", + kind: "def", + }) + + for _, ref := range symbol.Refs { + got = append(got, annotation{ + repoCommitPathPoint: types.RepoCommitPathPoint{ + RepoCommitPath: path, + Point: types.Point{ + Row: ref.Row, + Column: ref.Column, + }, + }, + symbol: "(unused)", + kind: "ref", + }) + } + } + + sortAnnotations(want) + sortAnnotations(got) + + if diff := cmp.Diff(want, got, compareAnnotations); diff != "" { + t.Fatalf("unexpected annotations (-want +got):\n%s", diff) + } +} + +func getLocalCodeIntel(t *testing.T, path types.RepoCommitPath, contents string) *types.LocalCodeIntelPayload { + readFile := func(ctx context.Context, path types.RepoCommitPath) ([]byte, error) { + return []byte(contents), nil + } + + squirrel := NewSquirrelService(readFile) + defer squirrel.Close() + + payload, err := squirrel.localCodeIntel(context.Background(), path) + fatalIfError(t, err) + + return payload +} + +type annotation struct { + repoCommitPathPoint types.RepoCommitPathPoint + symbol string + kind string +} + +func collectAnnotations(repoCommitPath types.RepoCommitPath, contents string) []annotation { + annotations := []annotation{} + + lines := strings.Split(contents, "\n") + + // Annotation at the end of the line + for i, line := range lines { + matches := regexp.MustCompile(`^([^<]+)< "([^"]+)" ([a-zA-Z0-9_.-]+) (def|ref)`).FindStringSubmatch(line) + if matches == nil { + continue + } + + substr, symbol, kind := matches[2], matches[3], matches[4] + + annotations = append(annotations, annotation{ + repoCommitPathPoint: types.RepoCommitPathPoint{ + RepoCommitPath: repoCommitPath, + Point: types.Point{ + Row: i, + Column: strings.Index(line, substr), + }, + }, + symbol: symbol, + kind: kind, + }) + } + + // Annotations below source lines +nextSourceLine: + for sourceLine := 0; ; { + for annLine := sourceLine + 1; ; annLine++ { + if annLine >= len(lines) { + break nextSourceLine + } + + matches := regexp.MustCompile(`([^^]*)\^+ ([a-zA-Z0-9_.-]+) (def|ref)`).FindStringSubmatch(lines[annLine]) + if matches == nil { + sourceLine = annLine + continue nextSourceLine + } + + prefix, symbol, kind := matches[1], matches[2], matches[3] + + annotations = append(annotations, annotation{ + repoCommitPathPoint: types.RepoCommitPathPoint{ + RepoCommitPath: repoCommitPath, + Point: types.Point{ + Row: sourceLine, + Column: spacesToColumn(lines[sourceLine], lengthInSpaces(prefix)), + }, + }, + symbol: symbol, + kind: kind, + }) + } + } + + // Annotations above source lines +previousSourceLine: + for sourceLine := len(lines) - 1; ; { + for annLine := sourceLine - 1; ; annLine-- { + if annLine < 0 { + break previousSourceLine + } + + matches := regexp.MustCompile(`([^v]*)v+ ([a-zA-Z0-9_.-]+) (def|ref)`).FindStringSubmatch(lines[annLine]) + if matches == nil { + sourceLine = annLine + continue previousSourceLine + } + + prefix, symbol, kind := matches[1], matches[2], matches[3] + + annotations = append(annotations, annotation{ + repoCommitPathPoint: types.RepoCommitPathPoint{ + RepoCommitPath: repoCommitPath, + Point: types.Point{ + Row: sourceLine, + Column: spacesToColumn(lines[sourceLine], lengthInSpaces(prefix)), + }, + }, + symbol: symbol, + kind: kind, + }) + } + } + + return annotations +} + +func sortAnnotations(annotations []annotation) { + sort.Slice(annotations, func(i, j int) bool { + rowi := annotations[i].repoCommitPathPoint.Point.Row + rowj := annotations[j].repoCommitPathPoint.Point.Row + coli := annotations[i].repoCommitPathPoint.Point.Column + colj := annotations[j].repoCommitPathPoint.Point.Column + kindi := annotations[i].kind + kindj := annotations[j].kind + if rowi != rowj { + return rowi < rowj + } else if coli != colj { + return coli < colj + } else { + return kindi < kindj + } + }) +} + +func printRowColumnKind(annotations []annotation) string { + sortAnnotations(annotations) + + lines := []string{} + for _, annotation := range annotations { + lines = append(lines, fmt.Sprintf( + "%d:%d %s", + annotation.repoCommitPathPoint.Row, + annotation.repoCommitPathPoint.Column, + annotation.kind, + )) + } + + return strings.Join(lines, "\n") +} + +var compareAnnotations = cmp.Comparer(func(a, b annotation) bool { + if a.repoCommitPathPoint.RepoCommitPath != b.repoCommitPathPoint.RepoCommitPath { + return false + } + if a.repoCommitPathPoint.Point.Row != b.repoCommitPathPoint.Point.Row { + return false + } + if a.repoCommitPathPoint.Point.Column != b.repoCommitPathPoint.Point.Column { + return false + } + if a.kind != b.kind { + return false + } + return true +}) diff --git a/cmd/symbols/squirrel/service.go b/cmd/symbols/squirrel/service.go new file mode 100644 index 00000000000..1703a0c6f90 --- /dev/null +++ b/cmd/symbols/squirrel/service.go @@ -0,0 +1,50 @@ +package squirrel + +import ( + "context" + + sitter "github.com/smacker/go-tree-sitter" + + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/gitserver" + "github.com/sourcegraph/sourcegraph/internal/types" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +// How to read a file. +type ReadFileFunc func(context.Context, types.RepoCommitPath) ([]byte, error) + +// SquirrelService uses tree-sitter to analyze code and collect symbols. +type SquirrelService struct { + readFile ReadFileFunc + parser *sitter.Parser + closables []func() +} + +// Creates a new SquirrelService. +func NewSquirrelService(readFile ReadFileFunc) *SquirrelService { + return &SquirrelService{ + readFile: readFile, + parser: sitter.NewParser(), + closables: []func(){}, + } +} + +// Remember to free memory allocated by tree-sitter. +func (squirrel *SquirrelService) Close() { + for _, close := range squirrel.closables { + close() + } + squirrel.parser.Close() +} + +// How to read a file from gitserver. +func readFileFromGitserver(ctx context.Context, repoCommitPath types.RepoCommitPath) ([]byte, error) { + cmd := gitserver.NewClient(nil).Command("git", "cat-file", "blob", repoCommitPath.Commit+":"+repoCommitPath.Path) + cmd.Repo = api.RepoName(repoCommitPath.Repo) + stdout, stderr, err := cmd.DividedOutput(ctx) + if err != nil { + return nil, errors.Newf("failed to get file contents: %s\n\nstdout:\n\n%s\n\nstderr:\n\n%s", err, stdout, stderr) + } + return stdout, nil +} diff --git a/cmd/symbols/squirrel/util.go b/cmd/symbols/squirrel/util.go new file mode 100644 index 00000000000..84f6170c2fc --- /dev/null +++ b/cmd/symbols/squirrel/util.go @@ -0,0 +1,243 @@ +package squirrel + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + sitter "github.com/smacker/go-tree-sitter" + + "github.com/sourcegraph/sourcegraph/internal/types" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +// Nominal type for the ID of a tree-sitter node. +type NodeId string + +// walk walks every node in the tree-sitter tree, calling f(node) on each node. +func walk(node *sitter.Node, f func(node *sitter.Node)) { + walkFilter(node, func(n *sitter.Node) bool { f(n); return true }) +} + +// walkFilter walks every node in the tree-sitter tree, calling f(node) on each node and descending into +// children if it returns true. +func walkFilter(node *sitter.Node, f func(node *sitter.Node) bool) { + if f(node) { + for i := 0; i < int(node.ChildCount()); i++ { + walkFilter(node.Child(i), f) + } + } +} + +// nodeId returns the ID of the node. +func nodeId(node *sitter.Node) NodeId { + return NodeId(fmt.Sprint(nodeToRange(node))) +} + +// getRoot returns the root node of the tree-sitter tree, given any node inside it. +func getRoot(node *sitter.Node) *sitter.Node { + var top *sitter.Node + for cur := node; cur != nil; cur = cur.Parent() { + top = cur + } + return top +} + +// isLessRange compares ranges. +func isLessRange(a, b types.Range) bool { + if a.Row == b.Row { + return a.Column < b.Column + } + return a.Row < b.Row +} + +// tabsToSpaces converts tabs to spaces. +func tabsToSpaces(s string) string { + return strings.ReplaceAll(s, "\t", " ") +} + +const TAB_SIZE = 4 + +// lengthInSpaces returns the length of the string in spaces (using TAB_SIZE). +func lengthInSpaces(s string) int { + total := 0 + for i := 0; i < len(s); i++ { + if s[i] == '\t' { + total += TAB_SIZE + } else { + total++ + } + } + return total +} + +// spacesToColumn measures the length in spaces from the start of the string to the given column. +func spacesToColumn(s string, column int) int { + total := 0 + for i := 0; i < len(s); i++ { + if total >= column { + return i + } + + if s[i] == '\t' { + total += TAB_SIZE + } else { + total++ + } + } + return total +} + +// colorSprintfFunc is a color printing function. +type colorSprintfFunc func(a ...interface{}) string + +// bracket prefixes all the lines of the given string with pretty brackets. +func bracket(text string) string { + lines := strings.Split(strings.TrimSpace(text), "\n") + if len(lines) == 1 { + return "- " + text + } + + for i, line := range lines { + if i == 0 { + lines[i] = "┌ " + line + } else if i < len(lines)-1 { + lines[i] = "│ " + line + } else { + lines[i] = "└ " + line + } + } + + return strings.Join(lines, "\n") +} + +// forEachCapture runs the given tree-sitter query on the given node and calls f(captureName, node) for +// each capture. +func forEachCapture(query string, node Node, f func(captureName string, node Node)) error { + sitterQuery, err := sitter.NewQuery([]byte(query), node.LangSpec.language) + if err != nil { + return errors.Newf("failed to parse query: %s\n%s", err, query) + } + defer sitterQuery.Close() + cursor := sitter.NewQueryCursor() + defer cursor.Close() + cursor.Exec(sitterQuery, node.Node) + + match, _, hasCapture := cursor.NextCapture() + for hasCapture { + for _, capture := range match.Captures { + captureName := sitterQuery.CaptureNameForId(capture.Index) + f(captureName, Node{ + RepoCommitPath: node.RepoCommitPath, + Node: capture.Node, + Contents: node.Contents, + LangSpec: node.LangSpec, + }) + } + match, _, hasCapture = cursor.NextCapture() + } + + return nil +} + +// nodeToRange returns the range of the node. +func nodeToRange(node *sitter.Node) types.Range { + length := 1 + if node.StartPoint().Row == node.EndPoint().Row { + length = int(node.EndPoint().Column - node.StartPoint().Column) + } + return types.Range{ + Row: int(node.StartPoint().Row), + Column: int(node.StartPoint().Column), + Length: length, + } +} + +// nodeLength returns the length of the node. +func nodeLength(node *sitter.Node) int { + length := 1 + if node.StartPoint().Row == node.EndPoint().Row { + length = int(node.EndPoint().Column - node.StartPoint().Column) + } + return length +} + +// Of course. +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// When generic? +func contains(slice []string, str string) bool { + for _, s := range slice { + if s == str { + return true + } + } + return false +} + +// A sitter.Node plus convenient info. +type Node struct { + RepoCommitPath types.RepoCommitPath + *sitter.Node + Contents []byte + LangSpec LangSpec +} + +func WithNode(other Node, newNode *sitter.Node) Node { + return Node{ + RepoCommitPath: other.RepoCommitPath, + Node: newNode, + Contents: other.Contents, + LangSpec: other.LangSpec, + } +} + +func WithNodePtr(other Node, newNode *sitter.Node) *Node { + return &Node{ + RepoCommitPath: other.RepoCommitPath, + Node: newNode, + Contents: other.Contents, + LangSpec: other.LangSpec, + } +} + +// Parses a file and returns info about it. +func (s *SquirrelService) parse(ctx context.Context, repoCommitPath types.RepoCommitPath) (*Node, error) { + ext := strings.TrimPrefix(filepath.Ext(repoCommitPath.Path), ".") + + langName, ok := extToLang[ext] + if !ok { + return nil, errors.Newf("unrecognized file extension %s", ext) + } + + langSpec, ok := langToLangSpec[langName] + if !ok { + return nil, errors.Newf("unsupported language %s", langName) + } + + s.parser.SetLanguage(langSpec.language) + + contents, err := s.readFile(ctx, repoCommitPath) + if err != nil { + return nil, err + } + + tree, err := s.parser.ParseCtx(ctx, nil, contents) + if err != nil { + return nil, errors.Newf("failed to parse file contents: %s", err) + } + s.closables = append(s.closables, tree.Close) + + root := tree.RootNode() + if root == nil { + return nil, errors.New("root is nil") + } + + return &Node{RepoCommitPath: repoCommitPath, Node: root, Contents: contents, LangSpec: langSpec}, nil +} diff --git a/go.mod b/go.mod index f638d5ad712..570e5e30ca3 100644 --- a/go.mod +++ b/go.mod @@ -128,6 +128,7 @@ require ( github.com/shurcooL/github_flavored_markdown v0.0.0-20210228213109-c3a9aa474629 github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 github.com/slack-go/slack v0.10.1 + github.com/smacker/go-tree-sitter v0.0.0-20220209044044-0d3022e933c3 github.com/snabb/sitemap v1.0.0 github.com/sourcegraph/ctxvfs v0.0.0-20180418081416-2b65f1b1ea81 github.com/sourcegraph/go-ctags v0.0.0-20220221141751-78951a22ec08 @@ -208,7 +209,6 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/philhofer/fwd v1.1.1 // indirect github.com/pseudomuto/protokit v0.2.0 // indirect - github.com/smacker/go-tree-sitter v0.0.0-20220209044044-0d3022e933c3 // indirect github.com/tinylib/msgp v1.1.2 // indirect github.com/yuin/goldmark v1.4.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect diff --git a/internal/gitserver/gitdomain/exec.go b/internal/gitserver/gitdomain/exec.go index b00030b98e2..0b00570e543 100644 --- a/internal/gitserver/gitdomain/exec.go +++ b/internal/gitserver/gitdomain/exec.go @@ -29,6 +29,7 @@ var ( "merge-base": {"--"}, "show-ref": {"--heads"}, "shortlog": {"-s", "-n", "-e", "--no-merges"}, + "cat-file": {}, // Used in tests to simulate errors with runCommand in handleExec of gitserver. "testcommand": {}, diff --git a/internal/symbols/client.go b/internal/symbols/client.go index 4c8609023a4..9264393b76d 100644 --- a/internal/symbols/client.go +++ b/internal/symbols/client.go @@ -28,6 +28,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/result" "github.com/sourcegraph/sourcegraph/internal/trace/ot" + "github.com/sourcegraph/sourcegraph/internal/types" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -193,6 +194,42 @@ func (c *Client) Search(ctx context.Context, args search.SymbolsParameters) (sym return filtered, nil } +func (c *Client) LocalCodeIntel(ctx context.Context, args types.RepoCommitPath) (result *types.LocalCodeIntelPayload, err error) { + span, ctx := ot.StartSpanFromContext(ctx, "squirrel.Client.LocalCodeIntel") + defer func() { + if err != nil { + ext.Error.Set(span, true) + span.LogFields(otlog.Error(err)) + } + span.Finish() + }() + span.SetTag("Repo", args.Repo) + span.SetTag("CommitID", args.Commit) + + resp, err := c.httpPost(ctx, "localCodeIntel", api.RepoName(args.Repo), args) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + // best-effort inclusion of body in error message + body, _ := io.ReadAll(io.LimitReader(resp.Body, 200)) + return nil, errors.Errorf( + "Squirrel.LocalCodeIntel http status %d: %s", + resp.StatusCode, + string(body), + ) + } + + err = json.NewDecoder(resp.Body).Decode(&result) + if err != nil { + return nil, errors.Wrap(err, "decoding response body") + } + + return result, nil +} + func (c *Client) httpPost( ctx context.Context, method string, diff --git a/internal/types/codeintel.go b/internal/types/codeintel.go index 4d996948c6e..d5445ee2a82 100644 --- a/internal/types/codeintel.go +++ b/internal/types/codeintel.go @@ -1,6 +1,9 @@ package types -import "time" +import ( + "fmt" + "time" +) // CodeIntelAggregatedEvent represents the total events and unique users within // the current week for a single event. The events are split again by language id @@ -92,3 +95,49 @@ type OldCodeIntelEventStatistics struct { UsersCount int32 EventsCount *int32 } + +type RepoCommitPath struct { + Repo string `json:"repo"` + Commit string `json:"commit"` + Path string `json:"path"` +} + +type LocalCodeIntelPayload struct { + Symbols []Symbol `json:"symbols"` +} + +type RepoCommitPathRange struct { + RepoCommitPath + Range +} + +type RepoCommitPathPoint struct { + RepoCommitPath + Point +} + +type Point struct { + Row int `json:"row"` + Column int `json:"column"` +} + +type Symbol struct { + Name string `json:"name"` + Hover string `json:"hover,omitempty"` + Def Range `json:"def,omitempty"` + Refs []Range `json:"refs,omitempty"` +} + +func (s Symbol) String() string { + return fmt.Sprintf("Symbol{Hover: %q, Def: %s, Refs: %+v", s.Hover, s.Def, s.Refs) +} + +type Range struct { + Row int `json:"row"` + Column int `json:"column"` + Length int `json:"length"` +} + +func (r Range) String() string { + return fmt.Sprintf("%d:%d:%d", r.Row, r.Column, r.Length) +}