artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 17:51:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

syntax-highlighter: scip-ctags implementation (#50600)

Browse Source 
- Write Rust one-for-one protocol compatible replacement for
universal-ctags in JSON streaming mode (scip-ctags)
- Use tree-sitter to generate scip symbols, then emit those through
scip-ctags
- These symbols will be reused for Cody context 

Currently, only zig is enabled (so other languages should remain unaffected by this change).

We will add other languages throughout the next week as we're able to check them off.

## Test plan

Unit and snapshot tests in the Rust symbol generation code - verified
working in the symbols sidebar and symbol search for enabled languages.

---------

Co-authored-by: SuperAuguste <19855629+SuperAuguste@users.noreply.github.com>
Co-authored-by: William Bezuidenhout <william.bezuidenhout@sourcegraph.com>
Co-authored-by: Eric Fritz <eric@eric-fritz.com>
Co-authored-by: Eric Fritz <eric@sourcegraph.com>
This commit is contained in:
TJ DeVries 2023-05-30 11:53:36 -04:00 committed by GitHub
parent 98be4259c1
commit 1ff6fb12d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
93 changed files with 4304 additions and 900 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/frontend/internal/highlight/BUILD.bazel generated
View File

@ -22,10 +22,10 @@ go_library(
"//internal/honey",
"//internal/observation",
"//internal/trace/ot",
"//lib/codeintel/languages",
"//lib/errors",
"@com_github_alecthomas_chroma_v2//:chroma",
"@com_github_alecthomas_chroma_v2//lexers",
"@com_github_go_enry_go_enry_v2//:go-enry",
"@com_github_grafana_regexp//:regexp",
"@com_github_inconshreveable_log15//:log15",
"@com_github_prometheus_client_golang//prometheus",
@ -33,7 +33,6 @@ go_library(
"@com_github_sourcegraph_scip//bindings/go/scip",
"@io_opentelemetry_go_otel//attribute",
"@org_golang_google_protobuf//proto",
"@org_golang_x_exp//slices",
"@org_golang_x_net//html",
"@org_golang_x_net//html/atom",
],

41
cmd/frontend/internal/highlight/language.go
View File

@ -5,13 +5,12 @@ import (
"path/filepath"
"strings"
"github.com/go-enry/go-enry/v2"
"github.com/grafana/regexp"
"golang.org/x/exp/slices"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/gosyntect"
"github.com/sourcegraph/sourcegraph/lib/codeintel/languages"
)
type EngineType int
@ -247,45 +246,11 @@ func getLanguage(path string, contents string) (string, bool) {
return lang, true
}
// Force the use of the shebang.
if shebangLang, ok := overrideViaShebang(path, contents); ok {
return shebangLang, false
}
// Lastly, fall back to whatever enry decides is a useful algorithm for calculating.
lang = enry.GetLanguage(path, []byte(contents))
// TODO: Consider if we should just ignore getting empty...?
lang, _ = languages.GetLanguage(path, contents)
return lang, false
}
// overrideViaShebang handles explicitly using the shebang whenever possible.
//
// It also covers some edge cases when enry eagerly returns more languages
// than necessary, which ends up overriding the shebang completely (which,
// IMO is the highest priority match we can have).
//
// For example, enry will return "Perl" and "Pod" for a shebang of `#!/usr/bin/env perl`.
// This is actually unhelpful, because then enry will *not* select "Perl" as the
// language (which is our desired behavior).
func overrideViaShebang(path, content string) (lang string, ok bool) {
shebangs := enry.GetLanguagesByShebang(path, []byte(content), []string{})
if len(shebangs) == 0 {
return "", false
}
if len(shebangs) == 1 {
return shebangs[0], true
}
// There are some shebangs that enry returns that are not really
// useful for our syntax highlighters to distinguish between.
if slices.Equal(shebangs, []string{"Perl", "Pod"}) {
return "Perl", true
}
return "", false
}
// DetectSyntaxHighlightingLanguage will calculate the SyntaxEngineQuery from a given
// path and contents. First it will determine if there are any configuration overrides
// and then, if none, return the 'enry' default language detection

2
cmd/server/build-bazel.sh
View File

@ -28,6 +28,7 @@ OSS_TARGETS=(
//cmd/gitserver
//cmd/searcher
//cmd/server
//docker-images/syntax-highlighter:scip-ctags
# https://github.com/sourcegraph/s3proxy is still the default for now.
# //cmd/blobstore
@com_github_sourcegraph_zoekt//cmd/zoekt-archive-index
@ -46,6 +47,7 @@ ENTERPRISE_TARGETS=(
//enterprise/cmd/repo-updater
//enterprise/cmd/precise-code-intel-worker
//enterprise/cmd/server
//docker-images/syntax-highlighter:scip-ctags
)
MUSL_TARGETS=(

21
cmd/symbols/Dockerfile
View File

@ -1,11 +1,27 @@
# NOTE: This layer of the docker image is also used in local development as a wrapper around universal-ctags
FROM sourcegraph/alpine-3.14:213466_2023-04-17_5.0-bdda34a71619@sha256:6354a4ff578b685e36c8fbde81f62125ae0011b047fb2cc22d1b0de616b3c59a AS ctags
# hadolint ignore=DL3002
USER root
COPY cmd/symbols/ctags-install-alpine.sh /ctags-install-alpine.sh
RUN /ctags-install-alpine.sh
FROM rust:1.68.0-alpine3.17@sha256:d119a621ae12f84ec0c5fed77c24795120ed1c7874b2428b5a6ccc0f294dbe18 as scip-ctags
# hadolint ignore=DL3002
USER root
RUN apk add --no-cache musl-dev>=1.1.24-r10 build-base
COPY docker-images/syntax-highlighter /repo
WORKDIR /repo
RUN cargo fetch
ARG TARGETARCH
# Because .cargo/config.toml doesnt support triplet-specific env
COPY cmd/symbols/cargo-config.sh /cargo-config.sh
RUN /cargo-config.sh
RUN cargo rustc --release --bin scip-ctags
RUN cp ./target/release/scip-ctags /usr/local/bin/scip-ctags
FROM golang:1.19.8-alpine@sha256:841c160ed35923d96c95c52403c4e6db5decd9cbce034aa851e412ade5d4b74f AS symbols-build
# hadolint ignore=DL3002
USER root
@ -60,6 +76,7 @@ LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/comm
RUN apk add --no-cache bind-tools ca-certificates mailcap tini jansson libstdc++ libgcc
COPY --from=ctags /usr/local/bin/universal-ctags /usr/local/bin/universal-ctags
COPY --from=scip-ctags /usr/local/bin/scip-ctags /usr/local/bin/scip-ctags
COPY --from=symbols-build /symbols /usr/local/bin/symbols

3
cmd/symbols/Dockerfile.bazel
View File

@ -29,6 +29,9 @@ RUN apk add --no-cache bind-tools ca-certificates mailcap tini jansson libstdc++
COPY --from=ctags /usr/local/bin/universal-ctags /usr/local/bin/universal-ctags
# the scip binary and symbols was already built by bazel
# see cmd/symbols/build-bazel.sh where it is built and put in the context directory aka $OUTPUT for docker
COPY scip-ctags /usr/local/bin/scip-ctags
COPY symbols /usr/local/bin/symbols
# symbols is cgo, ensure we have the requisite dynamic libraries

27
cmd/symbols/build-bazel.sh
View File

@ -11,7 +11,7 @@ cleanup() {
}
trap cleanup EXIT
echo "--- bazel build"
echo "--- :bazel: bazel build for targets //cmd/symbols"
bazelrc=(
--bazelrc=.bazelrc
@ -23,7 +23,6 @@ if [[ ${CI:-""} == "true" ]]; then
)
fi
bazel "${bazelrc[@]}" \
build \
//cmd/symbols \
@ -39,9 +38,31 @@ out=$(
--config incompat-zig-linux-amd64 \
--output=files
)
cp "$out" "$OUTPUT"
cp -v "$out" "$OUTPUT"
# we can't build scip-ctags with symbols since the platform args conflict
# NOTE: cmd/symbols/cargo-config.sh sets some specific config when running on arm64
# since this bazel run typically runs on CI that config change isn't made
echo "--- :bazel: bazel build for target //docker-images/syntax-highlighter:scip-ctags"
bazel "${bazelrc[@]}" \
build //docker-images/syntax-highlighter:scip-ctags \
--stamp \
--workspace_status_command=./dev/bazel_stamp_vars.sh
out=$(
bazel "${bazelrc[@]}" \
cquery //docker-images/syntax-highlighter:scip-ctags \
--stamp \
--workspace_status_command=./dev/bazel_stamp_vars.sh \
--output=files
)
cp -v "$out" "$OUTPUT"
cp cmd/symbols/ctags-install-alpine.sh "$OUTPUT"
echo ":docker: context directory contains the following:"
ls -lah "$OUTPUT"
echo "--- :docker: docker build for symbols"
docker build -f cmd/symbols/Dockerfile.bazel -t "$IMAGE" "$OUTPUT" \
--progress=plain \
--build-arg COMMIT_SHA \

34
cmd/symbols/build-ctags.sh
View File

@ -1,6 +1,6 @@
#!/usr/bin/env bash
# This script builds the ctags image for local development.
# This script builds the ctags images for local development.
cd "$(dirname "${BASH_SOURCE[0]}")/../.."
set -eu
@ -9,12 +9,30 @@ set -eu
# image. See /dev/universal-ctags-dev.
if [[ "${CTAGS_COMMAND}" != "dev/universal-ctags-dev" ]]; then
echo "CTAGS_COMMAND set to custom executable. Building of Docker image not necessary."
exit 0
else
# Build ctags docker image for universal-ctags-dev
echo "Building universal-ctags docker image"
docker build -f cmd/symbols/Dockerfile -t ctags . \
--platform linux/amd64 \
--target=ctags \
--progress=plain
fi
# Build ctags docker image for universal-ctags-dev
echo "Building ctags docker image"
docker build -f cmd/symbols/Dockerfile -t ctags . \
--platform linux/amd64 \
--target=ctags \
--progress=plain
# If SCIP_CTAGS_COMMAND is set to a custom executable, we don't need to build the
# image. See /dev/scip-ctags-dev.
if [[ "${SCIP_CTAGS_COMMAND}" != "dev/scip-ctags-dev" ]]; then
echo "SCIP_CTAGS_COMMAND set to custom executable. Building of Docker image or Rust code not necessary."
else
if [[ "$(uname -m)" == "arm64" ]]; then
# build ctags with cargo; prevent x86-64 slowdown on mac
root="$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null
"$root"/dev/scip-ctags-install.sh
else
# Build ctags docker image for scip-ctags-dev
echo "Building scip-ctags docker image"
docker build -f cmd/symbols/Dockerfile -t scip-ctags . \
--platform linux/amd64 \
--target=scip-ctags \
--progress=plain
fi
fi

8
cmd/symbols/cargo-config.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/sh
if [ "${TARGETARCH}" = "arm64" ]; then
cat <<- FOE >> .cargo/config.toml
[env]
CFLAGS="-mno-outline-atomics"
FOE
fi;

2
cmd/symbols/ctags-install-alpine.sh
View File

@ -1,6 +1,6 @@
#!/bin/sh
# This script installs ctags within an alpine container.
# This script installs universal-ctags within an alpine container.
# Commit hash of github.com/universal-ctags/ctags.
# Last bumped 2022-04-04.

1
cmd/symbols/internal/api/BUILD.bazel generated
View File

@ -52,6 +52,7 @@ go_test(
"//cmd/symbols/internal/database/writer",
"//cmd/symbols/parser",
"//internal/api",
"//internal/ctags_config",
"//internal/database",
"//internal/diskcache",
"//internal/endpoint",

3
cmd/symbols/internal/api/handler_test.go
View File

@ -17,6 +17,7 @@ import (
symbolsdatabase "github.com/sourcegraph/sourcegraph/cmd/symbols/internal/database"
"github.com/sourcegraph/sourcegraph/cmd/symbols/internal/database/writer"
"github.com/sourcegraph/sourcegraph/cmd/symbols/parser"
"github.com/sourcegraph/sourcegraph/internal/ctags_config"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/diskcache"
"github.com/sourcegraph/sourcegraph/internal/endpoint"
@ -38,7 +39,7 @@ func TestHandler(t *testing.T) {
cache := diskcache.NewStore(tmpDir, "symbols", diskcache.WithBackgroundTimeout(20*time.Minute))
parserFactory := func() (ctags.Parser, error) {
parserFactory := func(source ctags_config.ParserType) (ctags.Parser, error) {
pathToEntries := map[string][]*ctags.Entry{
"a.js": {
{

5
cmd/symbols/parser/BUILD.bazel generated
View File

@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "parser",
srcs = [
"config.go",
"filtering_parser.go",
"observability.go",
"parser.go",
@ -13,10 +14,14 @@ go_library(
deps = [
"//cmd/symbols/fetcher",
"//cmd/symbols/types",
"//internal/conf",
"//internal/conf/conftypes",
"//internal/ctags_config",
"//internal/metrics",
"//internal/observation",
"//internal/search",
"//internal/search/result",
"//lib/codeintel/languages",
"//lib/errors",
"@com_github_inconshreveable_log15//:log15",
"@com_github_prometheus_client_golang//prometheus",

63
cmd/symbols/parser/config.go Normal file
View File

@ -0,0 +1,63 @@
package parser
import (
"fmt"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/ctags_config"
)
var parserConfig = ctags_config.ParserConfiguration{
Default: ctags_config.UniversalCtags,
Engine: map[string]ctags_config.ParserType{},
}
func init() {
// Validation only: Do NOT set any values in the configuration in this function.
conf.ContributeValidator(func(c conftypes.SiteConfigQuerier) (problems conf.Problems) {
configuration := c.SiteConfig().SyntaxHighlighting
if configuration == nil {
return
}
for _, engine := range configuration.Symbols.Engine {
if _, err := ctags_config.ParserNameToParserType(engine); err != nil {
problems = append(problems, conf.NewSiteProblem(fmt.Sprintf("Not a valid Symbols.Engine: `%s`.", engine)))
}
}
return
})
go func() {
conf.Watch(func() {
c := conf.Get()
configuration := c.SiteConfig().SyntaxHighlighting
// Set the defaults
parserConfig.Engine = make(map[string]ctags_config.ParserType)
for lang, engine := range ctags_config.BaseParserConfig.Engine {
parserConfig.Engine[lang] = engine
}
if configuration != nil {
for lang, engine := range configuration.Symbols.Engine {
if engine, err := ctags_config.ParserNameToParserType(engine); err != nil {
parserConfig.Engine[lang] = engine
}
}
}
})
}()
}
func GetParserType(language string) ctags_config.ParserType {
parserType, ok := parserConfig.Engine[language]
if !ok {
return parserConfig.Default
} else {
return parserType
}
}

49
cmd/symbols/parser/parser.go
View File

@ -15,9 +15,11 @@ import (
"github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher"
"github.com/sourcegraph/sourcegraph/cmd/symbols/types"
"github.com/sourcegraph/sourcegraph/internal/ctags_config"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/search"
"github.com/sourcegraph/sourcegraph/internal/search/result"
"github.com/sourcegraph/sourcegraph/lib/codeintel/languages"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -134,14 +136,29 @@ func min(a, b int) int {
return b
}
func (p *parser) handleParseRequest(ctx context.Context, symbolOrErrors chan<- SymbolOrError, parseRequest fetcher.ParseRequest, totalSymbols *uint32) (err error) {
func (p *parser) handleParseRequest(
ctx context.Context,
symbolOrErrors chan<- SymbolOrError,
parseRequest fetcher.ParseRequest,
totalSymbols *uint32,
) (err error) {
ctx, trace, endObservation := p.operations.handleParseRequest.With(ctx, &err, observation.Args{Attrs: []attribute.KeyValue{
attribute.String("path", parseRequest.Path),
attribute.Int("fileSize", len(parseRequest.Data)),
}})
defer endObservation(1, observation.Args{})
parser, err := p.parserFromPool(ctx)
language, found := languages.GetLanguage(parseRequest.Path, string(parseRequest.Data))
if !found {
return nil
}
source := GetParserType(language)
if source == ctags_config.NoCtags || source == ctags_config.UnknownCtags {
return nil
}
parser, err := p.parserFromPool(ctx, source)
if err != nil {
return err
}
@ -155,12 +172,12 @@ func (p *parser) handleParseRequest(ctx context.Context, symbolOrErrors chan<- S
}
if err == nil {
p.parserPool.Done(parser)
p.parserPool.Done(parser, source)
} else {
// Close parser and return nil to pool, indicating that the next receiver should create a new parser
log15.Error("Closing failed parser", "error", err)
parser.Close()
p.parserPool.Done(nil)
p.parserPool.Done(nil, source)
p.operations.parseFailed.Inc()
}
}()
@ -219,11 +236,15 @@ func (p *parser) handleParseRequest(ctx context.Context, symbolOrErrors chan<- S
return nil
}
func (p *parser) parserFromPool(ctx context.Context) (ctags.Parser, error) {
func (p *parser) parserFromPool(ctx context.Context, source ctags_config.ParserType) (ctags.Parser, error) {
if source == ctags_config.NoCtags || source == ctags_config.UnknownCtags {
return nil, errors.New("Should not pass NoCtags to this function")
}
p.operations.parseQueueSize.Inc()
defer p.operations.parseQueueSize.Dec()
parser, err := p.parserPool.Get(ctx)
parser, err := p.parserPool.Get(ctx, source)
if err != nil {
if err == context.DeadlineExceeded {
p.operations.parseQueueTimeouts.Inc()
@ -250,12 +271,20 @@ func shouldPersistEntry(e *ctags.Entry) bool {
return true
}
func SpawnCtags(logger log.Logger, ctagsConfig types.CtagsConfig) (ctags.Parser, error) {
func SpawnCtags(logger log.Logger, ctagsConfig types.CtagsConfig, source ctags_config.ParserType) (ctags.Parser, error) {
logger = logger.Scoped("ctags", "ctags processes")
options := ctags.Options{
Bin: ctagsConfig.Command,
PatternLengthLimit: ctagsConfig.PatternLengthLimit,
var options ctags.Options
if source == ctags_config.UniversalCtags {
options = ctags.Options{
Bin: ctagsConfig.UniversalCommand,
PatternLengthLimit: ctagsConfig.PatternLengthLimit,
}
} else {
options = ctags.Options{
Bin: ctagsConfig.ScipCommand,
PatternLengthLimit: ctagsConfig.PatternLengthLimit,
}
}
if ctagsConfig.LogErrors {
options.Info = std.NewLogger(logger, log.LevelInfo)

47
cmd/symbols/parser/parser_pool.go
View File

@ -4,29 +4,39 @@ import (
"context"
"github.com/sourcegraph/go-ctags"
"github.com/sourcegraph/sourcegraph/internal/ctags_config"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
type ParserFactory func() (ctags.Parser, error)
type ParserFactory func(ctags_config.ParserType) (ctags.Parser, error)
type parserPool struct {
newParser ParserFactory
pool chan ctags.Parser
pool map[ctags_config.ParserType]chan ctags.Parser
}
func NewParserPool(newParser ParserFactory, numParserProcesses int) (*parserPool, error) {
pool := make(chan ctags.Parser, numParserProcesses)
for i := 0; i < numParserProcesses; i++ {
parser, err := newParser()
if err != nil {
return nil, err
pool := make(map[ctags_config.ParserType]chan ctags.Parser)
// NOTE: We obviously don't make `NoCtags` available in the pool.
for _, parserType := range []ctags_config.ParserType{ctags_config.UniversalCtags, ctags_config.ScipCtags} {
pool[parserType] = make(chan ctags.Parser, numParserProcesses)
for i := 0; i < numParserProcesses; i++ {
parser, err := newParser(parserType)
if err != nil {
return nil, err
}
pool[parserType] <- parser
}
pool <- parser
}
return &parserPool{
parserPool := &parserPool{
newParser: newParser,
pool: pool,
}, nil
}
return parserPool, nil
}
// Get a parser from the pool. Once this parser is no longer in use, the Done method
@ -35,20 +45,27 @@ func NewParserPool(newParser ParserFactory, numParserProcesses int) (*parserPool
// the pool. This method always returns a non-nil parser with a nil error value.
//
// This method blocks until a parser is available or the given context is canceled.
func (p *parserPool) Get(ctx context.Context) (ctags.Parser, error) {
func (p *parserPool) Get(ctx context.Context, source ctags_config.ParserType) (ctags.Parser, error) {
if source == ctags_config.NoCtags || source == ctags_config.UnknownCtags {
return nil, errors.New("NoCtags is not a valid ParserType")
}
pool := p.pool[source]
select {
case parser := <-p.pool:
case parser := <-pool:
if parser != nil {
return parser, nil
}
return p.newParser()
return p.newParser(source)
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (p *parserPool) Done(parser ctags.Parser) {
p.pool <- parser
func (p *parserPool) Done(parser ctags.Parser, source ctags_config.ParserType) {
pool := p.pool[source]
pool <- parser
}

1
cmd/symbols/shared/BUILD.bazel generated
View File

@ -23,6 +23,7 @@ go_library(
"//internal/conf",
"//internal/conf/conftypes",
"//internal/conf/deploy",
"//internal/ctags_config",
"//internal/database",
"//internal/database/connections/live",
"//internal/debugserver",

9
cmd/symbols/shared/sqlite.go
View File

@ -19,6 +19,7 @@ import (
symbolparser "github.com/sourcegraph/sourcegraph/cmd/symbols/parser"
"github.com/sourcegraph/sourcegraph/cmd/symbols/types"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/ctags_config"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/diskcache"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
@ -46,7 +47,7 @@ func SetupSqlite(observationCtx *observation.Context, db database.DB, gitserverC
// anything that tries to open a SQLite database.
sqlite.Init()
if deploy.IsSingleBinary() && config.Ctags.Command == "" {
if deploy.IsSingleBinary() && config.Ctags.UniversalCommand == "" {
// app: ctags is not available
searchFunc := func(ctx context.Context, params search.SymbolsParameters) (result.Symbols, error) {
return nil, nil
@ -54,8 +55,8 @@ func SetupSqlite(observationCtx *observation.Context, db database.DB, gitserverC
return searchFunc, nil, []goroutine.BackgroundRoutine{}, "", nil
}
parserFactory := func() (ctags.Parser, error) {
return symbolparser.SpawnCtags(logger, config.Ctags)
parserFactory := func(source ctags_config.ParserType) (ctags.Parser, error) {
return symbolparser.SpawnCtags(logger, config.Ctags, source)
}
parserPool, err := symbolparser.NewParserPool(parserFactory, config.NumCtagsProcesses)
if err != nil {
@ -76,5 +77,5 @@ func SetupSqlite(observationCtx *observation.Context, db database.DB, gitserverC
cacheSizeBytes := int64(config.CacheSizeMB) * 1000 * 1000
cacheEvicter := janitor.NewCacheEvicter(evictionInterval, cache, cacheSizeBytes, janitor.NewMetrics(observationCtx))
return searchFunc, nil, []goroutine.BackgroundRoutine{cacheEvicter}, config.Ctags.Command, nil
return searchFunc, nil, []goroutine.BackgroundRoutine{cacheEvicter}, config.Ctags.UniversalCommand, nil
}

11
cmd/symbols/types/types.go
View File

@ -45,7 +45,8 @@ func LoadSqliteConfig(baseConfig env.BaseConfig, ctags CtagsConfig, repositoryFe
}
type CtagsConfig struct {
Command string
UniversalCommand string
ScipCommand string
PatternLengthLimit int
LogErrors bool
DebugLogs bool
@ -64,8 +65,14 @@ func LoadCtagsConfig(baseConfig env.BaseConfig) CtagsConfig {
ctagsCommandDefault = ""
}
scipCtagsCommandDefault := "scip-ctags"
if deploy.IsSingleBinary() {
scipCtagsCommandDefault = ""
}
return CtagsConfig{
Command: baseConfig.Get("CTAGS_COMMAND", ctagsCommandDefault, "ctags command (should point to universal-ctags executable compiled with JSON and seccomp support)"),
UniversalCommand: baseConfig.Get("CTAGS_COMMAND", ctagsCommandDefault, "ctags command (should point to universal-ctags executable compiled with JSON and seccomp support)"),
ScipCommand: baseConfig.Get("SCIP_CTAGS_COMMAND", scipCtagsCommandDefault, "scip-ctags command"),
PatternLengthLimit: baseConfig.GetInt("CTAGS_PATTERN_LENGTH_LIMIT", "250", "the maximum length of the patterns output by ctags"),
LogErrors: baseConfig.GetBool("LOG_CTAGS_ERRORS", logCtagsErrorsDefault, "log ctags errors"),
DebugLogs: false,

22
dev/scip-ctags-dev Executable file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env bash
# This script is a wrapper around `scip-ctags`.
#
# It checks if scip-ctags has been installed through ./dev/scip-ctags-install.sh or falls back to a dockerized version.
#
# To use your own `scip-ctags` binary instead of this wrapper in your local dev server, use
# `SCIP_CTAGS_COMMAND=path/to/ctags sg start`.
root="$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null
TARGET=$("$root/dev/scip-ctags-install.sh" which)
if [ ! -f "${TARGET}" ]; then
exec docker run --rm -i \
-a stdin -a stdout -a stderr \
--user guest \
--name=scip-ctags-$$ \
--entrypoint /usr/local/bin/scip-ctags \
scip-ctags "$@"
else
${TARGET} "$@"
fi

35
dev/scip-ctags-install.sh Executable file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euf -o pipefail
pushd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null
mkdir -p .bin
# TODO: add similar task to zoekt alpine
NAME="scip-ctags"
TARGET="$PWD/.bin/${NAME}"
if [ $# -ne 0 ]; then
if [ "$1" == "which" ]; then
echo "$TARGET"
exit 0
fi
fi
function ctrl_c() {
printf "[-] Installation cancelled.\n"
exit 1
}
trap ctrl_c INT
function build_scip_ctags {
cd docker-images/syntax-highlighter
cargo fetch
cargo build --bin scip-ctags --release
cp ./target/release/scip-ctags "$TARGET"
}
build_scip_ctags
popd >/dev/null

6
docker-images/syntax-highlighter/.gitignore vendored
View File

@ -1,3 +1,9 @@
target
languages/libraries/
.vscode/
bench_data/
**/flamegraph.svg
**/perf.data
**/perf.data.old

15
docker-images/syntax-highlighter/BUILD.bazel
View File

@ -12,6 +12,7 @@ rust_binary(
normal = True,
) + [
"//docker-images/syntax-highlighter/crates/sg-syntax:sg-syntax",
"//docker-images/syntax-highlighter/crates/scip-syntax:scip-syntax",
"//docker-images/syntax-highlighter/crates/scip-treesitter-languages:scip-treesitter-languages",
],
)
@ -31,3 +32,17 @@ rust_test(
normal_dev = True,
),
)
rust_binary(
name = "scip-ctags",
srcs = ["src/bin/scip-ctags.rs"],
aliases = aliases(),
proc_macro_deps = all_crate_deps(
proc_macro = True,
),
deps = all_crate_deps(
normal = True,
) + [
"//docker-images/syntax-highlighter/crates/scip-syntax:scip-syntax",
],
)

1732
docker-images/syntax-highlighter/Cargo.Bazel.lock generated
View File

File diff suppressed because it is too large Load Diff

345
docker-images/syntax-highlighter/Cargo.lock generated
View File

@ -17,6 +17,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "ansi_term"
version = "0.12.1"
@ -129,6 +135,18 @@ version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1"
[[package]]
name = "bitvec"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]]
name = "bumpalo"
version = "3.9.1"
@ -141,6 +159,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.0.73"
@ -153,6 +177,45 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "ciborium"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
[[package]]
name = "ciborium-ll"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "3.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
dependencies = [
"bitflags 1.3.2",
"clap_lex 0.2.4",
"indexmap",
"textwrap",
]
[[package]]
name = "clap"
version = "4.1.11"
@ -161,7 +224,7 @@ checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098"
dependencies = [
"bitflags 2.0.2",
"clap_derive",
"clap_lex",
"clap_lex 0.3.3",
"is-terminal",
"once_cell",
"strsim",
@ -181,6 +244,15 @@ dependencies = [
"syn",
]
[[package]]
name = "clap_lex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
dependencies = [
"os_str_bytes",
]
[[package]]
name = "clap_lex"
version = "0.3.3"
@ -239,6 +311,85 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
dependencies = [
"anes",
"atty",
"cast",
"ciborium",
"clap 3.2.23",
"criterion-plot",
"itertools",
"lazy_static",
"num-traits",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset 0.8.0",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
dependencies = [
"cfg-if",
]
[[package]]
name = "ctor"
version = "0.1.22"
@ -425,6 +576,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "funty"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]]
name = "futures"
version = "0.3.27"
@ -563,6 +720,12 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.11.2"
@ -716,12 +879,30 @@ dependencies = [
"windows-sys 0.45.0",
]
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
[[package]]
name = "js-sys"
version = "0.3.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
@ -829,6 +1010,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.16"
@ -907,7 +1097,7 @@ dependencies = [
"cc",
"cfg-if",
"libc",
"memoffset",
"memoffset 0.6.5",
]
[[package]]
@ -919,6 +1109,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.13.1"
@ -940,9 +1139,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.13.0"
version = "1.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
[[package]]
name = "onig"
@ -966,6 +1165,12 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "oorandom"
version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "os_str_bytes"
version = "6.5.0"
@ -1073,6 +1278,34 @@ dependencies = [
"xml-rs",
]
[[package]]
name = "plotters"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
[[package]]
name = "plotters-svg"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
dependencies = [
"plotters-backend",
]
[[package]]
name = "ppv-lite86"
version = "0.2.16"
@ -1172,6 +1405,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "radium"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
[[package]]
name = "radix_trie"
version = "0.2.1"
@ -1212,6 +1451,28 @@ dependencies = [
"getrandom",
]
[[package]]
name = "rayon"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"num_cpus",
]
[[package]]
name = "redox_syscall"
version = "0.2.13"
@ -1485,14 +1746,19 @@ name = "scip-syntax"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"bitvec",
"clap 4.1.11",
"insta",
"itertools",
"once_cell",
"protobuf",
"rustc-hash",
"scip",
"scip-macros",
"scip-treesitter",
"scip-treesitter-languages",
"serde",
"serde_json",
"tree-sitter",
"walkdir",
]
@ -1812,13 +2078,17 @@ dependencies = [
name = "syntect_server"
version = "1.0.1"
dependencies = [
"clap",
"base64",
"clap 4.1.11",
"criterion",
"futures",
"futures-task",
"futures-util",
"protobuf",
"rocket",
"rustyline",
"scip",
"scip-syntax",
"scip-treesitter",
"scip-treesitter-languages",
"serde",
@ -1827,6 +2097,12 @@ dependencies = [
"syntect",
]
[[package]]
name = "tap"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "tempfile"
version = "3.3.0"
@ -1850,6 +2126,12 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "textwrap"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
[[package]]
name = "thiserror"
version = "1.0.30"
@ -1928,6 +2210,16 @@ dependencies = [
"syn",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tokio"
version = "1.17.0"
@ -2378,9 +2670,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.79"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@ -2388,13 +2680,13 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.79"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"once_cell",
"proc-macro2",
"quote",
"syn",
@ -2403,9 +2695,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.79"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@ -2413,9 +2705,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.79"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
dependencies = [
"proc-macro2",
"quote",
@ -2426,9 +2718,19 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.79"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
[[package]]
name = "web-sys"
version = "0.3.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi"
@ -2585,6 +2887,15 @@ version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
name = "wyz"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed"
dependencies = [
"tap",
]
[[package]]
name = "xml-rs"
version = "0.8.4"

9
docker-images/syntax-highlighter/Cargo.toml
View File

@ -15,12 +15,15 @@ scip.workspace = true
serde.workspace = true
serde_json.workspace = true
syntect.workspace = true
protobuf.workspace = true
rustyline = "9.1.2"
base64 = "0.13.0"
sg-syntax = { path = "./crates/sg-syntax" }
scip-treesitter = { path = "./crates/scip-treesitter" }
scip-treesitter-languages = { path = "./crates/scip-treesitter-languages" }
scip-syntax = { path = "./crates/scip-syntax" }
# March 20, 2023 - Pinned explicitly with features that match the features
# required by rocket. Once bazel rules correctly roll up all the features,
# we can remove this, but until then, this works just fine for building
@ -38,12 +41,12 @@ members = [
"crates/scip-syntax",
"crates/scip-treesitter",
"crates/scip-treesitter-languages",
# "crates/ctags",
]
[workspace.dependencies]
anyhow = "1"
clap = { version = "4.1.11", features = [ "derive" ] }
itertools = "0.10.5"
rocket = { version = "0.5.0-rc.1", features = ["json"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
@ -57,7 +60,11 @@ scip = { git = "https://github.com/sourcegraph/scip", rev="3856df76147ca4b86df78
protobuf = "3"
[profile.release]
# Enabled debug symbols in release build, so if we have a crash
# we can inspect the coredump.
debug = true
[dev-dependencies]
criterion = { version = "0.4", features = [ "html_reports" ] }

10
docker-images/syntax-highlighter/crates/ctags/Cargo.toml
View File

@ -1,10 +0,0 @@
[package]
name = "ctags"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
scip.workspace = true
anyhow.workspace = true

92
docker-images/syntax-highlighter/crates/ctags/src/lib.rs
View File

@ -1,92 +0,0 @@
// use anyhow::Result;
// Some existing symbol structs
//
// zoekt
// type Symbol struct {
// Sym string
// Kind string
// Parent string
// ParentKind string
// }
//
// go-ctags
// type Entry struct {
// Name string
// Path string
// Line int
// Kind string
// Language string
// Parent string
// ParentKind string
// Pattern string
// Signature string
//
// FileLimited bool
// }
// LSP Symbol Kinds, could be fine for us to use now
// export namespace SymbolKind {
// export const File = 1;
// export const Module = 2;
// export const Namespace = 3;
// export const Package = 4;
// export const Class = 5;
// export const Method = 6;
// export const Property = 7;
// export const Field = 8;
// export const Constructor = 9;
// export const Enum = 10;
// export const Interface = 11;
// export const Function = 12;
// export const Variable = 13;
// export const Constant = 14;
// export const String = 15;
// export const Number = 16;
// export const Boolean = 17;
// export const Array = 18;
// export const Object = 19;
// export const Key = 20;
// export const Null = 21;
// export const EnumMember = 22;
// export const Struct = 23;
// export const Event = 24;
// export const Operator = 25;
// export const TypeParameter = 26;
// }
use scip::types::{Descriptor, Document};
#[derive(Debug)]
pub enum TagKind {
Function,
Class,
}
impl std::str::FromStr for TagKind {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"definition.function" => Ok(Self::Function),
"definition.type" => Ok(Self::Function),
_ => anyhow::bail!("unknown tag kind: {}", s),
}
}
}
#[derive(Debug)]
pub struct TagEntry {
pub descriptors: Vec<Descriptor>,
pub kind: TagKind,
pub parent: Option<Box<TagEntry>>,
pub line: usize,
// pub column: usize,
}
impl TagEntry {
pub fn from_document(document: Document) -> Vec<TagEntry> {
todo!("{:?}", document)
}
}

5
docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml
View File

@ -8,9 +8,12 @@ edition = "2021"
[dependencies]
anyhow.workspace = true
clap.workspace = true
itertools.workspace = true
protobuf.workspace = true
scip.workspace = true
tree-sitter.workspace = true
serde.workspace = true
serde_json.workspace = true
scip-macros = { path = "../scip-macros" }
scip-treesitter = { path = "../scip-treesitter" }
@ -19,3 +22,5 @@ scip-treesitter-languages = { path = "../scip-treesitter-languages" }
rustc-hash = "1.1.0"
walkdir = "2"
insta = "*"
once_cell = "1.17.1"
bitvec = "1.0.1"

11
docker-images/syntax-highlighter/crates/scip-syntax/queries/c/scip-tags.scm Normal file
View File

@ -0,0 +1,11 @@
; Make use of @local
(translation_unit (declaration (init_declarator declarator: (_) @descriptor.term)))
(enum_specifier name: (_) @descriptor.type body: (enumerator_list (enumerator name: (_) @descriptor.term)) @descriptor.scope)
(field_declaration declarator: [
(pointer_declarator (field_identifier) @descriptor.term)
(field_identifier) @descriptor.term
])
(function_definition (function_declarator declarator: (_) @descriptor.method))

11
docker-images/syntax-highlighter/crates/scip-syntax/queries/c_sharp/scip-tags.scm Normal file
View File

@ -0,0 +1,11 @@
(class_declaration name: (_) @descriptor.type) @scope
(interface_declaration name: (_) @descriptor.type) @scope
(enum_declaration name: (_) @descriptor.type) @scope
(namespace_declaration name: (_) @descriptor.type) @scope
(method_declaration name: (_) @descriptor.method)
(constructor_declaration name: (_) @descriptor.method)
(field_declaration (variable_declaration (variable_declarator (identifier) @descriptor.term)))
(property_declaration (identifier) @descriptor.term)
(enum_member_declaration name: (_) @descriptor.term)

10
docker-images/syntax-highlighter/crates/scip-syntax/queries/cpp/scip-tags.scm Normal file
View File

@ -0,0 +1,10 @@
; Make use of @local
(translation_unit (declaration (init_declarator declarator: (_) @descriptor.term)))
(namespace_definition name: (_) @descriptor.type body: (_) @descriptor.scope)
(class_specifier name: (_) @descriptor.type body: (_) @descriptor.scope)
(enum_specifier name: (_) @descriptor.type body: (enumerator_list (enumerator name: (_) @descriptor.term)) @descriptor.scope)
(field_declaration declarator: (_) @descriptor.term)
(function_definition (function_declarator declarator: (_) @descriptor.method))

9
docker-images/syntax-highlighter/crates/scip-syntax/queries/go/scip-tags.scm
View File

@ -1,4 +1,4 @@
(source_file (package_clause (package_identifier) @descriptor.namespace) @scope)
(source_file (package_clause (package_identifier) @descriptor.namespace)) @scope
(function_declaration
name: (identifier) @descriptor.method)
@ -15,4 +15,9 @@
(parameter_declaration type: (type_identifier) @descriptor.type))
name: (field_identifier) @descriptor.method)
(type_declaration (type_spec name: (type_identifier) @descriptor.type))
(type_declaration (type_spec name: (type_identifier) @descriptor.type)) @scope
(field_declaration_list (field_declaration name: (_) @descriptor.term))
(const_spec name: (_) @descriptor.term)
(import_spec name: (_) @descriptor.term)

16
docker-images/syntax-highlighter/crates/scip-syntax/queries/java/scip-tags.scm Normal file
View File

@ -0,0 +1,16 @@
(program
(package_declaration
(scoped_identifier)
@descriptor.namespace
)
) @scope
(class_declaration name: (_) @descriptor.type) @scope
(interface_declaration name: (_) @descriptor.type) @scope
(enum_declaration name: (_) @descriptor.type) @scope
(method_declaration name: (_) @descriptor.method) @local
(constructor_declaration name: (_) @descriptor.method) @local
(field_declaration (variable_declarator name: (_) @descriptor.term))
(enum_constant name: (_) @descriptor.term)

22
docker-images/syntax-highlighter/crates/scip-syntax/queries/javascript/scip-tags.scm Normal file
View File

@ -0,0 +1,22 @@
(namespace_import (identifier) @descriptor.term)
(named_imports
[
(import_specifier alias: (_) @descriptor.term)
(import_specifier name: (_) @descriptor.term !alias)
]
)
(function_declaration (identifier) @descriptor.method body: (_) @local)
(lexical_declaration (variable_declarator name: (identifier) @descriptor.term))
(variable_declaration (variable_declarator name: (identifier) @descriptor.term))
(class_declaration name: (_) @descriptor.type body: (_) @scope)
(class_declaration
(class_body
[
(method_definition name: (_) @descriptor.method body: (_) @local)
]
)
)
[(if_statement) (while_statement) (for_statement) (do_statement)] @local

14
docker-images/syntax-highlighter/crates/scip-syntax/queries/kotlin/scip-tags.scm Normal file
View File

@ -0,0 +1,14 @@
(source_file
(package_header
(identifier)
@descriptor.namespace
)
) @scope
(function_declaration (simple_identifier) @descriptor.method) @local
(anonymous_function (_ (type_identifier) @descriptor.type . (type_identifier) @descriptor.method)) @local
(class_declaration (type_identifier) @descriptor.type)
(object_declaration (type_identifier) @descriptor.type)
(class_parameter (simple_identifier) @descriptor.term)
(enum_entry (simple_identifier) @descriptor.term)
(property_declaration (variable_declaration (simple_identifier) @descriptor.term))

8
docker-images/syntax-highlighter/crates/scip-syntax/queries/python/scip-tags.scm Normal file
View File

@ -0,0 +1,8 @@
(import_statement name: (_) @descriptor.term)
(import_from_statement name: (_) @descriptor.term)
(class_definition name: (_) @descriptor.type body: (_) @scope)
(function_definition name: (_) @descriptor.method body: (_) @local)
(expression_statement (assignment left: (identifier) @descriptor.term))
[(if_statement) (while_statement) (for_statement) (with_statement)] @local

4
docker-images/syntax-highlighter/crates/scip-syntax/queries/ruby/scip-tags.scm Normal file
View File

@ -0,0 +1,4 @@
(assignment left: [(identifier) (constant)] @descriptor.term)
(class name: (_) @descriptor.type) @scope
(method name: (_) @descriptor.method) @local
[(block) (unless)] @local

5
docker-images/syntax-highlighter/crates/scip-syntax/queries/rust/scip-tags.scm
View File

@ -7,16 +7,15 @@
name: (_) @descriptor.type) @scope
(impl_item
trait: (_) @descriptor.type
trait: (_)? @descriptor.type
type: (_) @descriptor.type) @scope
;; TODO: @local to stop traversal
(function_signature_item
name: (identifier) @descriptor.method)
;; TODO: @local to stop traversal
(function_item
name: (identifier) @descriptor.method)
name: (identifier) @descriptor.method body: (_) @local)
(struct_item
name: (type_identifier) @descriptor.type) @scope

19
docker-images/syntax-highlighter/crates/scip-syntax/queries/scala/scip-tags.scm Normal file
View File

@ -0,0 +1,19 @@
; TODO: Exclude functions in non-container blocks
(compilation_unit
(package_clause
(package_identifier)
@descriptor.namespace
)
) @scope
(compilation_unit (val_definition (identifier) @descriptor.term))
(compilation_unit (var_definition (identifier) @descriptor.term))
(template_body (val_definition (identifier) @descriptor.term))
(template_body (var_definition (identifier) @descriptor.term))
(function_definition (identifier) @descriptor.method)
(class_definition (identifier) @descriptor.type (template_body) @scope)
(object_definition (identifier) @descriptor.type (template_body) @scope)
(trait_definition (identifier) @descriptor.type (template_body) @scope)

42
docker-images/syntax-highlighter/crates/scip-syntax/queries/typescript/scip-tags.scm Normal file
View File

@ -0,0 +1,42 @@
(namespace_import (identifier) @descriptor.term)
(named_imports
[
(import_specifier alias: (_) @descriptor.term)
(import_specifier name: (_) @descriptor.term !alias)
]
)
(function_declaration (identifier) @descriptor.method body: (_) @local)
(lexical_declaration (variable_declarator name: (identifier) @descriptor.term))
(variable_declaration (variable_declarator name: (identifier) @descriptor.term))
(interface_declaration name: (_) @descriptor.type body: (_) @scope)
(interface_declaration
(object_type
[
(method_signature (property_identifier) @descriptor.method)
(property_signature (property_identifier) @descriptor.term)
]
)
)
(class_declaration name: (_) @descriptor.type body: (_) @scope)
(class_declaration
(class_body
[
(public_field_definition name: (_) @descriptor.term)
(method_definition name: (_) @descriptor.method body: (_) @local)
]
)
)
(enum_declaration name: (_) @descriptor.type body: (_) @scope)
(enum_declaration
(enum_body
(property_identifier) @descriptor.term
)
)
(module name: (string (string_fragment) @descriptor.namespace) body: (_) @scope)
[(if_statement) (while_statement) (for_statement) (do_statement)] @local

17
docker-images/syntax-highlighter/crates/scip-syntax/queries/zig/scip-tags.scm Normal file
View File

@ -0,0 +1,17 @@
(Decl
(VarDecl
variable_type_function: (_)
@descriptor.term
(
(ErrorUnionExpr
(SuffixExpr
(ContainerDecl)
)
)
@scope
)?
)
)
(Decl (FnProto function: (_) @descriptor.method)) @local
(ContainerField field_member: (IDENTIFIER) @descriptor.term)

307
docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs Normal file
View File

@ -0,0 +1,307 @@
use std::{
collections::HashMap,
io::{BufRead, BufReader, BufWriter, Read, Write},
ops::Not,
path,
};
use anyhow::{Context, Result};
use itertools::intersperse;
use scip::types::{descriptor::Suffix, Descriptor};
use scip_treesitter_languages::parsers::BundledParser;
use serde::{Deserialize, Serialize};
use crate::{get_globals, globals::Scope};
#[derive(Serialize, Deserialize, Debug)]
#[serde(tag = "command", rename_all = "kebab-case")]
pub enum Request {
GenerateTags { filename: String, size: usize },
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(tag = "_type", rename_all = "kebab-case")]
pub enum Reply<'a> {
Program {
name: String,
version: String,
},
Completed {
command: String,
},
Error {
message: String,
fatal: bool,
},
Tag {
name: String,
path: &'a str,
language: &'a str,
/// Starts at 1
line: usize,
kind: &'a str,
scope: Option<&'a str>,
// Can't find any uses of these. If someone reports a bug, we can support this
// scope_kind: Option<String>,
// signature: Option<String>,
},
}
impl<'a> Reply<'a> {
pub fn write<W: std::io::Write>(self, writer: &mut W) {
writer
.write_all(serde_json::to_string(&self).unwrap().as_bytes())
.unwrap();
writer.write_all("\n".as_bytes()).unwrap();
}
pub fn write_tag<W: std::io::Write>(
writer: &mut W,
scope: &Scope,
path: &'a str,
language: &'a str,
tag_scope: Option<&'a str>,
scope_deduplicator: &mut HashMap<String, ()>,
) {
let descriptors = &scope.descriptors;
let names = descriptors.iter().map(|d| d.name.as_str());
let name = intersperse(names, ".").collect::<String>();
let mut dedup = match tag_scope {
Some(ts) => vec![ts],
None => vec![],
};
dedup.push(&name);
let dedup = dedup.join(".");
if scope_deduplicator.contains_key(&dedup) {
return;
}
scope_deduplicator.insert(dedup, ());
let tag = Self::Tag {
name,
path,
language,
line: scope.scope_range.start_line as usize + 1,
kind: descriptors_to_kind(&scope.descriptors),
scope: tag_scope,
};
tag.write(writer);
}
}
fn descriptors_to_kind(descriptors: &[Descriptor]) -> &'static str {
match descriptors
.last()
.unwrap_or_default()
.suffix
.enum_value_or_default()
{
Suffix::Namespace => "namespace",
Suffix::Package => "package",
Suffix::Method => "method",
Suffix::Type => "type",
_ => "variable",
}
}
fn emit_tags_for_scope<W: std::io::Write>(
buf_writer: &mut BufWriter<W>,
path: &str,
parent_scopes: Vec<String>,
scope: &Scope,
language: &str,
scope_deduplicator: &mut HashMap<String, ()>,
) {
let curr_scopes = {
let mut curr_scopes = parent_scopes.clone();
for desc in &scope.descriptors {
curr_scopes.push(desc.name.clone());
}
curr_scopes
};
if !scope.descriptors.is_empty() {
let tag_scope = parent_scopes
.is_empty()
.not()
.then(|| parent_scopes.join("."));
let tag_scope = tag_scope.as_deref();
Reply::write_tag(
&mut *buf_writer,
scope,
path,
language,
tag_scope,
scope_deduplicator,
);
}
for subscope in &scope.children {
emit_tags_for_scope(
buf_writer,
path,
curr_scopes.clone(),
subscope,
language,
scope_deduplicator,
);
}
for global in &scope.globals {
let mut scope_name = curr_scopes.clone();
scope_name.extend(
global
.descriptors
.iter()
.take(global.descriptors.len() - 1)
.map(|d| d.name.clone()),
);
Reply::Tag {
name: global.descriptors.last().unwrap().name.clone(),
path,
language,
line: global.range.start_line as usize + 1,
kind: descriptors_to_kind(&global.descriptors),
scope: scope_name
.is_empty()
.not()
.then(|| scope_name.join("."))
.as_deref(),
}
.write(buf_writer);
}
}
pub fn generate_tags<W: std::io::Write>(
buf_writer: &mut BufWriter<W>,
filename: String,
file_data: &[u8],
) -> Option<()> {
let path = path::Path::new(&filename);
let extension = path.extension()?.to_str()?;
let filepath = path.file_name()?.to_str()?;
let parser = BundledParser::get_parser_from_extension(extension)?;
let (root_scope, _) = match get_globals(&parser, file_data)? {
Ok(vals) => vals,
Err(err) => {
// TODO: Not sure I want to keep this or not
#[cfg(debug_assertions)]
if true {
panic!("Could not parse file: {}", err);
}
return None;
}
};
let mut scope_deduplicator = HashMap::new();
emit_tags_for_scope(
buf_writer,
filepath,
vec![],
&root_scope,
// I don't believe the language name is actually used anywhere but we'll
// keep it to be compliant with the ctags spec
parser.get_language_name(),
&mut scope_deduplicator,
);
Some(())
}
pub fn ctags_runner<R: Read, W: Write>(
input: &mut BufReader<R>,
output: &mut std::io::BufWriter<W>,
) -> Result<()> {
Reply::Program {
name: "SCIP Ctags".to_string(),
version: "5.9.0".to_string(),
}
.write(output);
output.flush().unwrap();
loop {
let mut line = String::new();
input.read_line(&mut line)?;
if line.is_empty() {
break;
}
let request = serde_json::from_str::<Request>(&line);
let request = match request {
Ok(request) => request,
Err(_) => {
eprintln!("Could not parse request: {}", line);
continue;
}
};
match request {
Request::GenerateTags { filename, size } => {
let mut file_data = vec![0; size];
input
.read_exact(&mut file_data)
.expect("Could not fill file data exactly");
generate_tags(output, filename, &file_data);
}
}
Reply::Completed {
command: "generate-tags".to_string(),
}
.write(output);
output.flush().unwrap();
}
Ok(())
}
pub fn helper_execute_one_file(name: &str, contents: &str) -> Result<String> {
let command = format!(
r#"
{{ "command":"generate-tags","filename":"{}","size":{} }}
{}
"#,
name,
contents.len(),
contents
)
.trim()
.to_string();
let mut input = BufReader::new(command.as_bytes());
let mut output = BufWriter::new(Vec::new());
ctags_runner(&mut input, &mut output)?;
String::from_utf8(output.get_ref().to_vec()).context("Could not parse output")
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_ctags_runner_basic() -> Result<()> {
let file = r#"
fn main() {
println!("Hello, world!");
}
fn something() -> bool { true }
fn other() -> bool { false }
"#
.trim();
let output = helper_execute_one_file("main.rs", file)?;
insta::assert_snapshot!(output);
Ok(())
}
}

305
docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs Normal file
View File

@ -0,0 +1,305 @@
use anyhow::Result;
use bitvec::prelude::*;
use protobuf::Enum;
use scip::types::{Descriptor, Occurrence};
use scip_treesitter::types::PackedRange;
use crate::languages::TagConfiguration;
#[derive(Debug)]
pub struct Scope {
pub ident_range: PackedRange,
pub scope_range: PackedRange,
pub globals: Vec<Global>,
pub children: Vec<Scope>,
pub descriptors: Vec<Descriptor>,
}
#[derive(Debug)]
pub struct Global {
pub range: PackedRange,
pub descriptors: Vec<Descriptor>,
}
impl Scope {
pub fn insert_scope(&mut self, scope: Scope) {
if let Some(child) = self
.children
.iter_mut()
.find(|child| child.scope_range.contains(&scope.scope_range))
{
child.insert_scope(scope);
} else {
self.children.push(scope);
}
}
pub fn insert_global(&mut self, global: Global) {
if let Some(child) = self
.children
.iter_mut()
.find(|child| child.scope_range.contains(&global.range))
{
child.insert_global(global)
} else {
self.globals.push(global);
}
}
pub fn into_occurrences(
&mut self,
hint: usize,
base_descriptors: Vec<Descriptor>,
) -> Vec<Occurrence> {
let mut descriptor_stack = base_descriptors;
let mut occs = Vec::with_capacity(hint);
self.rec_into_occurrences(true, &mut occs, &mut descriptor_stack);
occs
}
fn rec_into_occurrences(
&self,
is_root: bool,
occurrences: &mut Vec<Occurrence>,
descriptor_stack: &mut Vec<Descriptor>,
) {
descriptor_stack.extend(self.descriptors.clone());
if !is_root {
occurrences.push(scip::types::Occurrence {
range: self.ident_range.to_vec(),
symbol: scip::symbol::format_symbol(scip::types::Symbol {
scheme: "scip-ctags".into(),
// TODO: Package?
package: None.into(),
descriptors: descriptor_stack.clone(),
..Default::default()
}),
symbol_roles: scip::types::SymbolRole::Definition.value(),
// TODO:
// syntax_kind: todo!(),
..Default::default()
});
}
for global in &self.globals {
let mut global_descriptors = descriptor_stack.clone();
global_descriptors.extend(global.descriptors.clone());
let symbol = scip::symbol::format_symbol(scip::types::Symbol {
scheme: "scip-ctags".into(),
// TODO: Package?
package: None.into(),
descriptors: global_descriptors,
..Default::default()
});
let symbol_roles = scip::types::SymbolRole::Definition.value();
occurrences.push(scip::types::Occurrence {
range: global.range.to_vec(),
symbol,
symbol_roles,
// TODO:
// syntax_kind: todo!(),
..Default::default()
});
}
self.children
.iter()
.for_each(|c| c.rec_into_occurrences(false, occurrences, descriptor_stack));
self.descriptors.iter().for_each(|_| {
descriptor_stack.pop();
});
}
}
pub fn parse_tree<'a>(
config: &TagConfiguration,
tree: &'a tree_sitter::Tree,
source_bytes: &'a [u8],
) -> Result<(Scope, usize)> {
let mut cursor = tree_sitter::QueryCursor::new();
let root_node = tree.root_node();
let capture_names = config.query.capture_names();
let mut scopes = vec![];
let mut globals = vec![];
let mut local_ranges = BitVec::<u8, Msb0>::repeat(false, source_bytes.len());
let matches = cursor.matches(&config.query, root_node, source_bytes);
for m in matches {
// eprintln!("\n==== NEW MATCH ====");
let mut node = None;
let mut scope = None;
let mut local_range = None;
let mut descriptors = vec![];
for capture in m.captures {
let capture_name = capture_names
.get(capture.index as usize)
.expect("capture indexes should always work");
if capture_name.starts_with("descriptor") {
descriptors.push((capture_name, capture.node.utf8_text(source_bytes)?));
node = Some(capture.node);
}
if capture_name.starts_with("scope") {
assert!(scope.is_none(), "declare only one scope per match");
scope = Some(capture);
}
if capture_name.starts_with("local") {
local_range = Some(capture.node.byte_range());
}
// eprintln!(
// "{}: {}",
// capture_name,
// capture.node.utf8_text(source_bytes).unwrap()
// );
}
match node {
Some(node) => {
if local_ranges[node.start_byte()] {
continue;
}
let descriptors = descriptors
.iter()
.map(|(capture, name)| {
crate::ts_scip::capture_name_to_descriptor(capture, name.to_string())
})
.collect();
// dbg!(node);
match scope {
Some(scope_ident) => scopes.push(Scope {
ident_range: node.into(),
scope_range: scope_ident.node.into(),
globals: vec![],
children: vec![],
descriptors,
}),
None => globals.push(Global {
range: node.into(),
descriptors,
}),
}
}
None => {
if local_range.is_none() {
panic!("there must always be at least one descriptor (except for @local)");
}
}
}
if let Some(local_range) = local_range {
local_ranges.get_mut(local_range).unwrap().fill(true);
}
}
let mut root = Scope {
ident_range: root_node.into(),
scope_range: root_node.into(),
globals: vec![],
children: vec![],
descriptors: vec![],
};
scopes.sort_by_key(|m| {
std::cmp::Reverse((
m.scope_range.start_line,
m.scope_range.end_line,
m.scope_range.start_col,
))
});
// Add all the scopes to our tree
while let Some(m) = scopes.pop() {
root.insert_scope(m);
}
while let Some(m) = globals.pop() {
root.insert_global(m);
}
Ok((root, globals.len()))
}
#[cfg(test)]
mod test {
use scip::types::Document;
use scip_treesitter::snapshot::dump_document;
use tree_sitter::Parser;
use super::*;
fn parse_file_for_lang(config: &TagConfiguration, source_code: &str) -> Result<Document> {
let source_bytes = source_code.as_bytes();
let mut parser = Parser::new();
parser.set_language(config.language).unwrap();
let tree = parser.parse(source_bytes, None).unwrap();
let mut occ = parse_tree(config, &tree, source_bytes)?;
let mut doc = Document::new();
doc.occurrences = occ.0.into_occurrences(occ.1, vec![]);
doc.symbols = doc
.occurrences
.iter()
.map(|o| scip::types::SymbolInformation {
symbol: o.symbol.clone(),
..Default::default()
})
.collect();
Ok(doc)
}
#[test]
fn test_can_parse_rust_tree() -> Result<()> {
let config = crate::languages::rust();
let source_code = include_str!("../testdata/scopes.rs");
let doc = parse_file_for_lang(config, source_code)?;
let dumped = dump_document(&doc, source_code)?;
insta::assert_snapshot!(dumped);
Ok(())
}
#[test]
fn test_can_parse_go_tree() -> Result<()> {
let config = crate::languages::go();
let source_code = include_str!("../testdata/example.go");
let doc = parse_file_for_lang(config, source_code)?;
// dbg!(doc);
let dumped = dump_document(&doc, source_code)?;
insta::assert_snapshot!(dumped);
Ok(())
}
#[test]
fn test_can_parse_go_internal_tree() -> Result<()> {
let config = crate::languages::go();
let source_code = include_str!("../testdata/internal_go.go");
let doc = parse_file_for_lang(config, source_code)?;
// dbg!(doc);
let dumped = dump_document(&doc, source_code)?;
insta::assert_snapshot!(dumped);
Ok(())
}
}

254
docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs
View File

@ -1,3 +1,4 @@
use once_cell::sync::OnceCell;
use scip_macros::include_scip_query;
use scip_treesitter_languages::parsers::BundledParser;
use tree_sitter::{Language, Parser, Query};
@ -5,35 +6,227 @@ use tree_sitter::{Language, Parser, Query};
pub struct TagConfiguration {
pub language: Language,
pub query: Query,
pub parser: Parser,
}
pub fn rust() -> TagConfiguration {
let language = BundledParser::Rust.get_language();
let query = include_scip_query!("rust", "scip-tags");
pub fn c() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
let mut parser = Parser::new();
parser.set_language(language).unwrap();
INSTANCE.get_or_init(|| {
let language = BundledParser::C.get_language();
let query = include_scip_query!("c", "scip-tags");
TagConfiguration {
language,
parser,
query: Query::new(language, query).unwrap(),
}
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn go() -> TagConfiguration {
let language = BundledParser::Go.get_language();
let query = include_scip_query!("go", "scip-tags");
pub fn javascript() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
let mut parser = Parser::new();
parser.set_language(language).unwrap();
INSTANCE.get_or_init(|| {
let language = BundledParser::Javascript.get_language();
let query = include_scip_query!("javascript", "scip-tags");
TagConfiguration {
language,
parser,
query: Query::new(language, query).unwrap(),
}
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn kotlin() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Kotlin.get_language();
let query = include_scip_query!("kotlin", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn ruby() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Ruby.get_language();
let query = include_scip_query!("ruby", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn python() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Python.get_language();
let query = include_scip_query!("python", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn cpp() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Cpp.get_language();
let query = include_scip_query!("cpp", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn typescript() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Typescript.get_language();
let query = include_scip_query!("typescript", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn scala() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Scala.get_language();
let query = include_scip_query!("scala", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn c_sharp() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::C_Sharp.get_language();
let query = include_scip_query!("c_sharp", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn java() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Java.get_language();
let query = include_scip_query!("java", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn rust() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Rust.get_language();
let query = include_scip_query!("rust", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn go() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Go.get_language();
let query = include_scip_query!("go", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub fn zig() -> &'static TagConfiguration {
static INSTANCE: OnceCell<TagConfiguration> = OnceCell::new();
INSTANCE.get_or_init(|| {
let language = BundledParser::Zig.get_language();
let query = include_scip_query!("zig", "scip-tags");
let mut parser = Parser::new();
parser.set_language(language).unwrap();
TagConfiguration {
language,
query: Query::new(language, query).unwrap(),
}
})
}
pub struct LocalConfiguration {
@ -70,6 +263,25 @@ fn perl_locals() -> LocalConfiguration {
}
}
pub fn get_tag_configuration(parser: &BundledParser) -> Option<&'static TagConfiguration> {
match parser {
BundledParser::C => Some(c()),
BundledParser::Javascript => Some(javascript()),
BundledParser::Kotlin => Some(kotlin()),
BundledParser::Ruby => Some(ruby()),
BundledParser::Python => Some(python()),
BundledParser::Cpp => Some(cpp()),
BundledParser::Typescript => Some(typescript()),
BundledParser::Scala => Some(scala()),
BundledParser::C_Sharp => Some(c_sharp()),
BundledParser::Java => Some(java()),
BundledParser::Rust => Some(rust()),
BundledParser::Go => Some(go()),
BundledParser::Zig => Some(zig()),
_ => None,
}
}
pub fn get_local_configuration(parser: BundledParser) -> Option<LocalConfiguration> {
match parser {
BundledParser::Go => Some(go_locals()),

63
docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs
View File

@ -1,14 +1,75 @@
use anyhow::Result;
use scip::types::Occurrence;
use scip_treesitter_languages::parsers::BundledParser;
use tree_sitter::Parser;
pub mod ctags;
pub mod globals;
pub mod languages;
pub mod locals;
pub mod matches;
pub mod ts_scip;
pub fn get_globals(
parser: &BundledParser,
source_bytes: &[u8],
) -> Option<Result<(globals::Scope, usize)>> {
let config = languages::get_tag_configuration(parser)?;
let mut parser = Parser::new();
parser.set_language(config.language).unwrap();
let tree = parser.parse(source_bytes, None).unwrap();
Some(globals::parse_tree(config, &tree, source_bytes))
}
pub fn get_locals(parser: BundledParser, source_bytes: &[u8]) -> Option<Result<Vec<Occurrence>>> {
let mut config = languages::get_local_configuration(parser)?;
let tree = config.parser.parse(source_bytes, None).unwrap();
Some(locals::parse_tree(&mut config, &tree, source_bytes))
}
#[macro_export]
macro_rules! generate_tags_and_snapshot {
($a:literal) => {{
let mut buffer = vec![0u8; 1024];
let mut buf_writer = BufWriter::new(&mut buffer);
generate_tags(&mut buf_writer, $a.to_string(), include_bytes!($a));
insta::assert_snapshot!(String::from_utf8_lossy(buf_writer.buffer()));
}};
}
#[cfg(test)]
mod test {
use std::io::BufWriter;
use crate::ctags::generate_tags;
#[test]
fn test_generate_ctags_go_globals() {
generate_tags_and_snapshot!("../testdata/go-globals.go");
}
#[test]
fn test_generate_ctags_empty_scope() {
generate_tags_and_snapshot!("../testdata/ctags-empty-scope.rs");
}
#[test]
fn test_generate_ctags_zig_globals() {
generate_tags_and_snapshot!("../testdata/globals.zig");
}
#[test]
fn test_generate_ctags_python_globals() {
generate_tags_and_snapshot!("../testdata/globals.py");
}
#[test]
fn test_generate_ctags_java_globals() {
generate_tags_and_snapshot!("../testdata/globals.java");
}
#[test]
fn test_generate_ctags_typescript_globals() {
generate_tags_and_snapshot!("../testdata/globals.ts");
}
}

53
docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs
View File

@ -5,27 +5,15 @@ use scip::{
symbol::format_symbol,
types::{Occurrence, Symbol},
};
use scip_treesitter::prelude::*;
use scip_treesitter::{prelude::*, types::PackedRange};
use tree_sitter::Node;
use crate::languages::LocalConfiguration;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct ByteRange {
start: usize,
end: usize,
}
impl ByteRange {
pub fn contains(&self, other: &Self) -> bool {
self.start <= other.start && self.end >= other.end
}
}
#[derive(Debug)]
pub struct Scope<'a> {
pub scope: Node<'a>,
pub range: ByteRange,
pub range: PackedRange,
pub definitions: HashMap<&'a str, Definition<'a>>,
pub references: HashMap<&'a str, Vec<Reference<'a>>>,
pub children: Vec<Scope<'a>>,
@ -55,10 +43,7 @@ impl<'a> Scope<'a> {
pub fn new(scope: Node<'a>) -> Self {
Self {
scope,
range: ByteRange {
start: scope.start_byte(),
end: scope.end_byte(),
},
range: scope.into(),
definitions: HashMap::default(),
references: HashMap::default(),
children: vec![],
@ -102,10 +87,10 @@ impl<'a> Scope<'a> {
}
}
match self
.children
.binary_search_by_key(&reference.range.start, |r| r.range.start)
{
match self.children.binary_search_by_key(
&(reference.range.start_line, reference.range.start_col),
|r| (r.range.start_line, r.range.start_col),
) {
Ok(_) => {
// self.children[idx].insert_reference(reference);
todo!("I'm not sure what to do yet, think more now");
@ -155,7 +140,8 @@ impl<'a> Scope<'a> {
self.children.extend(child.children);
}
self.children.sort_by_key(|s| s.range.start);
self.children
.sort_by_key(|s| (s.range.start_line, s.range.end_line, s.range.start_col));
}
pub fn into_occurrences(&mut self, hint: usize) -> Vec<Occurrence> {
@ -169,7 +155,7 @@ impl<'a> Scope<'a> {
// We could probably make this a runtime option, where `self` has a `sorted` value
// that decides whether we need to or not. But on a huge file, this made no difference.
let mut values = self.definitions.values().collect::<Vec<_>>();
values.sort_by_key(|d| d.range.start);
values.sort_by_key(|d| &d.range);
for definition in values {
*id += 1;
@ -270,7 +256,7 @@ pub struct Definition<'a> {
pub group: &'a str,
pub identifier: &'a str,
pub node: Node<'a>,
pub range: ByteRange,
pub range: PackedRange,
pub scope_modifier: ScopeModifier,
}
@ -279,7 +265,7 @@ pub struct Reference<'a> {
pub group: &'a str,
pub identifier: &'a str,
pub node: Node<'a>,
pub range: ByteRange,
pub range: PackedRange,
}
pub fn parse_tree<'a>(
@ -356,10 +342,7 @@ pub fn parse_tree<'a>(
let scope_modifier = scope_modifier.unwrap_or_default();
definitions.push(Definition {
range: ByteRange {
start: node.start_byte(),
end: node.end_byte(),
},
range: node.into(),
group,
identifier,
node,
@ -372,10 +355,7 @@ pub fn parse_tree<'a>(
};
references.push(Reference {
range: ByteRange {
start: node.start_byte(),
end: node.end_byte(),
},
range: node.into(),
group,
identifier,
node,
@ -395,8 +375,9 @@ pub fn parse_tree<'a>(
// Sort smallest to largest, so we can pop off the end of the list for the largest, first scope
scopes.sort_by_key(|m| {
(
std::cmp::Reverse(m.range.start),
m.range.end - m.range.start,
std::cmp::Reverse(m.range.start_line),
m.range.end_line - m.range.start_line,
m.range.end_col - m.range.start_col,
)
});

345
docker-images/syntax-highlighter/crates/scip-syntax/src/matches.rs
View File

@ -1,345 +0,0 @@
use anyhow::Result;
use protobuf::Enum;
use scip::types::Descriptor;
use scip_treesitter::prelude::*;
use tree_sitter::Node;
use crate::languages::TagConfiguration;
#[derive(Debug)]
pub struct Root<'a> {
pub root: Node<'a>,
pub children: Vec<Matched<'a>>,
}
// #[derive(Debug)]
// pub struct Namespace {}
pub struct Scope<'a> {
pub definer: Node<'a>,
pub scope: Node<'a>,
pub descriptors: Vec<Descriptor>,
pub children: Vec<Matched<'a>>,
}
impl<'a> std::fmt::Debug for Scope<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let descriptors = dbg_format_descriptors(&self.descriptors);
write!(
f,
"({}, {}, {:?}) -> {:?}",
self.scope.kind(),
self.scope.start_position(),
descriptors,
self.children
)
}
}
pub struct Global<'a> {
pub node: Node<'a>,
pub descriptors: Vec<Descriptor>,
}
impl<'a> std::fmt::Debug for Global<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let descriptors = dbg_format_descriptors(&self.descriptors);
write!(
f,
"({}, {}, {:?})",
self.node.kind(),
self.node.start_position(),
descriptors
)
}
}
#[derive(Debug)]
// TODO: Root as it's own type?
pub enum Matched<'a> {
/// The root node of a file
Root(Root<'a>),
/// Does not generate a definition, simply a place ot add new descriptors
/// TODO: Haven't done this one for real yet
// Namespace(Namespace),
/// Generates a new definition, and is itself a place to add additional descriptors
Scope(Scope<'a>),
/// Generates a new definition, but does not generate a new scope
Global(Global<'a>),
}
impl<'a> ContainsNode for Matched<'a> {
fn contains_node(&self, node: &Node) -> bool {
self.node().contains_node(node)
}
}
impl<'a> Matched<'a> {
pub fn node(&self) -> &Node<'a> {
match self {
Matched::Root(m) => &m.root,
Matched::Scope(m) => &m.scope,
Matched::Global(m) => &m.node,
}
}
// pub fn children(&self) -> &Vec<Matched<'a>> {
// match self {
// Matched::Root(m) => &m.children,
// Matched::Scope(s) => &s.children,
// Matched::Global(_) => todo!(),
// }
// }
pub fn insert(&mut self, m: Matched<'a>) {
match self {
Matched::Root(root) => {
if let Some(child) = root
.children
.iter_mut()
.find(|child| child.contains_node(m.node()))
{
child.insert(m);
} else {
root.children.push(m);
}
}
Matched::Scope(scope) => {
if let Some(child) = scope
.children
.iter_mut()
.find(|child| child.contains_node(m.node()))
{
child.insert(m);
} else {
scope.children.push(m);
}
}
Matched::Global(_) => unreachable!(),
}
}
pub fn into_occurences(&self) -> Vec<scip::types::Occurrence> {
self.rec_into_occurrences(&[])
}
// TODO: Could we use a dequeue for this to pop on and off quickly?
// TODO: Could we use a way to format the symbol w/out all the preamble?
// Perhaps just a "format_descriptors" function in the lib, that I didn't expose beforehand
fn rec_into_occurrences(&self, descriptors: &[Descriptor]) -> Vec<scip::types::Occurrence> {
match self {
Matched::Root(root) => {
assert!(descriptors.is_empty(), "root should not have descriptors");
root.children
.iter()
.flat_map(|c| c.rec_into_occurrences(descriptors))
.collect()
}
Matched::Scope(scope) => {
let mut these_descriptors = descriptors.to_vec();
these_descriptors.extend(scope.descriptors.iter().cloned());
let symbol = scip::symbol::format_symbol(scip::types::Symbol {
scheme: "scip-ctags".into(),
// TODO: Package?
package: None.into(),
descriptors: these_descriptors,
..Default::default()
});
let symbol_roles = scip::types::SymbolRole::Definition.value();
let mut children = vec![scip::types::Occurrence {
range: vec![
scope.definer.start_position().row as i32,
scope.definer.start_position().column as i32,
scope.definer.end_position().column as i32,
],
symbol,
symbol_roles,
// TODO:
// syntax_kind: todo!(),
..Default::default()
}];
children.extend(scope.children.iter().flat_map(|c| {
let mut descriptors = descriptors.to_vec();
descriptors.extend(scope.descriptors.iter().cloned());
c.rec_into_occurrences(&descriptors)
}));
children
}
Matched::Global(global) => {
let mut these_descriptors = descriptors.to_vec();
these_descriptors.extend(global.descriptors.iter().cloned());
let symbol = scip::symbol::format_symbol(scip::types::Symbol {
scheme: "scip-ctags".into(),
// TODO: Package?
package: None.into(),
descriptors: these_descriptors,
..Default::default()
});
let symbol_roles = scip::types::SymbolRole::Definition.value();
vec![scip::types::Occurrence {
range: vec![
global.node.start_position().row as i32,
global.node.start_position().column as i32,
global.node.end_position().column as i32,
],
symbol,
symbol_roles,
// TODO:
// syntax_kind: todo!(),
..Default::default()
}]
}
}
}
}
pub fn parse_tree<'a>(
config: &mut TagConfiguration,
tree: &'a tree_sitter::Tree,
source_bytes: &'a [u8],
) -> Result<Vec<scip::types::Occurrence>> {
let mut cursor = tree_sitter::QueryCursor::new();
let root_node = tree.root_node();
let capture_names = config.query.capture_names();
let mut matched = vec![];
for m in cursor.matches(&config.query, root_node, source_bytes) {
println!("\n==== NEW MATCH ====");
let mut node = None;
let mut scope = None;
let mut descriptors = vec![];
for capture in m.captures {
let capture_name = capture_names
.get(capture.index as usize)
.expect("capture indexes should always work");
if capture_name.starts_with("descriptor") {
descriptors.push((capture_name, capture.node.utf8_text(source_bytes)?));
node = Some(capture.node);
}
if capture_name.starts_with("scope") {
assert!(scope.is_none(), "declare only one scope per match");
scope = Some(capture);
}
println!(
"{}: {}",
capture_name,
capture.node.utf8_text(source_bytes).unwrap()
);
}
let descriptors = descriptors
.into_iter()
.map(|(capture, name)| {
crate::ts_scip::capture_name_to_descriptor(capture, name.to_string())
})
.collect::<Vec<_>>();
let node = node.expect("there must always be at least one descriptor");
dbg!(node);
matched.push(match scope {
Some(scope) => Matched::Scope(Scope {
definer: node,
scope: scope.node,
descriptors,
children: vec![],
}),
None => Matched::Global(Global { node, descriptors }),
})
}
dbg!(&matched);
let mut root = Matched::Root(Root {
root: root_node,
children: vec![],
});
matched.sort_by_key(|m| {
let node = m.node();
node.end_byte() - node.start_byte()
});
while let Some(m) = matched.pop() {
root.insert(m);
}
dbg!(&root);
let tags = root.into_occurences();
Ok(dbg!(tags))
}
fn dbg_format_descriptors(descriptors: &[Descriptor]) -> Vec<String> {
descriptors
.iter()
.map(|d| format!("{} ({:?})", d.name, d.suffix))
.collect::<Vec<_>>()
}
#[cfg(test)]
mod test {
use scip::types::Document;
use scip_treesitter::snapshot::dump_document;
use super::*;
fn parse_file_for_lang(config: &mut TagConfiguration, source_code: &str) -> Result<Document> {
let source_bytes = source_code.as_bytes();
let tree = config.parser.parse(source_bytes, None).unwrap();
let occ = parse_tree(config, &tree, source_bytes)?;
let mut doc = Document::new();
doc.occurrences = occ;
doc.symbols = doc
.occurrences
.iter()
.map(|o| scip::types::SymbolInformation {
symbol: o.symbol.clone(),
..Default::default()
})
.collect();
Ok(doc)
}
#[test]
fn test_can_parse_rust_tree() -> Result<()> {
let mut config = crate::languages::rust();
let source_code = include_str!("../testdata/scopes.rs");
let doc = parse_file_for_lang(&mut config, source_code)?;
let dumped = dump_document(&doc, source_code)?;
insta::assert_snapshot!(dumped);
Ok(())
}
#[test]
fn test_can_parse_go_tree() -> Result<()> {
let mut config = crate::languages::go();
let source_code = include_str!("../testdata/example.go");
let doc = dbg!(parse_file_for_lang(&mut config, source_code)?);
let dumped = dump_document(&doc, source_code)?;
insta::assert_snapshot!(dumped);
Ok(())
}
}

45
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_functions.snap
View File

@ -1,45 +0,0 @@
---
source: src/locals.rs
assertion_line: 469
expression: dumped
---
package example
// ^^^^^^^ definition local 1
import (
f "fmt"
// ^ definition local 2
"github.com/sourcegraph/"
)
func Something() {
// ^^^^^^^^^ definition local 3
y := ", world"
// ^ definition local 5
f.Println("hello", y)
// ^ reference local 2
// ^ reference local 5
}
func Another() {
// ^^^^^^^ definition local 4
Something()
// ^^^^^^^^^ reference local 3
if true {
x := true
// ^ definition local 6
}
if true {
x := true
// ^ definition local 7
if true {
x := true
// ^ definition local 8
}
}
if true {
x := true
// ^ definition local 9
}
}

31
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_go.snap
View File

@ -1,31 +0,0 @@
---
source: src/locals.rs
assertion_line: 463
expression: dumped
---
package main
// ^^^^ definition local 1
func main() {
// ^^^^ reference local 1
local := true
// ^^^^^ definition local 3
something := func(local int) int {
// ^^^^^^^^^ definition local 4
// ^^^^^ definition local 5
return local
// ^^^^^ reference local 5
}
println(local, something)
// ^^^^^ reference local 3
// ^^^^^^^^^ reference local 4
}
func Another(local int) int {
// ^^^^^^^ definition local 2
// ^^^^^ definition local 6
return local
// ^^^^^ reference local 6
}

42
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_nested_locals.snap
View File

@ -1,42 +0,0 @@
---
source: src/locals.rs
assertion_line: 502
expression: dumped
---
package main
// ^^^^ definition local 1
func main() {
// ^^^^ reference local 1
local := 5
// ^^^^^ definition local 2
something := func(unrelated int) int {
// ^^^^^^^^^ definition local 3
// ^^^^^^^^^ definition local 4
superNested := func(deeplyNested int) int {
// ^^^^^^^^^^^ definition local 5
// ^^^^^^^^^^^^ definition local 7
return local + unrelated + deeplyNested
// ^^^^^ reference local 2
// ^^^^^^^^^ reference local 4
// ^^^^^^^^^^^^ reference local 7
}
overwriteName := func(local int) int {
// ^^^^^^^^^^^^^ definition local 6
// ^^^^^ definition local 8
return local + unrelated
// ^^^^^ reference local 8
// ^^^^^^^^^ reference local 4
}
return superNested(1) + overwriteName(1)
// ^^^^^^^^^^^ reference local 5
// ^^^^^^^^^^^^^ reference local 6
}
println(local, something)
// ^^^^^ reference local 2
// ^^^^^^^^^ reference local 3
}

29
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_go_tree.snap
View File

@ -1,29 +0,0 @@
---
source: src/matches.rs
assertion_line: 341
expression: dumped
---
package example
// ^^^^^^^ definition scip-ctags example/
import (
f "fmt"
)
func Something() {
// ^^^^^^^^^ definition scip-ctags Something().
x := true
f.Println(x)
}
func Another() float64 { return 5 / 3 }
// ^^^^^^^ definition scip-ctags Another().
type MyThing struct{}
// ^^^^^^^ definition scip-ctags MyThing#
func (m *MyThing) DoSomething() {}
// ^^^^^^^^^^^ definition scip-ctags MyThing#DoSomething().
func (m MyThing) DoSomethingElse() {}
// ^^^^^^^^^^^^^^^ definition scip-ctags MyThing#DoSomethingElse().

34
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_rust_tree.snap
View File

@ -1,34 +0,0 @@
---
source: src/matches.rs
assertion_line: 334
expression: dumped
---
pub trait Tag {
// ^^^ definition scip-ctags Tag#
// This is a pretty big thing
// And some more things here
fn name(&self) -> &str;
// ^^^^ definition scip-ctags Tag#name().
}
mod namespace {
// ^^^^^^^^^ definition scip-ctags namespace/
mod nested {
// ^^^^^^ definition scip-ctags namespace/nested/
mod even_more_nested {
// ^^^^^^^^^^^^^^^^ definition scip-ctags namespace/nested/even_more_nested/
pub struct CoolStruct {}
// ^^^^^^^^^^ definition scip-ctags namespace/nested/even_more_nested/CoolStruct#
impl Tag for CoolStruct {
// ^^^^^^^^^^ definition scip-ctags namespace/nested/even_more_nested/Tag#CoolStruct#
fn name(&self) -> &str {}
// ^^^^ definition scip-ctags namespace/nested/even_more_nested/Tag#CoolStruct#name().
}
}
}
}
fn something() {}
// ^^^^^^^^^ definition scip-ctags something().

10
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__ctags__test__ctags_runner_basic.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: crates/scip-syntax/src/ctags.rs
expression: output
---
{"_type":"program","name":"SCIP Ctags","version":"5.9.0"}
{"_type":"tag","name":"other","path":"main.rs","language":"rust","line":6,"kind":"method","scope":null}
{"_type":"tag","name":"something","path":"main.rs","language":"rust","line":5,"kind":"method","scope":null}
{"_type":"tag","name":"main","path":"main.rs","language":"rust","line":1,"kind":"method","scope":null}
{"_type":"completed","command":"generate-tags"}

41
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_internal_tree.snap Normal file
View File

@ -0,0 +1,41 @@
---
source: crates/scip-syntax/src/globals.rs
expression: dumped
---
package memo
// ^^^^ definition scip-ctags memo/
import "sync"
// MemoizedConstructorWithArg wraps a function returning taking a
// single argument value and returning a value and an error, memoizing
// its result. Multiple calls to Init will result in the underlying
// constructor being called once. The arguments to the call will be the
// first call to occur. All callers will receive the same return values.
type MemoizedConstructorWithArg[A, T any] struct {
// ^^^^^^^^^^^^^^^^^^^^^^^^^^ definition scip-ctags memo/MemoizedConstructorWithArg#
ctor func(A) (T, error)
// ^^^^ definition scip-ctags memo/MemoizedConstructorWithArg#ctor.
value T
// ^^^^^ definition scip-ctags memo/MemoizedConstructorWithArg#value.
err error
// ^^^ definition scip-ctags memo/MemoizedConstructorWithArg#err.
once sync.Once
// ^^^^ definition scip-ctags memo/MemoizedConstructorWithArg#once.
}
// NewMemoizedConstructor memoizes the given constructor
func NewMemoizedConstructorWithArg[A, T any](ctor func(A) (T, error)) *MemoizedConstructorWithArg[A, T] {
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition scip-ctags memo/NewMemoizedConstructorWithArg().
return &MemoizedConstructorWithArg[A, T]{ctor: ctor}
}
// Init ensures that the given constructor has been called exactly
// once, then returns the constructor's result value and error.
func (m *MemoizedConstructorWithArg[A, T]) Init(arg A) (T, error) {
// ^^^^ definition scip-ctags memo/T#Init().
// ^^^^ definition scip-ctags memo/A#Init().
m.once.Do(func() { m.value, m.err = m.ctor(arg) })
return m.value, m.err
}

29
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_tree.snap Normal file
View File

@ -0,0 +1,29 @@
---
source: crates/scip-syntax/src/globals.rs
expression: dumped
---
package example
// ^^^^^^^ definition scip-ctags example/
import (
f "fmt"
// ^ definition scip-ctags example/f.
)
func Something() {
// ^^^^^^^^^ definition scip-ctags example/Something().
x := true
f.Println(x)
}
func Another() float64 { return 5 / 3 }
// ^^^^^^^ definition scip-ctags example/Another().
type MyThing struct{}
// ^^^^^^^ definition scip-ctags example/MyThing#
func (m *MyThing) DoSomething() {}
// ^^^^^^^^^^^ definition scip-ctags example/MyThing#DoSomething().
func (m MyThing) DoSomethingElse() {}
// ^^^^^^^^^^^^^^^ definition scip-ctags example/MyThing#DoSomethingElse().

4
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_rust_tree.snap → docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_rust_tree.snap
View File

@ -1,6 +1,6 @@
---
source: crates/scip-syntax/src/matches.rs
assertion_line: 329
source: crates/scip-syntax/src/globals.rs
assertion_line: 318
expression: dumped
---
pub trait Tag {

29
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_go_tree.snap
View File

@ -1,29 +0,0 @@
---
source: crates/scip-syntax/src/matches.rs
assertion_line: 341
expression: dumped
---
package example
// ^^^^^^^ definition scip-ctags example/
import (
f "fmt"
)
func Something() {
// ^^^^^^^^^ definition scip-ctags Something().
x := true
f.Println(x)
}
func Another() float64 { return 5 / 3 }
// ^^^^^^^ definition scip-ctags Another().
type MyThing struct{}
// ^^^^^^^ definition scip-ctags MyThing#
func (m *MyThing) DoSomething() {}
// ^^^^^^^^^^^ definition scip-ctags MyThing#DoSomething().
func (m MyThing) DoSomethingElse() {}
// ^^^^^^^^^^^^^^^ definition scip-ctags MyThing#DoSomethingElse().

10
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_empty_scope.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: crates/scip-syntax/src/lib.rs
expression: "String::from_utf8_lossy(buf_writer.buffer())"
---
{"_type":"tag","name":"Arguments","path":"ctags-empty-scope.rs","language":"rust","line":10,"kind":"type","scope":null}
{"_type":"tag","name":"ParseTiming","path":"ctags-empty-scope.rs","language":"rust","line":15,"kind":"type","scope":null}
{"_type":"tag","name":"main","path":"ctags-empty-scope.rs","language":"rust","line":28,"kind":"method","scope":null}
{"_type":"tag","name":"measure_parsing","path":"ctags-empty-scope.rs","language":"rust","line":24,"kind":"method","scope":null}
{"_type":"tag","name":"parse_files","path":"ctags-empty-scope.rs","language":"rust","line":20,"kind":"method","scope":null}

12
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_go_globals.snap Normal file
View File

@ -0,0 +1,12 @@
---
source: crates/scip-syntax/src/lib.rs
expression: "String::from_utf8_lossy(buf_writer.buffer())"
---
{"_type":"tag","name":"multierror","path":"go-globals.go","language":"go","line":1,"kind":"namespace","scope":null}
{"_type":"tag","name":"Group","path":"go-globals.go","language":"go","line":7,"kind":"type","scope":"multierror"}
{"_type":"tag","name":"wg","path":"go-globals.go","language":"go","line":10,"kind":"variable","scope":"multierror.Group"}
{"_type":"tag","name":"err","path":"go-globals.go","language":"go","line":9,"kind":"variable","scope":"multierror.Group"}
{"_type":"tag","name":"mutex","path":"go-globals.go","language":"go","line":8,"kind":"variable","scope":"multierror.Group"}
{"_type":"tag","name":"Wait","path":"go-globals.go","language":"go","line":33,"kind":"method","scope":"multierror.Group"}
{"_type":"tag","name":"Go","path":"go-globals.go","language":"go","line":17,"kind":"method","scope":"multierror.Group"}

31
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_java_globals.snap Normal file
View File

@ -0,0 +1,31 @@
---
source: crates/scip-syntax/src/lib.rs
expression: "String::from_utf8_lossy(buf_writer.buffer())"
---
{"_type":"tag","name":"globals","path":"globals.java","language":"java","line":1,"kind":"type","scope":null}
{"_type":"tag","name":"ClassInAClass","path":"globals.java","language":"java","line":18,"kind":"type","scope":"globals"}
{"_type":"tag","name":"Enum","path":"globals.java","language":"java","line":21,"kind":"type","scope":"globals.ClassInAClass"}
{"_type":"tag","name":"terms","path":"globals.java","language":"java","line":27,"kind":"variable","scope":"globals.ClassInAClass.Enum"}
{"_type":"tag","name":"as","path":"globals.java","language":"java","line":26,"kind":"variable","scope":"globals.ClassInAClass.Enum"}
{"_type":"tag","name":"recognized","path":"globals.java","language":"java","line":25,"kind":"variable","scope":"globals.ClassInAClass.Enum"}
{"_type":"tag","name":"be","path":"globals.java","language":"java","line":24,"kind":"variable","scope":"globals.ClassInAClass.Enum"}
{"_type":"tag","name":"should","path":"globals.java","language":"java","line":23,"kind":"variable","scope":"globals.ClassInAClass.Enum"}
{"_type":"tag","name":"these","path":"globals.java","language":"java","line":22,"kind":"variable","scope":"globals.ClassInAClass.Enum"}
{"_type":"tag","name":"Goated","path":"globals.java","language":"java","line":30,"kind":"type","scope":"globals.ClassInAClass"}
{"_type":"tag","name":"withTheSauce","path":"globals.java","language":"java","line":31,"kind":"method","scope":"globals.ClassInAClass.Goated"}
{"_type":"tag","name":"myCoolMethod","path":"globals.java","language":"java","line":34,"kind":"method","scope":"globals.ClassInAClass"}
{"_type":"tag","name":"classy","path":"globals.java","language":"java","line":19,"kind":"variable","scope":"globals.ClassInAClass"}
{"_type":"tag","name":"COOLEST_STRING","path":"globals.java","language":"java","line":16,"kind":"variable","scope":"globals"}
{"_type":"tag","name":"method6","path":"globals.java","language":"java","line":14,"kind":"method","scope":"globals"}
{"_type":"tag","name":"method5","path":"globals.java","language":"java","line":13,"kind":"method","scope":"globals"}
{"_type":"tag","name":"method4","path":"globals.java","language":"java","line":12,"kind":"method","scope":"globals"}
{"_type":"tag","name":"method3","path":"globals.java","language":"java","line":11,"kind":"method","scope":"globals"}
{"_type":"tag","name":"method2","path":"globals.java","language":"java","line":10,"kind":"method","scope":"globals"}
{"_type":"tag","name":"method1","path":"globals.java","language":"java","line":9,"kind":"method","scope":"globals"}
{"_type":"tag","name":"field6","path":"globals.java","language":"java","line":7,"kind":"variable","scope":"globals"}
{"_type":"tag","name":"field5","path":"globals.java","language":"java","line":6,"kind":"variable","scope":"globals"}
{"_type":"tag","name":"field4","path":"globals.java","language":"java","line":5,"kind":"variable","scope":"globals"}
{"_type":"tag","name":"field3","path":"globals.java","language":"java","line":4,"kind":"variable","scope":"globals"}
{"_type":"tag","name":"field2","path":"globals.java","language":"java","line":3,"kind":"variable","scope":"globals"}
{"_type":"tag","name":"field1","path":"globals.java","language":"java","line":2,"kind":"variable","scope":"globals"}

10
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_python_globals.snap Normal file
View File

@ -0,0 +1,10 @@
---
source: crates/scip-syntax/src/lib.rs
expression: "String::from_utf8_lossy(buf_writer.buffer())"
---
{"_type":"tag","name":"Bruh","path":"globals.py","language":"python","line":6,"kind":"type","scope":null}
{"_type":"tag","name":"dab","path":"globals.py","language":"python","line":11,"kind":"method","scope":"Bruh"}
{"_type":"tag","name":"__init__","path":"globals.py","language":"python","line":8,"kind":"method","scope":"Bruh"}
{"_type":"tag","name":"a","path":"globals.py","language":"python","line":6,"kind":"variable","scope":"Bruh"}
{"_type":"tag","name":"bruh","path":"globals.py","language":"python","line":3,"kind":"variable","scope":null}

22
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_typescript_globals.snap Normal file
View File

@ -0,0 +1,22 @@
---
source: crates/scip-syntax/src/lib.rs
expression: "String::from_utf8_lossy(buf_writer.buffer())"
---
{"_type":"tag","name":"MyClass","path":"globals.ts","language":"typescript","line":1,"kind":"type","scope":null}
{"_type":"tag","name":"also_private_method","path":"globals.ts","language":"typescript","line":8,"kind":"method","scope":"MyClass"}
{"_type":"tag","name":"#private_method","path":"globals.ts","language":"typescript","line":7,"kind":"method","scope":"MyClass"}
{"_type":"tag","name":"public_method","path":"globals.ts","language":"typescript","line":6,"kind":"method","scope":"MyClass"}
{"_type":"tag","name":"also_private_field","path":"globals.ts","language":"typescript","line":4,"kind":"variable","scope":"MyClass"}
{"_type":"tag","name":"#private_field","path":"globals.ts","language":"typescript","line":3,"kind":"variable","scope":"MyClass"}
{"_type":"tag","name":"public_field","path":"globals.ts","language":"typescript","line":2,"kind":"variable","scope":"MyClass"}
{"_type":"tag","name":"MyInterface","path":"globals.ts","language":"typescript","line":11,"kind":"type","scope":null}
{"_type":"tag","name":"sayBruh","path":"globals.ts","language":"typescript","line":13,"kind":"method","scope":"MyInterface"}
{"_type":"tag","name":"bruh","path":"globals.ts","language":"typescript","line":12,"kind":"variable","scope":"MyInterface"}
{"_type":"tag","name":"MyEnum","path":"globals.ts","language":"typescript","line":16,"kind":"type","scope":null}
{"_type":"tag","name":"go","path":"globals.ts","language":"typescript","line":19,"kind":"variable","scope":"MyEnum"}
{"_type":"tag","name":"rust","path":"globals.ts","language":"typescript","line":18,"kind":"variable","scope":"MyEnum"}
{"_type":"tag","name":"zig","path":"globals.ts","language":"typescript","line":17,"kind":"variable","scope":"MyEnum"}
{"_type":"tag","name":"func","path":"globals.ts","language":"typescript","line":25,"kind":"method","scope":null}
{"_type":"tag","name":"global2","path":"globals.ts","language":"typescript","line":23,"kind":"variable","scope":null}
{"_type":"tag","name":"global1","path":"globals.ts","language":"typescript","line":22,"kind":"variable","scope":null}

27
docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_zig_globals.snap Normal file
View File

@ -0,0 +1,27 @@
---
source: crates/scip-syntax/src/lib.rs
expression: "String::from_utf8_lossy(buf_writer.buffer())"
---
{"_type":"tag","name":"Bruh","path":"globals.zig","language":"zig","line":1,"kind":"variable","scope":null}
{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":4,"kind":"method","scope":"Bruh"}
{"_type":"tag","name":"zig_is_cool","path":"globals.zig","language":"zig","line":2,"kind":"variable","scope":"Bruh"}
{"_type":"tag","name":"MyUnion","path":"globals.zig","language":"zig","line":10,"kind":"variable","scope":null}
{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":16,"kind":"method","scope":"MyUnion"}
{"_type":"tag","name":"b","path":"globals.zig","language":"zig","line":14,"kind":"variable","scope":"MyUnion"}
{"_type":"tag","name":"a","path":"globals.zig","language":"zig","line":13,"kind":"variable","scope":"MyUnion"}
{"_type":"tag","name":"decl","path":"globals.zig","language":"zig","line":11,"kind":"variable","scope":"MyUnion"}
{"_type":"tag","name":"MyEnum","path":"globals.zig","language":"zig","line":19,"kind":"variable","scope":null}
{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":25,"kind":"method","scope":"MyEnum"}
{"_type":"tag","name":"b","path":"globals.zig","language":"zig","line":23,"kind":"variable","scope":"MyEnum"}
{"_type":"tag","name":"a","path":"globals.zig","language":"zig","line":22,"kind":"variable","scope":"MyEnum"}
{"_type":"tag","name":"decl","path":"globals.zig","language":"zig","line":20,"kind":"variable","scope":"MyEnum"}
{"_type":"tag","name":"MyUnionEnum","path":"globals.zig","language":"zig","line":28,"kind":"variable","scope":null}
{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":34,"kind":"method","scope":"MyUnionEnum"}
{"_type":"tag","name":"b","path":"globals.zig","language":"zig","line":32,"kind":"variable","scope":"MyUnionEnum"}
{"_type":"tag","name":"a","path":"globals.zig","language":"zig","line":31,"kind":"variable","scope":"MyUnionEnum"}
{"_type":"tag","name":"decl","path":"globals.zig","language":"zig","line":29,"kind":"variable","scope":"MyUnionEnum"}
{"_type":"tag","name":"Ahh","path":"globals.zig","language":"zig","line":37,"kind":"variable","scope":null}
{"_type":"tag","name":"opaqueFn","path":"globals.zig","language":"zig","line":38,"kind":"method","scope":"Ahh"}
{"_type":"tag","name":"complex","path":"globals.zig","language":"zig","line":47,"kind":"method","scope":null}
{"_type":"tag","name":"bruh","path":"globals.zig","language":"zig","line":41,"kind":"method","scope":null}

1
docker-images/syntax-highlighter/crates/scip-syntax/src/ts_scip.rs
View File

@ -6,6 +6,7 @@ pub fn capture_name_to_descriptor(capture: &str, name: String) -> Descriptor {
"descriptor.method" => Suffix::Method,
"descriptor.namespace" => Suffix::Namespace,
"descriptor.type" => Suffix::Type,
"descriptor.term" => Suffix::Term,
// TODO: Should consider moving to result here.
_ => Suffix::UnspecifiedSuffix,

30
docker-images/syntax-highlighter/crates/scip-syntax/testdata/ctags-empty-scope.rs vendored Normal file
View File

@ -0,0 +1,30 @@
use std::{path::Path, time::Instant};
use clap::Parser;
use scip_syntax::locals::parse_tree;
use scip_treesitter_languages::parsers::BundledParser;
use walkdir::WalkDir;
#[derive(Parser)]
#[command(author, version, about, long_about = None)]
struct Arguments {
/// Root directory to run local navigation over
root_dir: String,
}
struct ParseTiming {
pub filepath: String,
pub duration: std::time::Duration,
}
fn parse_files(dir: &Path) -> Vec<ParseTiming> {
// TODO
}
fn measure_parsing() {
// TODO
}
fn main() {
// TODO
}

39
docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.java vendored Normal file
View File

@ -0,0 +1,39 @@
public class globals {
private static int field1;
protected static int field2;
public static int field3;
private int field4;
protected int field5;
public int field6;
private static void method1() {}
protected static void method2() {}
public static void method3() {}
private void method4() {}
protected void method5() {}
public void method6() {}
public static final String COOLEST_STRING = "probably this one";
public class ClassInAClass {
boolean classy = true;
public static enum Enum {
these,
should,
be,
recognized,
as,
terms
}
public interface Goated {
boolean withTheSauce();
}
public void myCoolMethod() {
class WhatIsGoingOn {}
boolean iThinkThisIsAllowedButWeDontReallyCare = true;
}
}
}

28
docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.py vendored Normal file
View File

@ -0,0 +1,28 @@
# TODO: Deal with duplicates (bruh = 10; bruh = 10;) being marked as definitions
bruh = 10
class Bruh(object):
a: int
def __init__(self) -> None:
pass
def dab():
print("yay!")
def more():
print("a function in a function!!")
pass
more()
if 1 == 1:
notHere = False
while False:
notHereEither = False
for i in range(0, 0):
definitelyNotInHere = False
with 1:
what = "is this even allowed in Python anymore?"

33
docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.ts vendored Normal file
View File

@ -0,0 +1,33 @@
class MyClass {
public_field: number
#private_field: number
private also_private_field: number
public_method() {}
#private_method() {}
private also_private_method() {}
}
interface MyInterface {
bruh: number,
sayBruh(): void,
}
enum MyEnum {
zig,
rust,
go,
}
var global1 = 0;
var global2;
function func() {
var c;
function inAnotherFunc() {
var b;
function inAnother() {
var a;
}
}
}

49
docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.zig vendored Normal file
View File

@ -0,0 +1,49 @@
pub const Bruh = struct {
zig_is_cool: bool = true,
pub fn init() Bruh {
var aaa = false;
return .{};
}
};
const MyUnion = union {
const decl = 10;
a: u8,
b: u40,
pub fn init() void {};
};
const MyEnum = enum {
const decl = 10;
a,
b,
pub fn init() void {};
};
const MyUnionEnum = union(enum) {
const decl = 10;
a: u8,
b: u40,
pub fn init() void {};
};
const Ahh = opaque {
pub fn opaqueFn() void {}
}
fn bruh() void {
const ThisShouldntBeRegistered = struct {
fn bruh2() void {}
}
}
fn complex(a: struct {bruh: bool}) struct {dab: u8} {
return .{.dab = if (a.bruh) 10 else 20};
}

38
docker-images/syntax-highlighter/crates/scip-syntax/testdata/go-globals.go vendored Normal file
View File

@ -0,0 +1,38 @@
package multierror
import "sync"
// Group is a collection of goroutines which return errors that need to be
// coalesced.
type Group struct {
mutex sync.Mutex
err *Error
wg sync.WaitGroup
}
// Go calls the given function in a new goroutine.
//
// If the function returns an error it is added to the group multierror which
// is returned by Wait.
func (g *Group) Go(f func() error) {
g.wg.Add(1)
go func() {
defer g.wg.Done()
if err := f(); err != nil {
g.mutex.Lock()
g.err = Append(g.err, err)
g.mutex.Unlock()
}
}()
}
// Wait blocks until all function calls from the Go method have returned, then
// returns the multierror.
func (g *Group) Wait() *Error {
g.wg.Wait()
g.mutex.Lock()
defer g.mutex.Unlock()
return g.err
}

27
docker-images/syntax-highlighter/crates/scip-syntax/testdata/internal_go.go vendored Normal file
View File

@ -0,0 +1,27 @@
package memo
import "sync"
// MemoizedConstructorWithArg wraps a function returning taking a
// single argument value and returning a value and an error, memoizing
// its result. Multiple calls to Init will result in the underlying
// constructor being called once. The arguments to the call will be the
// first call to occur. All callers will receive the same return values.
type MemoizedConstructorWithArg[A, T any] struct {
ctor func(A) (T, error)
value T
err error
once sync.Once
}
// NewMemoizedConstructor memoizes the given constructor
func NewMemoizedConstructorWithArg[A, T any](ctor func(A) (T, error)) *MemoizedConstructorWithArg[A, T] {
return &MemoizedConstructorWithArg[A, T]{ctor: ctor}
}
// Init ensures that the given constructor has been called exactly
// once, then returns the constructor's result value and error.
func (m *MemoizedConstructorWithArg[A, T]) Init(arg A) (T, error) {
m.once.Do(func() { m.value, m.err = m.ctor(arg) })
return m.value, m.err
}

52
docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs
View File

@ -78,4 +78,56 @@ impl BundledParser {
_ => None,
}
}
pub fn get_language_name(&self) -> &str {
match self {
BundledParser::C => "c",
BundledParser::Cpp => "cpp",
BundledParser::C_Sharp => "c_sharp",
BundledParser::Go => "go",
BundledParser::Java => "java",
BundledParser::Javascript => "javascript",
BundledParser::Jsonnet => "jsonnet",
BundledParser::Kotlin => "kotlin",
BundledParser::Nickel => "nickel",
BundledParser::Perl => "perl",
BundledParser::Pod => "pod",
BundledParser::Python => "python",
BundledParser::Ruby => "ruby",
BundledParser::Rust => "rust",
BundledParser::Scala => "scala",
BundledParser::Sql => "sql",
BundledParser::Typescript => "typescript",
BundledParser::Tsx => "tsx",
BundledParser::Xlsg => "xlsg",
BundledParser::Zig => "zig",
}
}
// TODO(SuperAuguste): language detection library
pub fn get_parser_from_extension(name: &str) -> Option<Self> {
match name {
"c" => Some(BundledParser::C),
"cpp" => Some(BundledParser::Cpp),
"cs" => Some(BundledParser::C_Sharp),
"go" => Some(BundledParser::Go),
"java" => Some(BundledParser::Java),
"js" => Some(BundledParser::Javascript),
"jsonnet" => Some(BundledParser::Jsonnet),
"kt" => Some(BundledParser::Kotlin),
"ncl" => Some(BundledParser::Nickel),
"pl" => Some(BundledParser::Perl),
"pod" => Some(BundledParser::Pod),
"py" => Some(BundledParser::Python),
"rb" => Some(BundledParser::Ruby),
"rs" => Some(BundledParser::Rust),
"scala" => Some(BundledParser::Scala),
"sql" => Some(BundledParser::Sql),
"ts" => Some(BundledParser::Typescript),
"tsx" => Some(BundledParser::Tsx),
"xlsg" => Some(BundledParser::Xlsg),
"zig" => Some(BundledParser::Zig),
_ => None,
}
}
}

78
docker-images/syntax-highlighter/crates/scip-treesitter/src/types.rs
View File

@ -1,3 +1,5 @@
use tree_sitter::Node;
#[derive(Debug, PartialEq, Eq, Default)]
pub struct PackedRange {
pub start_line: i32,
@ -29,6 +31,14 @@ impl PackedRange {
}
}
pub fn to_vec(&self) -> Vec<i32> {
if self.start_line == self.end_line {
vec![self.start_line, self.start_col, self.end_col]
} else {
vec![self.start_line, self.start_col, self.end_line, self.end_col]
}
}
/// Checks if the range is equal to the given vector.
/// If the other vector is not a valid PackedRange then it returns false
pub fn eq_vec(&self, v: &[i32]) -> bool {
@ -48,6 +58,13 @@ impl PackedRange {
_ => false,
}
}
pub fn contains(&self, other: &PackedRange) -> bool {
other.start_line >= self.start_line
&& other.end_line <= self.end_line
&& (other.start_line != self.start_line || other.start_col >= self.start_col)
&& (other.end_line != self.end_line || other.end_col <= self.end_col)
}
}
impl PartialOrd for PackedRange {
@ -69,3 +86,64 @@ impl Ord for PackedRange {
))
}
}
impl<'a> From<Node<'a>> for PackedRange {
fn from(node: Node<'a>) -> Self {
let start = node.start_position();
let end = node.end_position();
Self {
start_line: start.row as i32,
start_col: start.column as i32,
end_line: end.row as i32,
end_col: end.column as i32,
}
}
}
#[cfg(test)]
mod tests {
use super::PackedRange;
#[test]
fn test_packed_range_contains() {
let outer = PackedRange {
start_line: 1,
start_col: 1,
end_line: 4,
end_col: 4,
};
let inner = PackedRange {
start_line: 2,
start_col: 2,
end_line: 3,
end_col: 3,
};
let overlapping = PackedRange {
start_line: 3,
start_col: 3,
end_line: 5,
end_col: 5,
};
let outside = PackedRange {
start_line: 5,
start_col: 5,
end_line: 6,
end_col: 6,
};
let same = PackedRange {
start_line: 1,
start_col: 1,
end_line: 4,
end_col: 4,
};
assert!(outer.contains(&inner));
assert!(!outer.contains(&overlapping));
assert!(!outer.contains(&outside));
assert!(outer.contains(&same));
}
}

2
docker-images/syntax-highlighter/insta.yaml Normal file
View File

@ -0,0 +1,2 @@
review:
warn_undiscovered: false

7
docker-images/syntax-highlighter/justfile Normal file
View File

@ -0,0 +1,7 @@
download-bench:
mkdir -p bench_data/
test -f bench_data/event.rs || curl -o bench_data/event.rs https://raw.githubusercontent.com/alacritty/alacritty/ead65221ebe06ff5689e65b866d735d4365d0e9e/alacritty/src/event.rs
test -f bench_data/big.cpp || curl -o bench_data/big.cpp https://raw.githubusercontent.com/llvm/llvm-project/ff2e6199b23525b06947785368cc3e2e93eab381/llvm/lib/Target/X86/X86ISelLowering.cpp
bench: download-bench
cargo bench

12
docker-images/syntax-highlighter/src/bin/scip-ctags.rs Normal file
View File

@ -0,0 +1,12 @@
use std::io::{BufReader, BufWriter};
use scip_syntax::ctags::ctags_runner;
fn main() {
let mut stdin = BufReader::new(std::io::stdin());
let mut stdout = BufWriter::new(std::io::stdout());
if let Err(err) = ctags_runner(&mut stdin, &mut stdout) {
eprintln!("Error while executing: {}", err);
}
}

67
docker-images/syntax-highlighter/src/main.rs
View File

@ -3,7 +3,14 @@
#[macro_use]
extern crate rocket;
use std::path;
use ::scip::types::Document;
use protobuf::Message;
use rocket::serde::json::{json, Json, Value as JsonValue};
use scip_syntax::get_globals;
use scip_treesitter_languages::parsers::BundledParser;
use serde::Deserialize;
use sg_syntax::{ScipHighlightQuery, SourcegraphQuery};
#[post("/", format = "application/json", data = "<q>")]
@ -38,6 +45,60 @@ fn scip(q: Json<ScipHighlightQuery>) -> JsonValue {
}
}
#[derive(Deserialize, Default, Debug)]
pub struct SymbolQuery {
filename: String,
content: String,
}
pub fn jsonify_err(e: impl ToString) -> JsonValue {
json!({"error": e.to_string()})
}
#[post("/symbols", format = "application/json", data = "<q>")]
fn symbols(q: Json<SymbolQuery>) -> JsonValue {
let path = path::Path::new(&q.filename);
let extension = match match path.extension() {
Some(vals) => vals,
None => {
return json!({"error": "Extensionless file"});
}
}
.to_str()
{
Some(vals) => vals,
None => {
return json!({"error": "Invalid codepoint"});
}
};
let parser = match BundledParser::get_parser_from_extension(extension) {
Some(parser) => parser,
None => return json!({"error": "Could not infer parser from extension"}),
};
let (mut scope, hint) = match match get_globals(&parser, q.content.as_bytes()) {
Some(globals) => globals,
None => return json!({"error": "Failed to get globals"}),
} {
Ok(vals) => vals,
Err(err) => {
return jsonify_err(err);
}
};
let mut document = Document::default();
document.occurrences = scope.into_occurrences(hint, vec![]);
let encoded = match document.write_to_bytes() {
Ok(vals) => vals,
Err(err) => {
return jsonify_err(err);
}
};
json!({"scip": base64::encode(encoded), "plaintext": false})
}
#[get("/health")]
fn health() -> &'static str {
"OK"
@ -59,8 +120,8 @@ fn rocket() -> _ {
Ok(v) if v == "true" => {
println!("Sanity check passed, exiting without error");
std::process::exit(0)
},
_ => {},
}
_ => {}
};
// load configurations on-startup instead of on-first-request.
@ -76,6 +137,6 @@ fn rocket() -> _ {
};
rocket::build()
.mount("/", routes![syntect, lsif, scip, health])
.mount("/", routes![syntect, lsif, scip, symbols, health])
.register("/", catchers![not_found])
}

33
enterprise/cmd/symbols/build-bazel.sh
View File

@ -11,6 +11,8 @@ cleanup() {
}
trap cleanup EXIT
echo "--- :bazel: bazel build for targets //enterprise/cmd/symbols"
bazelrc=(
--bazelrc=.bazelrc
)
@ -21,7 +23,6 @@ if [[ ${CI:-""} == "true" ]]; then
)
fi
echo "--- bazel build"
bazel "${bazelrc[@]}" \
build \
//enterprise/cmd/symbols \
@ -30,18 +31,38 @@ bazel "${bazelrc[@]}" \
--config incompat-zig-linux-amd64
out=$(
bazel \
"${bazelrc[@]}" \
cquery \
//enterprise/cmd/symbols \
bazel "${bazelrc[@]}" \
cquery //enterprise/cmd/symbols \
--stamp \
--workspace_status_command=./dev/bazel_stamp_vars.sh \
--config incompat-zig-linux-amd64 \
--output=files
)
cp "$out" "$OUTPUT"
cp -v "$out" "$OUTPUT"
# we can't build scip-ctags with symbols since the platform args conflict
# NOTE: cmd/symbols/cargo-config.sh sets some specific config when running on arm64
# since this bazel run typically runs on CI that config change isn't made
echo "--- :bazel: bazel build for target //docker-images/syntax-highlighter:scip-ctags"
bazel "${bazelrc[@]}" \
build //docker-images/syntax-highlighter:scip-ctags \
--stamp \
--workspace_status_command=./dev/bazel_stamp_vars.sh
out=$(
bazel "${bazelrc[@]}" \
cquery //docker-images/syntax-highlighter:scip-ctags \
--stamp \
--workspace_status_command=./dev/bazel_stamp_vars.sh \
--output=files
)
cp -v "$out" "$OUTPUT"
cp cmd/symbols/ctags-install-alpine.sh "$OUTPUT"
echo ":docker: context directory contains the following:"
ls -lah "$OUTPUT"
echo "--- :docker: docker build for symbols"
docker build -f cmd/symbols/Dockerfile.bazel -t "$IMAGE" "$OUTPUT" \
--progress=plain \
--build-arg COMMIT_SHA \

1
enterprise/cmd/symbols/shared/BUILD.bazel generated
View File

@ -17,6 +17,7 @@ go_library(
"//enterprise/internal/rockskip",
"//internal/conf",
"//internal/conf/conftypes",
"//internal/ctags_config",
"//internal/database",
"//internal/database/connections/live",
"//internal/debugserver",

7
enterprise/cmd/symbols/shared/setup.go
View File

@ -17,6 +17,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/internal/rockskip"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/ctags_config"
"github.com/sourcegraph/sourcegraph/internal/database"
connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live"
"github.com/sourcegraph/sourcegraph/internal/env"
@ -127,14 +128,14 @@ func setupRockskip(observationCtx *observation.Context, config rockskipConfig, g
codeintelDB := mustInitializeCodeIntelDB(observationCtx)
createParser := func() (ctags.Parser, error) {
return symbolsParser.SpawnCtags(log.Scoped("parser", "ctags parser"), config.Ctags)
return symbolsParser.SpawnCtags(log.Scoped("parser", "ctags parser"), config.Ctags, ctags_config.UniversalCtags)
}
server, err := rockskip.NewService(codeintelDB, gitserverClient, repositoryFetcher, createParser, config.MaxConcurrentlyIndexing, config.MaxRepos, config.LogQueries, config.IndexRequestsQueueSize, config.SymbolsCacheSize, config.PathSymbolsCacheSize, config.SearchLastIndexedCommit)
if err != nil {
return nil, nil, config.Ctags.Command, err
return nil, nil, config.Ctags.UniversalCommand, err
}
return server.Search, server.HandleStatus, config.Ctags.Command, nil
return server.Search, server.HandleStatus, config.Ctags.UniversalCommand, nil
}
func mustInitializeCodeIntelDB(observationCtx *observation.Context) *sql.DB {

4
go.mod
View File

@ -113,7 +113,7 @@ require (
github.com/getsentry/sentry-go v0.21.0
github.com/ghodss/yaml v1.0.0
github.com/gitchander/permutation v0.0.0-20210517125447-a5d73722e1b1
github.com/go-enry/go-enry/v2 v2.8.3
github.com/go-enry/go-enry/v2 v2.8.4
github.com/go-git/go-git/v5 v5.5.2
github.com/go-openapi/strfmt v0.21.3
github.com/gobwas/glob v0.2.3
@ -533,7 +533,7 @@ require (
go.opentelemetry.io/collector/pdata v1.0.0-rc5 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
golang.org/x/mod v0.8.0
golang.org/x/term v0.8.0 // indirect
golang.org/x/text v0.9.0

8
go.sum
View File

@ -793,8 +793,8 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
github.com/go-enry/go-enry/v2 v2.8.3 h1:BwvNrN58JqBJhyyVdZSl5QD3xoxEEGYUrRyPh31FGhw=
github.com/go-enry/go-enry/v2 v2.8.3/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ=
github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs=
github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8=
github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo=
github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
@ -2476,8 +2476,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

9
internal/ctags_config/BUILD.bazel generated Normal file
View File

@ -0,0 +1,9 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "ctags_config",
srcs = ["ctags_config.go"],
importpath = "github.com/sourcegraph/sourcegraph/internal/ctags_config",
visibility = ["//:__subpackages__"],
deps = ["//lib/errors"],
)

42
internal/ctags_config/ctags_config.go Normal file
View File

@ -0,0 +1,42 @@
package ctags_config
import "github.com/sourcegraph/sourcegraph/lib/errors"
type ParserType = uint8
const (
UnknownCtags ParserType = iota
NoCtags
UniversalCtags
ScipCtags
)
func ParserNameToParserType(name string) (ParserType, error) {
switch name {
case "off":
return NoCtags, nil
case "universal-ctags":
return UniversalCtags, nil
case "scip-ctags":
return ScipCtags, nil
default:
return UnknownCtags, errors.Errorf("unknown parser type: %s", name)
}
}
type ParserConfiguration struct {
Default ParserType
Engine map[string]ParserType
}
var SupportLanguages = map[string]struct{}{
"Zig": {},
}
var BaseParserConfig = ParserConfiguration{
Engine: map[string]ParserType{
// TODO: put our other languages here
// TODO: also list the languages we support
"Zig": ScipCtags,
},
}

12
lib/codeintel/languages/BUILD.bazel generated Normal file
View File

@ -0,0 +1,12 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "languages",
srcs = ["languages.go"],
importpath = "github.com/sourcegraph/sourcegraph/lib/codeintel/languages",
visibility = ["//visibility:public"],
deps = [
"@com_github_go_enry_go_enry_v2//:go-enry",
"@org_golang_x_exp//slices",
],
)

53
lib/codeintel/languages/languages.go Normal file
View File

@ -0,0 +1,53 @@
package languages
import (
"github.com/go-enry/go-enry/v2"
"golang.org/x/exp/slices"
)
// TODO: We probably want to move the config for language detection into here from the syntax highlighting part
// I didn't add that yet.
// GetLanguage returns the language for the given path and contents.
func GetLanguage(path, contents string) (lang string, found bool) {
// Force the use of the shebang.
if shebangLang, ok := overrideViaShebang(path, contents); ok {
return shebangLang, true
}
// Lastly, fall back to whatever enry decides is a useful algorithm for calculating.
lang = enry.GetLanguage(path, []byte(contents))
if lang != "" {
return lang, true
}
return lang, false
}
// overrideViaShebang handles explicitly using the shebang whenever possible.
//
// It also covers some edge cases when enry eagerly returns more languages
// than necessary, which ends up overriding the shebang completely (which,
// IMO is the highest priority match we can have).
//
// For example, enry will return "Perl" and "Pod" for a shebang of `#!/usr/bin/env perl`.
// This is actually unhelpful, because then enry will *not* select "Perl" as the
// language (which is our desired behavior).
func overrideViaShebang(path, content string) (lang string, ok bool) {
shebangs := enry.GetLanguagesByShebang(path, []byte(content), []string{})
if len(shebangs) == 0 {
return "", false
}
if len(shebangs) == 1 {
return shebangs[0], true
}
// There are some shebangs that enry returns that are not really
// useful for our syntax highlighters to distinguish between.
if slices.Equal(shebangs, []string{"Perl", "Pod"}) {
return "Perl", true
}
return "", false
}

5
lib/go.mod
View File

@ -9,6 +9,7 @@ require (
github.com/derision-test/go-mockgen v1.3.7
github.com/fatih/color v1.13.0
github.com/ghodss/yaml v1.0.0
github.com/go-enry/go-enry/v2 v2.8.4
github.com/gobwas/glob v0.2.3
github.com/google/go-cmp v0.5.9
github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db
@ -27,7 +28,7 @@ require (
github.com/stretchr/testify v1.8.1
github.com/urfave/cli/v2 v2.23.7
github.com/xeipuuv/gojsonschema v1.2.0
go.uber.org/atomic v1.10.0
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
golang.org/x/sync v0.1.0
golang.org/x/sys v0.3.0
golang.org/x/term v0.3.0
@ -55,6 +56,7 @@ require (
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/envoyproxy/protoc-gen-validate v0.6.13 // indirect
github.com/getsentry/sentry-go v0.15.0 // indirect
github.com/go-enry/go-oniguruma v1.2.1 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gofrs/uuid v4.2.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
@ -102,6 +104,7 @@ require (
github.com/yuin/goldmark v1.5.2 // indirect
github.com/yuin/goldmark-emoji v1.0.1 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/goleak v1.2.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.24.0 // indirect

6
lib/go.sum
View File

@ -151,6 +151,10 @@ github.com/getsentry/sentry-go v0.15.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2Ht
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs=
github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8=
github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo=
github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@ -572,6 +576,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

2
monitoring/go.mod
View File

@ -61,7 +61,7 @@ require (
go.uber.org/goleak v1.2.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
golang.org/x/oauth2 v0.2.0 // indirect
golang.org/x/sys v0.3.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect

6
monitoring/go.sum
View File

@ -448,8 +448,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -645,7 +645,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM=
golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

8
schema/schema.go
View File

@ -2772,10 +2772,18 @@ type SubRepoPermissions struct {
UserCacheTTLSeconds int `json:"userCacheTTLSeconds,omitempty"`
}
// SymbolConfiguration description: Configure symbol generation
type SymbolConfiguration struct {
// Engine description: Manually specify overrides for symbol generation engine per language
Engine map[string]string `json:"engine"`
}
// SyntaxHighlighting description: Syntax highlighting configuration
type SyntaxHighlighting struct {
Engine SyntaxHighlightingEngine `json:"engine"`
Languages SyntaxHighlightingLanguage `json:"languages"`
// Symbols description: Configure symbol generation
Symbols SymbolConfiguration `json:"symbols"`
}
type SyntaxHighlightingEngine struct {
// Default description: The default syntax highlighting engine to use

18
schema/site.schema.json
View File

@ -848,7 +848,7 @@
"title": "SyntaxHighlighting",
"description": "Syntax highlighting configuration",
"type": "object",
"required": ["engine", "languages"],
"required": ["engine", "languages", "symbols"],
"properties": {
"engine": {
"title": "SyntaxHighlightingEngine",
@ -903,6 +903,22 @@
}
}
}
},
"symbols": {
"title": "SymbolConfiguration",
"description": "Configure symbol generation",
"type": "object",
"required": ["engine"],
"properties": {
"engine": {
"description": "Manually specify overrides for symbol generation engine per language",
"type": "object",
"additionalProperties": {
"type": "string",
"enum": ["universal-ctags", "scip-ctags", "off"]
}
}
}
}
},
"examples": [

5
sg.config.yaml
View File

@ -340,6 +340,7 @@ commands:
checkBinary: .bin/oss-symbols
env:
CTAGS_COMMAND: dev/universal-ctags-dev
SCIP_CTAGS_COMMAND: dev/scip-ctags-dev
CTAGS_PROCESSES: 2
watch:
- lib
@ -358,6 +359,7 @@ commands:
checkBinary: .bin/symbols
env:
CTAGS_COMMAND: dev/universal-ctags-dev
SCIP_CTAGS_COMMAND: dev/scip-ctags-dev
CTAGS_PROCESSES: 2
USE_ROCKSKIP: "false"
watch:
@ -544,6 +546,7 @@ commands:
checkBinary: .bin/zoekt-sourcegraph-indexserver
env: &zoektenv
CTAGS_COMMAND: dev/universal-ctags-dev
SCIP_CTAGS_COMMAND: dev/scip-ctags-dev
GRPC_ENABLED: true
zoekt-index-0:
@ -956,6 +959,7 @@ bazelCommands:
target: //cmd/symbols
env:
CTAGS_COMMAND: dev/universal-ctags-dev
SCIP_CTAGS_COMMAND: dev/scip-ctags-dev
CTAGS_PROCESSES: 2
oss-gitserver-0:
target: //cmd/gitserver
@ -1023,6 +1027,7 @@ bazelCommands:
checkBinary: .bin/symbols
env:
CTAGS_COMMAND: dev/universal-ctags-dev
SCIP_CTAGS_COMMAND: dev/scip-ctags-dev
CTAGS_PROCESSES: 2
USE_ROCKSKIP: "false"
gitserver-template: &gitserver_bazel_template