From 1ff6fb12d9066d8a082184031530bfe82abfd813 Mon Sep 17 00:00:00 2001 From: TJ DeVries Date: Tue, 30 May 2023 11:53:36 -0400 Subject: [PATCH] syntax-highlighter: scip-ctags implementation (#50600) - Write Rust one-for-one protocol compatible replacement for universal-ctags in JSON streaming mode (scip-ctags) - Use tree-sitter to generate scip symbols, then emit those through scip-ctags - These symbols will be reused for Cody context Currently, only zig is enabled (so other languages should remain unaffected by this change). We will add other languages throughout the next week as we're able to check them off. ## Test plan Unit and snapshot tests in the Rust symbol generation code - verified working in the symbols sidebar and symbol search for enabled languages. --------- Co-authored-by: SuperAuguste <19855629+SuperAuguste@users.noreply.github.com> Co-authored-by: William Bezuidenhout Co-authored-by: Eric Fritz Co-authored-by: Eric Fritz --- cmd/frontend/internal/highlight/BUILD.bazel | 3 +- cmd/frontend/internal/highlight/language.go | 41 +- cmd/server/build-bazel.sh | 2 + cmd/symbols/Dockerfile | 21 +- cmd/symbols/Dockerfile.bazel | 3 + cmd/symbols/build-bazel.sh | 27 +- cmd/symbols/build-ctags.sh | 34 +- cmd/symbols/cargo-config.sh | 8 + cmd/symbols/ctags-install-alpine.sh | 2 +- cmd/symbols/internal/api/BUILD.bazel | 1 + cmd/symbols/internal/api/handler_test.go | 3 +- cmd/symbols/parser/BUILD.bazel | 5 + cmd/symbols/parser/config.go | 63 + cmd/symbols/parser/parser.go | 49 +- cmd/symbols/parser/parser_pool.go | 47 +- cmd/symbols/shared/BUILD.bazel | 1 + cmd/symbols/shared/sqlite.go | 9 +- cmd/symbols/types/types.go | 11 +- dev/scip-ctags-dev | 22 + dev/scip-ctags-install.sh | 35 + docker-images/syntax-highlighter/.gitignore | 6 + docker-images/syntax-highlighter/BUILD.bazel | 15 + .../syntax-highlighter/Cargo.Bazel.lock | 1732 ++++++++++++++++- docker-images/syntax-highlighter/Cargo.lock | 345 +++- docker-images/syntax-highlighter/Cargo.toml | 9 +- .../crates/ctags/Cargo.toml | 10 - .../crates/ctags/src/lib.rs | 92 - .../crates/scip-syntax/Cargo.toml | 5 + .../scip-syntax/queries/c/scip-tags.scm | 11 + .../scip-syntax/queries/c_sharp/scip-tags.scm | 11 + .../scip-syntax/queries/cpp/scip-tags.scm | 10 + .../scip-syntax/queries/go/scip-tags.scm | 9 +- .../scip-syntax/queries/java/scip-tags.scm | 16 + .../queries/javascript/scip-tags.scm | 22 + .../scip-syntax/queries/kotlin/scip-tags.scm | 14 + .../scip-syntax/queries/python/scip-tags.scm | 8 + .../scip-syntax/queries/ruby/scip-tags.scm | 4 + .../scip-syntax/queries/rust/scip-tags.scm | 5 +- .../scip-syntax/queries/scala/scip-tags.scm | 19 + .../queries/typescript/scip-tags.scm | 42 + .../scip-syntax/queries/zig/scip-tags.scm | 17 + .../crates/scip-syntax/src/ctags.rs | 307 +++ .../crates/scip-syntax/src/globals.rs | 305 +++ .../crates/scip-syntax/src/languages.rs | 254 ++- .../crates/scip-syntax/src/lib.rs | 63 +- .../crates/scip-syntax/src/locals.rs | 53 +- .../crates/scip-syntax/src/matches.rs | 345 ---- ...antic__locals__test__can_do_functions.snap | 45 - ...cip_semantic__locals__test__can_do_go.snap | 31 - ...c__locals__test__can_do_nested_locals.snap | 42 - ...tic__matches__test__can_parse_go_tree.snap | 29 - ...c__matches__test__can_parse_rust_tree.snap | 34 - ...ntax__ctags__test__ctags_runner_basic.snap | 10 + ...als__test__can_parse_go_internal_tree.snap | 41 + ...tax__globals__test__can_parse_go_tree.snap | 29 + ...__globals__test__can_parse_rust_tree.snap} | 4 +- ...tax__matches__test__can_parse_go_tree.snap | 29 - ...tax__test__generate_ctags_empty_scope.snap | 10 + ...ntax__test__generate_ctags_go_globals.snap | 12 + ...ax__test__generate_ctags_java_globals.snap | 31 + ...__test__generate_ctags_python_globals.snap | 10 + ...st__generate_ctags_typescript_globals.snap | 22 + ...tax__test__generate_ctags_zig_globals.snap | 27 + .../crates/scip-syntax/src/ts_scip.rs | 1 + .../scip-syntax/testdata/ctags-empty-scope.rs | 30 + .../crates/scip-syntax/testdata/globals.java | 39 + .../crates/scip-syntax/testdata/globals.py | 28 + .../crates/scip-syntax/testdata/globals.ts | 33 + .../crates/scip-syntax/testdata/globals.zig | 49 + .../crates/scip-syntax/testdata/go-globals.go | 38 + .../scip-syntax/testdata/internal_go.go | 27 + .../scip-treesitter-languages/src/parsers.rs | 52 + .../crates/scip-treesitter/src/types.rs | 78 + docker-images/syntax-highlighter/insta.yaml | 2 + docker-images/syntax-highlighter/justfile | 7 + .../syntax-highlighter/src/bin/scip-ctags.rs | 12 + docker-images/syntax-highlighter/src/main.rs | 67 +- enterprise/cmd/symbols/build-bazel.sh | 33 +- enterprise/cmd/symbols/shared/BUILD.bazel | 1 + enterprise/cmd/symbols/shared/setup.go | 7 +- go.mod | 4 +- go.sum | 8 +- internal/ctags_config/BUILD.bazel | 9 + internal/ctags_config/ctags_config.go | 42 + lib/codeintel/languages/BUILD.bazel | 12 + lib/codeintel/languages/languages.go | 53 + lib/go.mod | 5 +- lib/go.sum | 6 + monitoring/go.mod | 2 +- monitoring/go.sum | 6 +- schema/schema.go | 8 + schema/site.schema.json | 18 +- sg.config.yaml | 5 + 93 files changed, 4304 insertions(+), 900 deletions(-) create mode 100755 cmd/symbols/cargo-config.sh create mode 100644 cmd/symbols/parser/config.go create mode 100755 dev/scip-ctags-dev create mode 100755 dev/scip-ctags-install.sh delete mode 100644 docker-images/syntax-highlighter/crates/ctags/Cargo.toml delete mode 100644 docker-images/syntax-highlighter/crates/ctags/src/lib.rs create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/c/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/c_sharp/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/cpp/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/java/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/javascript/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/kotlin/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/python/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/ruby/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/scala/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/typescript/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/queries/zig/scip-tags.scm create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs delete mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/matches.rs delete mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_functions.snap delete mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_go.snap delete mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_nested_locals.snap delete mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_go_tree.snap delete mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_rust_tree.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__ctags__test__ctags_runner_basic.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_internal_tree.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_tree.snap rename docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/{scip_syntax__matches__test__can_parse_rust_tree.snap => scip_syntax__globals__test__can_parse_rust_tree.snap} (94%) delete mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_go_tree.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_empty_scope.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_go_globals.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_java_globals.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_python_globals.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_typescript_globals.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_zig_globals.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/ctags-empty-scope.rs create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.java create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.py create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.ts create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.zig create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/go-globals.go create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/internal_go.go create mode 100644 docker-images/syntax-highlighter/insta.yaml create mode 100644 docker-images/syntax-highlighter/justfile create mode 100644 docker-images/syntax-highlighter/src/bin/scip-ctags.rs create mode 100644 internal/ctags_config/BUILD.bazel create mode 100644 internal/ctags_config/ctags_config.go create mode 100644 lib/codeintel/languages/BUILD.bazel create mode 100644 lib/codeintel/languages/languages.go diff --git a/cmd/frontend/internal/highlight/BUILD.bazel b/cmd/frontend/internal/highlight/BUILD.bazel index 56c17e1c8d3..b0c23fbb6e4 100644 --- a/cmd/frontend/internal/highlight/BUILD.bazel +++ b/cmd/frontend/internal/highlight/BUILD.bazel @@ -22,10 +22,10 @@ go_library( "//internal/honey", "//internal/observation", "//internal/trace/ot", + "//lib/codeintel/languages", "//lib/errors", "@com_github_alecthomas_chroma_v2//:chroma", "@com_github_alecthomas_chroma_v2//lexers", - "@com_github_go_enry_go_enry_v2//:go-enry", "@com_github_grafana_regexp//:regexp", "@com_github_inconshreveable_log15//:log15", "@com_github_prometheus_client_golang//prometheus", @@ -33,7 +33,6 @@ go_library( "@com_github_sourcegraph_scip//bindings/go/scip", "@io_opentelemetry_go_otel//attribute", "@org_golang_google_protobuf//proto", - "@org_golang_x_exp//slices", "@org_golang_x_net//html", "@org_golang_x_net//html/atom", ], diff --git a/cmd/frontend/internal/highlight/language.go b/cmd/frontend/internal/highlight/language.go index 1dd68fb46fd..e93c811f5ed 100644 --- a/cmd/frontend/internal/highlight/language.go +++ b/cmd/frontend/internal/highlight/language.go @@ -5,13 +5,12 @@ import ( "path/filepath" "strings" - "github.com/go-enry/go-enry/v2" "github.com/grafana/regexp" - "golang.org/x/exp/slices" "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/gosyntect" + "github.com/sourcegraph/sourcegraph/lib/codeintel/languages" ) type EngineType int @@ -247,45 +246,11 @@ func getLanguage(path string, contents string) (string, bool) { return lang, true } - // Force the use of the shebang. - if shebangLang, ok := overrideViaShebang(path, contents); ok { - return shebangLang, false - } - - // Lastly, fall back to whatever enry decides is a useful algorithm for calculating. - lang = enry.GetLanguage(path, []byte(contents)) - + // TODO: Consider if we should just ignore getting empty...? + lang, _ = languages.GetLanguage(path, contents) return lang, false } -// overrideViaShebang handles explicitly using the shebang whenever possible. -// -// It also covers some edge cases when enry eagerly returns more languages -// than necessary, which ends up overriding the shebang completely (which, -// IMO is the highest priority match we can have). -// -// For example, enry will return "Perl" and "Pod" for a shebang of `#!/usr/bin/env perl`. -// This is actually unhelpful, because then enry will *not* select "Perl" as the -// language (which is our desired behavior). -func overrideViaShebang(path, content string) (lang string, ok bool) { - shebangs := enry.GetLanguagesByShebang(path, []byte(content), []string{}) - if len(shebangs) == 0 { - return "", false - } - - if len(shebangs) == 1 { - return shebangs[0], true - } - - // There are some shebangs that enry returns that are not really - // useful for our syntax highlighters to distinguish between. - if slices.Equal(shebangs, []string{"Perl", "Pod"}) { - return "Perl", true - } - - return "", false -} - // DetectSyntaxHighlightingLanguage will calculate the SyntaxEngineQuery from a given // path and contents. First it will determine if there are any configuration overrides // and then, if none, return the 'enry' default language detection diff --git a/cmd/server/build-bazel.sh b/cmd/server/build-bazel.sh index 9bdb37c6494..349a73efd6a 100755 --- a/cmd/server/build-bazel.sh +++ b/cmd/server/build-bazel.sh @@ -28,6 +28,7 @@ OSS_TARGETS=( //cmd/gitserver //cmd/searcher //cmd/server + //docker-images/syntax-highlighter:scip-ctags # https://github.com/sourcegraph/s3proxy is still the default for now. # //cmd/blobstore @com_github_sourcegraph_zoekt//cmd/zoekt-archive-index @@ -46,6 +47,7 @@ ENTERPRISE_TARGETS=( //enterprise/cmd/repo-updater //enterprise/cmd/precise-code-intel-worker //enterprise/cmd/server + //docker-images/syntax-highlighter:scip-ctags ) MUSL_TARGETS=( diff --git a/cmd/symbols/Dockerfile b/cmd/symbols/Dockerfile index 109ff522ad8..b053ad941ed 100644 --- a/cmd/symbols/Dockerfile +++ b/cmd/symbols/Dockerfile @@ -1,11 +1,27 @@ # NOTE: This layer of the docker image is also used in local development as a wrapper around universal-ctags FROM sourcegraph/alpine-3.14:213466_2023-04-17_5.0-bdda34a71619@sha256:6354a4ff578b685e36c8fbde81f62125ae0011b047fb2cc22d1b0de616b3c59a AS ctags # hadolint ignore=DL3002 -USER root - COPY cmd/symbols/ctags-install-alpine.sh /ctags-install-alpine.sh RUN /ctags-install-alpine.sh +FROM rust:1.68.0-alpine3.17@sha256:d119a621ae12f84ec0c5fed77c24795120ed1c7874b2428b5a6ccc0f294dbe18 as scip-ctags +# hadolint ignore=DL3002 +USER root +RUN apk add --no-cache musl-dev>=1.1.24-r10 build-base +COPY docker-images/syntax-highlighter /repo +WORKDIR /repo +RUN cargo fetch + +ARG TARGETARCH + +# Because .cargo/config.toml doesnt support triplet-specific env +COPY cmd/symbols/cargo-config.sh /cargo-config.sh +RUN /cargo-config.sh + +RUN cargo rustc --release --bin scip-ctags + +RUN cp ./target/release/scip-ctags /usr/local/bin/scip-ctags + FROM golang:1.19.8-alpine@sha256:841c160ed35923d96c95c52403c4e6db5decd9cbce034aa851e412ade5d4b74f AS symbols-build # hadolint ignore=DL3002 USER root @@ -60,6 +76,7 @@ LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/comm RUN apk add --no-cache bind-tools ca-certificates mailcap tini jansson libstdc++ libgcc COPY --from=ctags /usr/local/bin/universal-ctags /usr/local/bin/universal-ctags +COPY --from=scip-ctags /usr/local/bin/scip-ctags /usr/local/bin/scip-ctags COPY --from=symbols-build /symbols /usr/local/bin/symbols diff --git a/cmd/symbols/Dockerfile.bazel b/cmd/symbols/Dockerfile.bazel index 971980849a6..50e0801aa93 100644 --- a/cmd/symbols/Dockerfile.bazel +++ b/cmd/symbols/Dockerfile.bazel @@ -29,6 +29,9 @@ RUN apk add --no-cache bind-tools ca-certificates mailcap tini jansson libstdc++ COPY --from=ctags /usr/local/bin/universal-ctags /usr/local/bin/universal-ctags +# the scip binary and symbols was already built by bazel +# see cmd/symbols/build-bazel.sh where it is built and put in the context directory aka $OUTPUT for docker +COPY scip-ctags /usr/local/bin/scip-ctags COPY symbols /usr/local/bin/symbols # symbols is cgo, ensure we have the requisite dynamic libraries diff --git a/cmd/symbols/build-bazel.sh b/cmd/symbols/build-bazel.sh index 91ef8ae50a5..736864bdc6f 100755 --- a/cmd/symbols/build-bazel.sh +++ b/cmd/symbols/build-bazel.sh @@ -11,7 +11,7 @@ cleanup() { } trap cleanup EXIT -echo "--- bazel build" +echo "--- :bazel: bazel build for targets //cmd/symbols" bazelrc=( --bazelrc=.bazelrc @@ -23,7 +23,6 @@ if [[ ${CI:-""} == "true" ]]; then ) fi - bazel "${bazelrc[@]}" \ build \ //cmd/symbols \ @@ -39,9 +38,31 @@ out=$( --config incompat-zig-linux-amd64 \ --output=files ) -cp "$out" "$OUTPUT" +cp -v "$out" "$OUTPUT" + +# we can't build scip-ctags with symbols since the platform args conflict +# NOTE: cmd/symbols/cargo-config.sh sets some specific config when running on arm64 +# since this bazel run typically runs on CI that config change isn't made +echo "--- :bazel: bazel build for target //docker-images/syntax-highlighter:scip-ctags" +bazel "${bazelrc[@]}" \ + build //docker-images/syntax-highlighter:scip-ctags \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh + +out=$( + bazel "${bazelrc[@]}" \ + cquery //docker-images/syntax-highlighter:scip-ctags \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --output=files +) +cp -v "$out" "$OUTPUT" + cp cmd/symbols/ctags-install-alpine.sh "$OUTPUT" +echo ":docker: context directory contains the following:" +ls -lah "$OUTPUT" +echo "--- :docker: docker build for symbols" docker build -f cmd/symbols/Dockerfile.bazel -t "$IMAGE" "$OUTPUT" \ --progress=plain \ --build-arg COMMIT_SHA \ diff --git a/cmd/symbols/build-ctags.sh b/cmd/symbols/build-ctags.sh index 8f2b4faaff5..7970ebb44a0 100755 --- a/cmd/symbols/build-ctags.sh +++ b/cmd/symbols/build-ctags.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This script builds the ctags image for local development. +# This script builds the ctags images for local development. cd "$(dirname "${BASH_SOURCE[0]}")/../.." set -eu @@ -9,12 +9,30 @@ set -eu # image. See /dev/universal-ctags-dev. if [[ "${CTAGS_COMMAND}" != "dev/universal-ctags-dev" ]]; then echo "CTAGS_COMMAND set to custom executable. Building of Docker image not necessary." - exit 0 +else + # Build ctags docker image for universal-ctags-dev + echo "Building universal-ctags docker image" + docker build -f cmd/symbols/Dockerfile -t ctags . \ + --platform linux/amd64 \ + --target=ctags \ + --progress=plain fi -# Build ctags docker image for universal-ctags-dev -echo "Building ctags docker image" -docker build -f cmd/symbols/Dockerfile -t ctags . \ - --platform linux/amd64 \ - --target=ctags \ - --progress=plain +# If SCIP_CTAGS_COMMAND is set to a custom executable, we don't need to build the +# image. See /dev/scip-ctags-dev. +if [[ "${SCIP_CTAGS_COMMAND}" != "dev/scip-ctags-dev" ]]; then + echo "SCIP_CTAGS_COMMAND set to custom executable. Building of Docker image or Rust code not necessary." +else + if [[ "$(uname -m)" == "arm64" ]]; then + # build ctags with cargo; prevent x86-64 slowdown on mac + root="$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null + "$root"/dev/scip-ctags-install.sh + else + # Build ctags docker image for scip-ctags-dev + echo "Building scip-ctags docker image" + docker build -f cmd/symbols/Dockerfile -t scip-ctags . \ + --platform linux/amd64 \ + --target=scip-ctags \ + --progress=plain + fi +fi diff --git a/cmd/symbols/cargo-config.sh b/cmd/symbols/cargo-config.sh new file mode 100755 index 00000000000..bbb1ae155c0 --- /dev/null +++ b/cmd/symbols/cargo-config.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +if [ "${TARGETARCH}" = "arm64" ]; then + cat <<- FOE >> .cargo/config.toml + [env] + CFLAGS="-mno-outline-atomics" + FOE +fi; diff --git a/cmd/symbols/ctags-install-alpine.sh b/cmd/symbols/ctags-install-alpine.sh index e07500bbbc9..ce7bb50c5c4 100755 --- a/cmd/symbols/ctags-install-alpine.sh +++ b/cmd/symbols/ctags-install-alpine.sh @@ -1,6 +1,6 @@ #!/bin/sh -# This script installs ctags within an alpine container. +# This script installs universal-ctags within an alpine container. # Commit hash of github.com/universal-ctags/ctags. # Last bumped 2022-04-04. diff --git a/cmd/symbols/internal/api/BUILD.bazel b/cmd/symbols/internal/api/BUILD.bazel index 2f07eb8ef88..a35beaa0167 100644 --- a/cmd/symbols/internal/api/BUILD.bazel +++ b/cmd/symbols/internal/api/BUILD.bazel @@ -52,6 +52,7 @@ go_test( "//cmd/symbols/internal/database/writer", "//cmd/symbols/parser", "//internal/api", + "//internal/ctags_config", "//internal/database", "//internal/diskcache", "//internal/endpoint", diff --git a/cmd/symbols/internal/api/handler_test.go b/cmd/symbols/internal/api/handler_test.go index 5edf4048b17..f3f3af7be9f 100644 --- a/cmd/symbols/internal/api/handler_test.go +++ b/cmd/symbols/internal/api/handler_test.go @@ -17,6 +17,7 @@ import ( symbolsdatabase "github.com/sourcegraph/sourcegraph/cmd/symbols/internal/database" "github.com/sourcegraph/sourcegraph/cmd/symbols/internal/database/writer" "github.com/sourcegraph/sourcegraph/cmd/symbols/parser" + "github.com/sourcegraph/sourcegraph/internal/ctags_config" "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/diskcache" "github.com/sourcegraph/sourcegraph/internal/endpoint" @@ -38,7 +39,7 @@ func TestHandler(t *testing.T) { cache := diskcache.NewStore(tmpDir, "symbols", diskcache.WithBackgroundTimeout(20*time.Minute)) - parserFactory := func() (ctags.Parser, error) { + parserFactory := func(source ctags_config.ParserType) (ctags.Parser, error) { pathToEntries := map[string][]*ctags.Entry{ "a.js": { { diff --git a/cmd/symbols/parser/BUILD.bazel b/cmd/symbols/parser/BUILD.bazel index 1b3ecda0de3..62e95c34a26 100644 --- a/cmd/symbols/parser/BUILD.bazel +++ b/cmd/symbols/parser/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "parser", srcs = [ + "config.go", "filtering_parser.go", "observability.go", "parser.go", @@ -13,10 +14,14 @@ go_library( deps = [ "//cmd/symbols/fetcher", "//cmd/symbols/types", + "//internal/conf", + "//internal/conf/conftypes", + "//internal/ctags_config", "//internal/metrics", "//internal/observation", "//internal/search", "//internal/search/result", + "//lib/codeintel/languages", "//lib/errors", "@com_github_inconshreveable_log15//:log15", "@com_github_prometheus_client_golang//prometheus", diff --git a/cmd/symbols/parser/config.go b/cmd/symbols/parser/config.go new file mode 100644 index 00000000000..258f23783bd --- /dev/null +++ b/cmd/symbols/parser/config.go @@ -0,0 +1,63 @@ +package parser + +import ( + "fmt" + + "github.com/sourcegraph/sourcegraph/internal/conf" + "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" + "github.com/sourcegraph/sourcegraph/internal/ctags_config" +) + +var parserConfig = ctags_config.ParserConfiguration{ + Default: ctags_config.UniversalCtags, + Engine: map[string]ctags_config.ParserType{}, +} + +func init() { + // Validation only: Do NOT set any values in the configuration in this function. + conf.ContributeValidator(func(c conftypes.SiteConfigQuerier) (problems conf.Problems) { + configuration := c.SiteConfig().SyntaxHighlighting + if configuration == nil { + return + } + + for _, engine := range configuration.Symbols.Engine { + if _, err := ctags_config.ParserNameToParserType(engine); err != nil { + problems = append(problems, conf.NewSiteProblem(fmt.Sprintf("Not a valid Symbols.Engine: `%s`.", engine))) + } + } + + return + }) + + go func() { + conf.Watch(func() { + c := conf.Get() + + configuration := c.SiteConfig().SyntaxHighlighting + + // Set the defaults + parserConfig.Engine = make(map[string]ctags_config.ParserType) + for lang, engine := range ctags_config.BaseParserConfig.Engine { + parserConfig.Engine[lang] = engine + } + + if configuration != nil { + for lang, engine := range configuration.Symbols.Engine { + if engine, err := ctags_config.ParserNameToParserType(engine); err != nil { + parserConfig.Engine[lang] = engine + } + } + } + }) + }() +} + +func GetParserType(language string) ctags_config.ParserType { + parserType, ok := parserConfig.Engine[language] + if !ok { + return parserConfig.Default + } else { + return parserType + } +} diff --git a/cmd/symbols/parser/parser.go b/cmd/symbols/parser/parser.go index 1846b661be5..a7874b06bf9 100644 --- a/cmd/symbols/parser/parser.go +++ b/cmd/symbols/parser/parser.go @@ -15,9 +15,11 @@ import ( "github.com/sourcegraph/sourcegraph/cmd/symbols/fetcher" "github.com/sourcegraph/sourcegraph/cmd/symbols/types" + "github.com/sourcegraph/sourcegraph/internal/ctags_config" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/search" "github.com/sourcegraph/sourcegraph/internal/search/result" + "github.com/sourcegraph/sourcegraph/lib/codeintel/languages" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -134,14 +136,29 @@ func min(a, b int) int { return b } -func (p *parser) handleParseRequest(ctx context.Context, symbolOrErrors chan<- SymbolOrError, parseRequest fetcher.ParseRequest, totalSymbols *uint32) (err error) { +func (p *parser) handleParseRequest( + ctx context.Context, + symbolOrErrors chan<- SymbolOrError, + parseRequest fetcher.ParseRequest, + totalSymbols *uint32, +) (err error) { ctx, trace, endObservation := p.operations.handleParseRequest.With(ctx, &err, observation.Args{Attrs: []attribute.KeyValue{ attribute.String("path", parseRequest.Path), attribute.Int("fileSize", len(parseRequest.Data)), }}) defer endObservation(1, observation.Args{}) - parser, err := p.parserFromPool(ctx) + language, found := languages.GetLanguage(parseRequest.Path, string(parseRequest.Data)) + if !found { + return nil + } + + source := GetParserType(language) + if source == ctags_config.NoCtags || source == ctags_config.UnknownCtags { + return nil + } + + parser, err := p.parserFromPool(ctx, source) if err != nil { return err } @@ -155,12 +172,12 @@ func (p *parser) handleParseRequest(ctx context.Context, symbolOrErrors chan<- S } if err == nil { - p.parserPool.Done(parser) + p.parserPool.Done(parser, source) } else { // Close parser and return nil to pool, indicating that the next receiver should create a new parser log15.Error("Closing failed parser", "error", err) parser.Close() - p.parserPool.Done(nil) + p.parserPool.Done(nil, source) p.operations.parseFailed.Inc() } }() @@ -219,11 +236,15 @@ func (p *parser) handleParseRequest(ctx context.Context, symbolOrErrors chan<- S return nil } -func (p *parser) parserFromPool(ctx context.Context) (ctags.Parser, error) { +func (p *parser) parserFromPool(ctx context.Context, source ctags_config.ParserType) (ctags.Parser, error) { + if source == ctags_config.NoCtags || source == ctags_config.UnknownCtags { + return nil, errors.New("Should not pass NoCtags to this function") + } + p.operations.parseQueueSize.Inc() defer p.operations.parseQueueSize.Dec() - parser, err := p.parserPool.Get(ctx) + parser, err := p.parserPool.Get(ctx, source) if err != nil { if err == context.DeadlineExceeded { p.operations.parseQueueTimeouts.Inc() @@ -250,12 +271,20 @@ func shouldPersistEntry(e *ctags.Entry) bool { return true } -func SpawnCtags(logger log.Logger, ctagsConfig types.CtagsConfig) (ctags.Parser, error) { +func SpawnCtags(logger log.Logger, ctagsConfig types.CtagsConfig, source ctags_config.ParserType) (ctags.Parser, error) { logger = logger.Scoped("ctags", "ctags processes") - options := ctags.Options{ - Bin: ctagsConfig.Command, - PatternLengthLimit: ctagsConfig.PatternLengthLimit, + var options ctags.Options + if source == ctags_config.UniversalCtags { + options = ctags.Options{ + Bin: ctagsConfig.UniversalCommand, + PatternLengthLimit: ctagsConfig.PatternLengthLimit, + } + } else { + options = ctags.Options{ + Bin: ctagsConfig.ScipCommand, + PatternLengthLimit: ctagsConfig.PatternLengthLimit, + } } if ctagsConfig.LogErrors { options.Info = std.NewLogger(logger, log.LevelInfo) diff --git a/cmd/symbols/parser/parser_pool.go b/cmd/symbols/parser/parser_pool.go index 73e17b807cc..57a6ece607a 100644 --- a/cmd/symbols/parser/parser_pool.go +++ b/cmd/symbols/parser/parser_pool.go @@ -4,29 +4,39 @@ import ( "context" "github.com/sourcegraph/go-ctags" + + "github.com/sourcegraph/sourcegraph/internal/ctags_config" + "github.com/sourcegraph/sourcegraph/lib/errors" ) -type ParserFactory func() (ctags.Parser, error) +type ParserFactory func(ctags_config.ParserType) (ctags.Parser, error) type parserPool struct { newParser ParserFactory - pool chan ctags.Parser + pool map[ctags_config.ParserType]chan ctags.Parser } func NewParserPool(newParser ParserFactory, numParserProcesses int) (*parserPool, error) { - pool := make(chan ctags.Parser, numParserProcesses) - for i := 0; i < numParserProcesses; i++ { - parser, err := newParser() - if err != nil { - return nil, err + pool := make(map[ctags_config.ParserType]chan ctags.Parser) + + // NOTE: We obviously don't make `NoCtags` available in the pool. + for _, parserType := range []ctags_config.ParserType{ctags_config.UniversalCtags, ctags_config.ScipCtags} { + pool[parserType] = make(chan ctags.Parser, numParserProcesses) + for i := 0; i < numParserProcesses; i++ { + parser, err := newParser(parserType) + if err != nil { + return nil, err + } + pool[parserType] <- parser } - pool <- parser } - return &parserPool{ + parserPool := &parserPool{ newParser: newParser, pool: pool, - }, nil + } + + return parserPool, nil } // Get a parser from the pool. Once this parser is no longer in use, the Done method @@ -35,20 +45,27 @@ func NewParserPool(newParser ParserFactory, numParserProcesses int) (*parserPool // the pool. This method always returns a non-nil parser with a nil error value. // // This method blocks until a parser is available or the given context is canceled. -func (p *parserPool) Get(ctx context.Context) (ctags.Parser, error) { +func (p *parserPool) Get(ctx context.Context, source ctags_config.ParserType) (ctags.Parser, error) { + if source == ctags_config.NoCtags || source == ctags_config.UnknownCtags { + return nil, errors.New("NoCtags is not a valid ParserType") + } + + pool := p.pool[source] + select { - case parser := <-p.pool: + case parser := <-pool: if parser != nil { return parser, nil } - return p.newParser() + return p.newParser(source) case <-ctx.Done(): return nil, ctx.Err() } } -func (p *parserPool) Done(parser ctags.Parser) { - p.pool <- parser +func (p *parserPool) Done(parser ctags.Parser, source ctags_config.ParserType) { + pool := p.pool[source] + pool <- parser } diff --git a/cmd/symbols/shared/BUILD.bazel b/cmd/symbols/shared/BUILD.bazel index e39544e356c..ce4b4b08cb4 100644 --- a/cmd/symbols/shared/BUILD.bazel +++ b/cmd/symbols/shared/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//internal/conf", "//internal/conf/conftypes", "//internal/conf/deploy", + "//internal/ctags_config", "//internal/database", "//internal/database/connections/live", "//internal/debugserver", diff --git a/cmd/symbols/shared/sqlite.go b/cmd/symbols/shared/sqlite.go index ade08d01b57..6fd6dec16aa 100644 --- a/cmd/symbols/shared/sqlite.go +++ b/cmd/symbols/shared/sqlite.go @@ -19,6 +19,7 @@ import ( symbolparser "github.com/sourcegraph/sourcegraph/cmd/symbols/parser" "github.com/sourcegraph/sourcegraph/cmd/symbols/types" "github.com/sourcegraph/sourcegraph/internal/conf/deploy" + "github.com/sourcegraph/sourcegraph/internal/ctags_config" "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/diskcache" "github.com/sourcegraph/sourcegraph/internal/goroutine" @@ -46,7 +47,7 @@ func SetupSqlite(observationCtx *observation.Context, db database.DB, gitserverC // anything that tries to open a SQLite database. sqlite.Init() - if deploy.IsSingleBinary() && config.Ctags.Command == "" { + if deploy.IsSingleBinary() && config.Ctags.UniversalCommand == "" { // app: ctags is not available searchFunc := func(ctx context.Context, params search.SymbolsParameters) (result.Symbols, error) { return nil, nil @@ -54,8 +55,8 @@ func SetupSqlite(observationCtx *observation.Context, db database.DB, gitserverC return searchFunc, nil, []goroutine.BackgroundRoutine{}, "", nil } - parserFactory := func() (ctags.Parser, error) { - return symbolparser.SpawnCtags(logger, config.Ctags) + parserFactory := func(source ctags_config.ParserType) (ctags.Parser, error) { + return symbolparser.SpawnCtags(logger, config.Ctags, source) } parserPool, err := symbolparser.NewParserPool(parserFactory, config.NumCtagsProcesses) if err != nil { @@ -76,5 +77,5 @@ func SetupSqlite(observationCtx *observation.Context, db database.DB, gitserverC cacheSizeBytes := int64(config.CacheSizeMB) * 1000 * 1000 cacheEvicter := janitor.NewCacheEvicter(evictionInterval, cache, cacheSizeBytes, janitor.NewMetrics(observationCtx)) - return searchFunc, nil, []goroutine.BackgroundRoutine{cacheEvicter}, config.Ctags.Command, nil + return searchFunc, nil, []goroutine.BackgroundRoutine{cacheEvicter}, config.Ctags.UniversalCommand, nil } diff --git a/cmd/symbols/types/types.go b/cmd/symbols/types/types.go index baced279968..29f60e8b38d 100644 --- a/cmd/symbols/types/types.go +++ b/cmd/symbols/types/types.go @@ -45,7 +45,8 @@ func LoadSqliteConfig(baseConfig env.BaseConfig, ctags CtagsConfig, repositoryFe } type CtagsConfig struct { - Command string + UniversalCommand string + ScipCommand string PatternLengthLimit int LogErrors bool DebugLogs bool @@ -64,8 +65,14 @@ func LoadCtagsConfig(baseConfig env.BaseConfig) CtagsConfig { ctagsCommandDefault = "" } + scipCtagsCommandDefault := "scip-ctags" + if deploy.IsSingleBinary() { + scipCtagsCommandDefault = "" + } + return CtagsConfig{ - Command: baseConfig.Get("CTAGS_COMMAND", ctagsCommandDefault, "ctags command (should point to universal-ctags executable compiled with JSON and seccomp support)"), + UniversalCommand: baseConfig.Get("CTAGS_COMMAND", ctagsCommandDefault, "ctags command (should point to universal-ctags executable compiled with JSON and seccomp support)"), + ScipCommand: baseConfig.Get("SCIP_CTAGS_COMMAND", scipCtagsCommandDefault, "scip-ctags command"), PatternLengthLimit: baseConfig.GetInt("CTAGS_PATTERN_LENGTH_LIMIT", "250", "the maximum length of the patterns output by ctags"), LogErrors: baseConfig.GetBool("LOG_CTAGS_ERRORS", logCtagsErrorsDefault, "log ctags errors"), DebugLogs: false, diff --git a/dev/scip-ctags-dev b/dev/scip-ctags-dev new file mode 100755 index 00000000000..fda37498e82 --- /dev/null +++ b/dev/scip-ctags-dev @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# This script is a wrapper around `scip-ctags`. +# +# It checks if scip-ctags has been installed through ./dev/scip-ctags-install.sh or falls back to a dockerized version. +# +# To use your own `scip-ctags` binary instead of this wrapper in your local dev server, use +# `SCIP_CTAGS_COMMAND=path/to/ctags sg start`. + +root="$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null +TARGET=$("$root/dev/scip-ctags-install.sh" which) + +if [ ! -f "${TARGET}" ]; then + exec docker run --rm -i \ + -a stdin -a stdout -a stderr \ + --user guest \ + --name=scip-ctags-$$ \ + --entrypoint /usr/local/bin/scip-ctags \ + scip-ctags "$@" +else + ${TARGET} "$@" +fi diff --git a/dev/scip-ctags-install.sh b/dev/scip-ctags-install.sh new file mode 100755 index 00000000000..75441910a4d --- /dev/null +++ b/dev/scip-ctags-install.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -euf -o pipefail +pushd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null +mkdir -p .bin + +# TODO: add similar task to zoekt alpine + +NAME="scip-ctags" +TARGET="$PWD/.bin/${NAME}" + +if [ $# -ne 0 ]; then + if [ "$1" == "which" ]; then + echo "$TARGET" + exit 0 + fi +fi + +function ctrl_c() { + printf "[-] Installation cancelled.\n" + exit 1 +} + +trap ctrl_c INT + +function build_scip_ctags { + cd docker-images/syntax-highlighter + cargo fetch + cargo build --bin scip-ctags --release + cp ./target/release/scip-ctags "$TARGET" +} + +build_scip_ctags + +popd >/dev/null diff --git a/docker-images/syntax-highlighter/.gitignore b/docker-images/syntax-highlighter/.gitignore index 5d533915004..be125da8a1b 100644 --- a/docker-images/syntax-highlighter/.gitignore +++ b/docker-images/syntax-highlighter/.gitignore @@ -1,3 +1,9 @@ target languages/libraries/ .vscode/ + +bench_data/ + +**/flamegraph.svg +**/perf.data +**/perf.data.old diff --git a/docker-images/syntax-highlighter/BUILD.bazel b/docker-images/syntax-highlighter/BUILD.bazel index 884b96ff8b0..8e79189032d 100644 --- a/docker-images/syntax-highlighter/BUILD.bazel +++ b/docker-images/syntax-highlighter/BUILD.bazel @@ -12,6 +12,7 @@ rust_binary( normal = True, ) + [ "//docker-images/syntax-highlighter/crates/sg-syntax:sg-syntax", + "//docker-images/syntax-highlighter/crates/scip-syntax:scip-syntax", "//docker-images/syntax-highlighter/crates/scip-treesitter-languages:scip-treesitter-languages", ], ) @@ -31,3 +32,17 @@ rust_test( normal_dev = True, ), ) + +rust_binary( + name = "scip-ctags", + srcs = ["src/bin/scip-ctags.rs"], + aliases = aliases(), + proc_macro_deps = all_crate_deps( + proc_macro = True, + ), + deps = all_crate_deps( + normal = True, + ) + [ + "//docker-images/syntax-highlighter/crates/scip-syntax:scip-syntax", + ], +) diff --git a/docker-images/syntax-highlighter/Cargo.Bazel.lock b/docker-images/syntax-highlighter/Cargo.Bazel.lock index 2a04ff5fc87..dd38fa4b7b8 100644 --- a/docker-images/syntax-highlighter/Cargo.Bazel.lock +++ b/docker-images/syntax-highlighter/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "91500ff2dd3fc0cfb2535fbc16a924bcaa9cb382e21b91005f7d968136274df7", + "checksum": "a83e6ef20d1de75bea46dc774af1b413e4dfde7f02a62f8321ecbe812c120317", "crates": { "adler 1.0.2": { "name": "adler", @@ -77,6 +77,42 @@ }, "license": "Unlicense/MIT" }, + "anes 0.1.6": { + "name": "anes", + "version": "0.1.6", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/anes/0.1.6/download", + "sha256": "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + } + }, + "targets": [ + { + "Library": { + "crate_name": "anes", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "anes", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "edition": "2018", + "version": "0.1.6" + }, + "license": "MIT OR Apache-2.0" + }, "ansi_term 0.12.1": { "name": "ansi_term", "version": "0.12.1", @@ -707,6 +743,66 @@ }, "license": "MIT OR Apache-2.0" }, + "bitvec 1.0.1": { + "name": "bitvec", + "version": "1.0.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/bitvec/1.0.1/download", + "sha256": "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "bitvec", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "bitvec", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "atomic", + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "funty 2.0.0", + "target": "funty" + }, + { + "id": "radium 0.7.0", + "target": "radium" + }, + { + "id": "tap 1.0.1", + "target": "tap" + }, + { + "id": "wyz 0.5.1", + "target": "wyz" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.0.1" + }, + "license": "MIT" + }, "bumpalo 3.9.1": { "name": "bumpalo", "version": "3.9.1", @@ -780,6 +876,36 @@ }, "license": "MIT" }, + "cast 0.3.0": { + "name": "cast", + "version": "0.3.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/cast/0.3.0/download", + "sha256": "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + } + }, + "targets": [ + { + "Library": { + "crate_name": "cast", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "cast", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.3.0" + }, + "license": "MIT OR Apache-2.0" + }, "cc 1.0.73": { "name": "cc", "version": "1.0.73", @@ -840,6 +966,197 @@ }, "license": "MIT/Apache-2.0" }, + "ciborium 0.2.0": { + "name": "ciborium", + "version": "0.2.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/ciborium/0.2.0/download", + "sha256": "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ciborium", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ciborium", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "ciborium-io 0.2.0", + "target": "ciborium_io" + }, + { + "id": "ciborium-ll 0.2.0", + "target": "ciborium_ll" + }, + { + "id": "serde 1.0.136", + "target": "serde" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.0" + }, + "license": "Apache-2.0" + }, + "ciborium-io 0.2.0": { + "name": "ciborium-io", + "version": "0.2.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/ciborium-io/0.2.0/download", + "sha256": "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ciborium_io", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ciborium_io", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.0" + }, + "license": "Apache-2.0" + }, + "ciborium-ll 0.2.0": { + "name": "ciborium-ll", + "version": "0.2.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/ciborium-ll/0.2.0/download", + "sha256": "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "ciborium_ll", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "ciborium_ll", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "ciborium-io 0.2.0", + "target": "ciborium_io" + }, + { + "id": "half 1.8.2", + "target": "half" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.0" + }, + "license": "Apache-2.0" + }, + "clap 3.2.23": { + "name": "clap", + "version": "3.2.23", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/clap/3.2.23/download", + "sha256": "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" + } + }, + "targets": [ + { + "Library": { + "crate_name": "clap", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "clap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "bitflags 1.3.2", + "target": "bitflags" + }, + { + "id": "clap_lex 0.2.4", + "target": "clap_lex" + }, + { + "id": "indexmap 1.8.1", + "target": "indexmap" + }, + { + "id": "textwrap 0.16.0", + "target": "textwrap" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "3.2.23" + }, + "license": "MIT OR Apache-2.0" + }, "clap 4.1.11": { "name": "clap", "version": "4.1.11", @@ -893,7 +1210,7 @@ "target": "is_terminal" }, { - "id": "once_cell 1.13.0", + "id": "once_cell 1.17.1", "target": "once_cell" }, { @@ -982,6 +1299,45 @@ }, "license": "MIT OR Apache-2.0" }, + "clap_lex 0.2.4": { + "name": "clap_lex", + "version": "0.2.4", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/clap_lex/0.2.4/download", + "sha256": "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" + } + }, + "targets": [ + { + "Library": { + "crate_name": "clap_lex", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "clap_lex", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "os_str_bytes 6.5.0", + "target": "os_str_bytes" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.2.4" + }, + "license": "MIT OR Apache-2.0" + }, "clap_lex 0.3.3": { "name": "clap_lex", "version": "0.3.3", @@ -1318,6 +1674,427 @@ }, "license": "MIT OR Apache-2.0" }, + "criterion 0.4.0": { + "name": "criterion", + "version": "0.4.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/criterion/0.4.0/download", + "sha256": "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" + } + }, + "targets": [ + { + "Library": { + "crate_name": "criterion", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "criterion", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "cargo_bench_support", + "default", + "html_reports", + "plotters", + "rayon" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "anes 0.1.6", + "target": "anes" + }, + { + "id": "atty 0.2.14", + "target": "atty" + }, + { + "id": "cast 0.3.0", + "target": "cast" + }, + { + "id": "ciborium 0.2.0", + "target": "ciborium" + }, + { + "id": "clap 3.2.23", + "target": "clap" + }, + { + "id": "criterion-plot 0.5.0", + "target": "criterion_plot" + }, + { + "id": "itertools 0.10.5", + "target": "itertools" + }, + { + "id": "lazy_static 1.4.0", + "target": "lazy_static" + }, + { + "id": "num-traits 0.2.15", + "target": "num_traits" + }, + { + "id": "oorandom 11.1.3", + "target": "oorandom" + }, + { + "id": "plotters 0.3.4", + "target": "plotters" + }, + { + "id": "rayon 1.7.0", + "target": "rayon" + }, + { + "id": "regex 1.5.5", + "target": "regex" + }, + { + "id": "serde 1.0.136", + "target": "serde" + }, + { + "id": "serde_json 1.0.79", + "target": "serde_json" + }, + { + "id": "tinytemplate 1.2.1", + "target": "tinytemplate" + }, + { + "id": "walkdir 2.3.2", + "target": "walkdir" + } + ], + "selects": {} + }, + "edition": "2018", + "proc_macro_deps": { + "common": [ + { + "id": "serde_derive 1.0.136", + "target": "serde_derive" + } + ], + "selects": {} + }, + "version": "0.4.0" + }, + "license": "Apache-2.0/MIT" + }, + "criterion-plot 0.5.0": { + "name": "criterion-plot", + "version": "0.5.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/criterion-plot/0.5.0/download", + "sha256": "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" + } + }, + "targets": [ + { + "Library": { + "crate_name": "criterion_plot", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "criterion_plot", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cast 0.3.0", + "target": "cast" + }, + { + "id": "itertools 0.10.5", + "target": "itertools" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.0" + }, + "license": "MIT/Apache-2.0" + }, + "crossbeam-channel 0.5.8": { + "name": "crossbeam-channel", + "version": "0.5.8", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/crossbeam-channel/0.5.8/download", + "sha256": "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crossbeam_channel", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crossbeam_channel", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "crossbeam-utils", + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "crossbeam-utils 0.8.15", + "target": "crossbeam_utils" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.8" + }, + "license": "MIT OR Apache-2.0" + }, + "crossbeam-deque 0.8.3": { + "name": "crossbeam-deque", + "version": "0.8.3", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/crossbeam-deque/0.8.3/download", + "sha256": "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crossbeam_deque", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crossbeam_deque", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "crossbeam-epoch", + "crossbeam-utils", + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "crossbeam-epoch 0.9.14", + "target": "crossbeam_epoch" + }, + { + "id": "crossbeam-utils 0.8.15", + "target": "crossbeam_utils" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.3" + }, + "license": "MIT OR Apache-2.0" + }, + "crossbeam-epoch 0.9.14": { + "name": "crossbeam-epoch", + "version": "0.9.14", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/crossbeam-epoch/0.9.14/download", + "sha256": "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crossbeam_epoch", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crossbeam_epoch", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "crossbeam-epoch 0.9.14", + "target": "build_script_build" + }, + { + "id": "crossbeam-utils 0.8.15", + "target": "crossbeam_utils" + }, + { + "id": "memoffset 0.8.0", + "target": "memoffset" + }, + { + "id": "scopeguard 1.1.0", + "target": "scopeguard" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.9.14" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "autocfg 1.1.0", + "target": "autocfg" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, + "crossbeam-utils 0.8.15": { + "name": "crossbeam-utils", + "version": "0.8.15", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/crossbeam-utils/0.8.15/download", + "sha256": "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crossbeam_utils", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "crossbeam_utils", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + }, + { + "id": "crossbeam-utils 0.8.15", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.8.15" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "ctor 0.1.22": { "name": "ctor", "version": "0.1.22", @@ -2288,6 +3065,36 @@ }, "license": "Apache-2.0 / MIT" }, + "funty 2.0.0": { + "name": "funty", + "version": "2.0.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/funty/2.0.0/download", + "sha256": "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "funty", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "funty", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "2.0.0" + }, + "license": "MIT" + }, "futures 0.3.27": { "name": "futures", "version": "0.3.27", @@ -3099,6 +3906,36 @@ }, "license": "MIT" }, + "half 1.8.2": { + "name": "half", + "version": "1.8.2", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/half/1.8.2/download", + "sha256": "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + } + }, + "targets": [ + { + "Library": { + "crate_name": "half", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "half", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "1.8.2" + }, + "license": "MIT OR Apache-2.0" + }, "hashbrown 0.11.2": { "name": "hashbrown", "version": "0.11.2", @@ -3939,6 +4776,53 @@ }, "license": "MIT" }, + "itertools 0.10.5": { + "name": "itertools", + "version": "0.10.5", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/itertools/0.10.5/download", + "sha256": "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" + } + }, + "targets": [ + { + "Library": { + "crate_name": "itertools", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "itertools", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "use_alloc", + "use_std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "either 1.6.1", + "target": "either" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.10.5" + }, + "license": "MIT/Apache-2.0" + }, "itoa 1.0.1": { "name": "itoa", "version": "1.0.1", @@ -3969,6 +4853,45 @@ }, "license": "MIT OR Apache-2.0" }, + "js-sys 0.3.61": { + "name": "js-sys", + "version": "0.3.61", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/js-sys/0.3.61/download", + "sha256": "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" + } + }, + "targets": [ + { + "Library": { + "crate_name": "js_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "js_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "wasm-bindgen 0.2.84", + "target": "wasm_bindgen" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.3.61" + }, + "license": "MIT/Apache-2.0" + }, "lazy_static 1.4.0": { "name": "lazy_static", "version": "1.4.0", @@ -4705,6 +5628,74 @@ }, "license": "MIT" }, + "memoffset 0.8.0": { + "name": "memoffset", + "version": "0.8.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/memoffset/0.8.0/download", + "sha256": "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" + } + }, + "targets": [ + { + "Library": { + "crate_name": "memoffset", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "memoffset", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "memoffset 0.8.0", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.8.0" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "autocfg 1.1.0", + "target": "autocfg" + } + ], + "selects": {} + } + }, + "license": "MIT" + }, "mime 0.3.16": { "name": "mime", "version": "0.3.16", @@ -5191,6 +6182,75 @@ }, "license": "Apache-2.0 OR MIT" }, + "num-traits 0.2.15": { + "name": "num-traits", + "version": "0.2.15", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/num-traits/0.2.15/download", + "sha256": "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" + } + }, + "targets": [ + { + "Library": { + "crate_name": "num_traits", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "num_traits", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "num-traits 0.2.15", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "0.2.15" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "autocfg 1.1.0", + "target": "autocfg" + } + ], + "selects": {} + } + }, + "license": "MIT OR Apache-2.0" + }, "num_cpus 1.13.1": { "name": "num_cpus", "version": "1.13.1", @@ -5279,13 +6339,13 @@ }, "license": "MIT OR Apache-2.0" }, - "once_cell 1.13.0": { + "once_cell 1.17.1": { "name": "once_cell", - "version": "1.13.0", + "version": "1.17.1", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/once_cell/1.13.0/download", - "sha256": "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" + "url": "https://crates.io/api/v1/crates/once_cell/1.17.1/download", + "sha256": "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" } }, "targets": [ @@ -5313,8 +6373,8 @@ ], "selects": {} }, - "edition": "2018", - "version": "1.13.0" + "edition": "2021", + "version": "1.17.1" }, "license": "MIT OR Apache-2.0" }, @@ -5439,6 +6499,36 @@ }, "license": "MIT" }, + "oorandom 11.1.3": { + "name": "oorandom", + "version": "11.1.3", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/oorandom/11.1.3/download", + "sha256": "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + } + }, + "targets": [ + { + "Library": { + "crate_name": "oorandom", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "oorandom", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "11.1.3" + }, + "license": "MIT" + }, "os_str_bytes 6.5.0": { "name": "os_str_bytes", "version": "6.5.0", @@ -5979,6 +7069,142 @@ }, "license": "MIT" }, + "plotters 0.3.4": { + "name": "plotters", + "version": "0.3.4", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/plotters/0.3.4/download", + "sha256": "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" + } + }, + "targets": [ + { + "Library": { + "crate_name": "plotters", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "plotters", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "area_series", + "line_series", + "plotters-svg", + "svg_backend" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "num-traits 0.2.15", + "target": "num_traits" + }, + { + "id": "plotters-backend 0.3.4", + "target": "plotters_backend" + }, + { + "id": "plotters-svg 0.3.3", + "target": "plotters_svg" + } + ], + "selects": { + "cfg(all(target_arch = \"wasm32\", not(target_os = \"wasi\")))": [ + { + "id": "wasm-bindgen 0.2.84", + "target": "wasm_bindgen" + }, + { + "id": "web-sys 0.3.61", + "target": "web_sys" + } + ] + } + }, + "edition": "2018", + "version": "0.3.4" + }, + "license": "MIT" + }, + "plotters-backend 0.3.4": { + "name": "plotters-backend", + "version": "0.3.4", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/plotters-backend/0.3.4/download", + "sha256": "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" + } + }, + "targets": [ + { + "Library": { + "crate_name": "plotters_backend", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "plotters_backend", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.3.4" + }, + "license": "MIT" + }, + "plotters-svg 0.3.3": { + "name": "plotters-svg", + "version": "0.3.3", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/plotters-svg/0.3.3/download", + "sha256": "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "plotters_svg", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "plotters_svg", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "plotters-backend 0.3.4", + "target": "plotters_backend" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.3.3" + }, + "license": "MIT" + }, "ppv-lite86 0.2.16": { "name": "ppv-lite86", "version": "0.2.16", @@ -6491,7 +7717,7 @@ "deps": { "common": [ { - "id": "once_cell 1.13.0", + "id": "once_cell 1.17.1", "target": "once_cell" }, { @@ -6604,6 +7830,59 @@ }, "license": "MIT OR Apache-2.0" }, + "radium 0.7.0": { + "name": "radium", + "version": "0.7.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/radium/0.7.0/download", + "sha256": "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + } + }, + "targets": [ + { + "Library": { + "crate_name": "radium", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "radium", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "radium 0.7.0", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.7.0" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT" + }, "radix_trie 0.2.1": { "name": "radix_trie", "version": "0.2.1", @@ -6805,6 +8084,119 @@ }, "license": "MIT OR Apache-2.0" }, + "rayon 1.7.0": { + "name": "rayon", + "version": "1.7.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/rayon/1.7.0/download", + "sha256": "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "rayon", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rayon", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "either 1.6.1", + "target": "either" + }, + { + "id": "rayon-core 1.11.0", + "target": "rayon_core" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.7.0" + }, + "license": "MIT OR Apache-2.0" + }, + "rayon-core 1.11.0": { + "name": "rayon-core", + "version": "1.11.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/rayon-core/1.11.0/download", + "sha256": "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "rayon_core", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "rayon_core", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "crossbeam-channel 0.5.8", + "target": "crossbeam_channel" + }, + { + "id": "crossbeam-deque 0.8.3", + "target": "crossbeam_deque" + }, + { + "id": "crossbeam-utils 0.8.15", + "target": "crossbeam_utils" + }, + { + "id": "num_cpus 1.13.1", + "target": "num_cpus" + }, + { + "id": "rayon-core 1.11.0", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "1.11.0" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ], + "links": "rayon-core" + }, + "license": "MIT OR Apache-2.0" + }, "redox_syscall 0.2.13": { "name": "redox_syscall", "version": "0.2.13", @@ -8242,6 +9634,10 @@ "id": "anyhow 1.0.69", "target": "anyhow" }, + { + "id": "bitvec 1.0.1", + "target": "bitvec" + }, { "id": "clap 4.1.11", "target": "clap" @@ -8250,6 +9646,14 @@ "id": "insta 1.28.0", "target": "insta" }, + { + "id": "itertools 0.10.5", + "target": "itertools" + }, + { + "id": "once_cell 1.17.1", + "target": "once_cell" + }, { "id": "protobuf 3.2.0", "target": "protobuf" @@ -8262,6 +9666,14 @@ "id": "scip 0.1.1", "target": "scip" }, + { + "id": "serde 1.0.136", + "target": "serde" + }, + { + "id": "serde_json 1.0.79", + "target": "serde_json" + }, { "id": "tree-sitter 0.20.9", "target": "tree_sitter" @@ -8861,7 +10273,7 @@ "target": "libloading" }, { - "id": "once_cell 1.13.0", + "id": "once_cell 1.17.1", "target": "once_cell" }, { @@ -9475,7 +10887,7 @@ "selects": { "cfg(all(target_arch = \"wasm32\", target_vendor = \"unknown\", target_os = \"unknown\", not(cargo_web)))": [ { - "id": "wasm-bindgen 0.2.79", + "id": "wasm-bindgen 0.2.84", "target": "wasm_bindgen" } ] @@ -9917,7 +11329,7 @@ "target": "lazy_static" }, { - "id": "once_cell 1.13.0", + "id": "once_cell 1.17.1", "target": "once_cell" }, { @@ -9977,6 +11389,10 @@ ], "deps": { "common": [ + { + "id": "base64 0.13.0", + "target": "base64" + }, { "id": "clap 4.1.11", "target": "clap" @@ -9993,6 +11409,10 @@ "id": "futures-util 0.3.27", "target": "futures_util" }, + { + "id": "protobuf 3.2.0", + "target": "protobuf" + }, { "id": "rocket 0.5.0-rc.1", "target": "rocket" @@ -10020,11 +11440,50 @@ ], "selects": {} }, + "deps_dev": { + "common": [ + { + "id": "criterion 0.4.0", + "target": "criterion" + } + ], + "selects": {} + }, "edition": "2021", "version": "1.0.1" }, "license": "MIT" }, + "tap 1.0.1": { + "name": "tap", + "version": "1.0.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/tap/1.0.1/download", + "sha256": "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + } + }, + "targets": [ + { + "Library": { + "crate_name": "tap", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "tap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.0.1" + }, + "license": "MIT" + }, "tempfile 3.3.0": { "name": "tempfile", "version": "3.3.0", @@ -10132,6 +11591,36 @@ }, "license": "Unlicense OR MIT" }, + "textwrap 0.16.0": { + "name": "textwrap", + "version": "0.16.0", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/textwrap/0.16.0/download", + "sha256": "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "textwrap", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "textwrap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "0.16.0" + }, + "license": "MIT" + }, "thiserror 1.0.30": { "name": "thiserror", "version": "1.0.30", @@ -10246,7 +11735,7 @@ "deps": { "common": [ { - "id": "once_cell 1.13.0", + "id": "once_cell 1.17.1", "target": "once_cell" } ], @@ -10530,6 +12019,49 @@ }, "license": "MIT OR Apache-2.0" }, + "tinytemplate 1.2.1": { + "name": "tinytemplate", + "version": "1.2.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/tinytemplate/1.2.1/download", + "sha256": "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" + } + }, + "targets": [ + { + "Library": { + "crate_name": "tinytemplate", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "tinytemplate", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "serde 1.0.136", + "target": "serde" + }, + { + "id": "serde_json 1.0.79", + "target": "serde_json" + } + ], + "selects": {} + }, + "edition": "2015", + "version": "1.2.1" + }, + "license": "Apache-2.0 OR MIT" + }, "tokio 1.17.0": { "name": "tokio", "version": "1.17.0", @@ -10609,7 +12141,7 @@ "target": "num_cpus" }, { - "id": "once_cell 1.13.0", + "id": "once_cell 1.17.1", "target": "once_cell" }, { @@ -13205,13 +14737,13 @@ }, "license": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" }, - "wasm-bindgen 0.2.79": { + "wasm-bindgen 0.2.84": { "name": "wasm-bindgen", - "version": "0.2.79", + "version": "0.2.84", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/wasm-bindgen/0.2.79/download", - "sha256": "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" + "url": "https://crates.io/api/v1/crates/wasm-bindgen/0.2.84/download", + "sha256": "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" } }, "targets": [ @@ -13254,7 +14786,7 @@ "target": "cfg_if" }, { - "id": "wasm-bindgen 0.2.79", + "id": "wasm-bindgen 0.2.84", "target": "build_script_build" } ], @@ -13264,13 +14796,13 @@ "proc_macro_deps": { "common": [ { - "id": "wasm-bindgen-macro 0.2.79", + "id": "wasm-bindgen-macro 0.2.84", "target": "wasm_bindgen_macro" } ], "selects": {} }, - "version": "0.2.79" + "version": "0.2.84" }, "build_script_attrs": { "data_glob": [ @@ -13279,13 +14811,13 @@ }, "license": "MIT/Apache-2.0" }, - "wasm-bindgen-backend 0.2.79": { + "wasm-bindgen-backend 0.2.84": { "name": "wasm-bindgen-backend", - "version": "0.2.79", + "version": "0.2.84", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/wasm-bindgen-backend/0.2.79/download", - "sha256": "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" + "url": "https://crates.io/api/v1/crates/wasm-bindgen-backend/0.2.84/download", + "sha256": "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" } }, "targets": [ @@ -13316,14 +14848,14 @@ "id": "bumpalo 3.9.1", "target": "bumpalo" }, - { - "id": "lazy_static 1.4.0", - "target": "lazy_static" - }, { "id": "log 0.4.16", "target": "log" }, + { + "id": "once_cell 1.17.1", + "target": "once_cell" + }, { "id": "proc-macro2 1.0.52", "target": "proc_macro2" @@ -13337,24 +14869,24 @@ "target": "syn" }, { - "id": "wasm-bindgen-shared 0.2.79", + "id": "wasm-bindgen-shared 0.2.84", "target": "wasm_bindgen_shared" } ], "selects": {} }, "edition": "2018", - "version": "0.2.79" + "version": "0.2.84" }, "license": "MIT/Apache-2.0" }, - "wasm-bindgen-macro 0.2.79": { + "wasm-bindgen-macro 0.2.84": { "name": "wasm-bindgen-macro", - "version": "0.2.79", + "version": "0.2.84", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/wasm-bindgen-macro/0.2.79/download", - "sha256": "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" + "url": "https://crates.io/api/v1/crates/wasm-bindgen-macro/0.2.84/download", + "sha256": "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" } }, "targets": [ @@ -13386,24 +14918,24 @@ "target": "quote" }, { - "id": "wasm-bindgen-macro-support 0.2.79", + "id": "wasm-bindgen-macro-support 0.2.84", "target": "wasm_bindgen_macro_support" } ], "selects": {} }, "edition": "2018", - "version": "0.2.79" + "version": "0.2.84" }, "license": "MIT/Apache-2.0" }, - "wasm-bindgen-macro-support 0.2.79": { + "wasm-bindgen-macro-support 0.2.84": { "name": "wasm-bindgen-macro-support", - "version": "0.2.79", + "version": "0.2.84", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/wasm-bindgen-macro-support/0.2.79/download", - "sha256": "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" + "url": "https://crates.io/api/v1/crates/wasm-bindgen-macro-support/0.2.84/download", + "sha256": "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" } }, "targets": [ @@ -13443,28 +14975,28 @@ "target": "syn" }, { - "id": "wasm-bindgen-backend 0.2.79", + "id": "wasm-bindgen-backend 0.2.84", "target": "wasm_bindgen_backend" }, { - "id": "wasm-bindgen-shared 0.2.79", + "id": "wasm-bindgen-shared 0.2.84", "target": "wasm_bindgen_shared" } ], "selects": {} }, "edition": "2018", - "version": "0.2.79" + "version": "0.2.84" }, "license": "MIT/Apache-2.0" }, - "wasm-bindgen-shared 0.2.79": { + "wasm-bindgen-shared 0.2.84": { "name": "wasm-bindgen-shared", - "version": "0.2.79", + "version": "0.2.84", "repository": { "Http": { - "url": "https://crates.io/api/v1/crates/wasm-bindgen-shared/0.2.79/download", - "sha256": "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + "url": "https://crates.io/api/v1/crates/wasm-bindgen-shared/0.2.84/download", + "sha256": "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" } }, "targets": [ @@ -13495,14 +15027,14 @@ "deps": { "common": [ { - "id": "wasm-bindgen-shared 0.2.79", + "id": "wasm-bindgen-shared 0.2.84", "target": "build_script_build" } ], "selects": {} }, "edition": "2018", - "version": "0.2.79" + "version": "0.2.84" }, "build_script_attrs": { "data_glob": [ @@ -13512,6 +15044,64 @@ }, "license": "MIT/Apache-2.0" }, + "web-sys 0.3.61": { + "name": "web-sys", + "version": "0.3.61", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/web-sys/0.3.61/download", + "sha256": "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" + } + }, + "targets": [ + { + "Library": { + "crate_name": "web_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "web_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "CanvasRenderingContext2d", + "Document", + "DomRect", + "DomRectReadOnly", + "Element", + "EventTarget", + "HtmlCanvasElement", + "HtmlElement", + "Node", + "Window" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "js-sys 0.3.61", + "target": "js_sys" + }, + { + "id": "wasm-bindgen 0.2.84", + "target": "wasm_bindgen" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.3.61" + }, + "license": "MIT/Apache-2.0" + }, "winapi 0.3.9": { "name": "winapi", "version": "0.3.9", @@ -14798,6 +16388,45 @@ }, "license": "MIT OR Apache-2.0" }, + "wyz 0.5.1": { + "name": "wyz", + "version": "0.5.1", + "repository": { + "Http": { + "url": "https://crates.io/api/v1/crates/wyz/0.5.1/download", + "sha256": "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" + } + }, + "targets": [ + { + "Library": { + "crate_name": "wyz", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "wyz", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "tap 1.0.1", + "target": "tap" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.5.1" + }, + "license": "MIT" + }, "xml-rs 0.8.4": { "name": "xml-rs", "version": "0.8.4", @@ -14936,6 +16565,9 @@ "i686-unknown-linux-gnu", "x86_64-unknown-linux-gnu" ], + "cfg(all(target_arch = \"wasm32\", not(target_os = \"wasi\")))": [ + "wasm32-unknown-unknown" + ], "cfg(all(target_arch = \"wasm32\", target_vendor = \"unknown\", target_os = \"unknown\", not(cargo_web)))": [ "wasm32-unknown-unknown" ], diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index 68a1b03c412..a784a873851 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -17,6 +17,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "ansi_term" version = "0.12.1" @@ -129,6 +135,18 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "bumpalo" version = "3.9.1" @@ -141,6 +159,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.73" @@ -153,6 +177,45 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" + +[[package]] +name = "ciborium-ll" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "bitflags 1.3.2", + "clap_lex 0.2.4", + "indexmap", + "textwrap", +] + [[package]] name = "clap" version = "4.1.11" @@ -161,7 +224,7 @@ checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" dependencies = [ "bitflags 2.0.2", "clap_derive", - "clap_lex", + "clap_lex 0.3.3", "is-terminal", "once_cell", "strsim", @@ -181,6 +244,15 @@ dependencies = [ "syn", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "0.3.3" @@ -239,6 +311,85 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap 3.2.23", + "criterion-plot", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset 0.8.0", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + [[package]] name = "ctor" version = "0.1.22" @@ -425,6 +576,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.27" @@ -563,6 +720,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" @@ -716,12 +879,30 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -829,6 +1010,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -907,7 +1097,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] @@ -919,6 +1109,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -940,9 +1139,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.13.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "onig" @@ -966,6 +1165,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -1073,6 +1278,34 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "plotters" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" + +[[package]] +name = "plotters-svg" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -1172,6 +1405,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "radix_trie" version = "0.2.1" @@ -1212,6 +1451,28 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.13" @@ -1485,14 +1746,19 @@ name = "scip-syntax" version = "0.1.0" dependencies = [ "anyhow", - "clap", + "bitvec", + "clap 4.1.11", "insta", + "itertools", + "once_cell", "protobuf", "rustc-hash", "scip", "scip-macros", "scip-treesitter", "scip-treesitter-languages", + "serde", + "serde_json", "tree-sitter", "walkdir", ] @@ -1812,13 +2078,17 @@ dependencies = [ name = "syntect_server" version = "1.0.1" dependencies = [ - "clap", + "base64", + "clap 4.1.11", + "criterion", "futures", "futures-task", "futures-util", + "protobuf", "rocket", "rustyline", "scip", + "scip-syntax", "scip-treesitter", "scip-treesitter-languages", "serde", @@ -1827,6 +2097,12 @@ dependencies = [ "syntect", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.3.0" @@ -1850,6 +2126,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.30" @@ -1928,6 +2210,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.17.0" @@ -2378,9 +2670,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2388,13 +2680,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -2403,9 +2695,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2413,9 +2705,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -2426,9 +2718,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "winapi" @@ -2585,6 +2887,15 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xml-rs" version = "0.8.4" diff --git a/docker-images/syntax-highlighter/Cargo.toml b/docker-images/syntax-highlighter/Cargo.toml index 40444d4fa50..58b034b3257 100644 --- a/docker-images/syntax-highlighter/Cargo.toml +++ b/docker-images/syntax-highlighter/Cargo.toml @@ -15,12 +15,15 @@ scip.workspace = true serde.workspace = true serde_json.workspace = true syntect.workspace = true +protobuf.workspace = true rustyline = "9.1.2" +base64 = "0.13.0" sg-syntax = { path = "./crates/sg-syntax" } scip-treesitter = { path = "./crates/scip-treesitter" } scip-treesitter-languages = { path = "./crates/scip-treesitter-languages" } +scip-syntax = { path = "./crates/scip-syntax" } # March 20, 2023 - Pinned explicitly with features that match the features # required by rocket. Once bazel rules correctly roll up all the features, # we can remove this, but until then, this works just fine for building @@ -38,12 +41,12 @@ members = [ "crates/scip-syntax", "crates/scip-treesitter", "crates/scip-treesitter-languages", - # "crates/ctags", ] [workspace.dependencies] anyhow = "1" clap = { version = "4.1.11", features = [ "derive" ] } +itertools = "0.10.5" rocket = { version = "0.5.0-rc.1", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -57,7 +60,11 @@ scip = { git = "https://github.com/sourcegraph/scip", rev="3856df76147ca4b86df78 protobuf = "3" + [profile.release] # Enabled debug symbols in release build, so if we have a crash # we can inspect the coredump. debug = true + +[dev-dependencies] +criterion = { version = "0.4", features = [ "html_reports" ] } diff --git a/docker-images/syntax-highlighter/crates/ctags/Cargo.toml b/docker-images/syntax-highlighter/crates/ctags/Cargo.toml deleted file mode 100644 index f4af8cc6684..00000000000 --- a/docker-images/syntax-highlighter/crates/ctags/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "ctags" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -scip.workspace = true -anyhow.workspace = true diff --git a/docker-images/syntax-highlighter/crates/ctags/src/lib.rs b/docker-images/syntax-highlighter/crates/ctags/src/lib.rs deleted file mode 100644 index 6070c99dcd6..00000000000 --- a/docker-images/syntax-highlighter/crates/ctags/src/lib.rs +++ /dev/null @@ -1,92 +0,0 @@ -// use anyhow::Result; - -// Some existing symbol structs -// -// zoekt -// type Symbol struct { -// Sym string -// Kind string -// Parent string -// ParentKind string -// } -// -// go-ctags -// type Entry struct { -// Name string -// Path string -// Line int -// Kind string -// Language string -// Parent string -// ParentKind string -// Pattern string -// Signature string -// -// FileLimited bool -// } - -// LSP Symbol Kinds, could be fine for us to use now -// export namespace SymbolKind { -// export const File = 1; -// export const Module = 2; -// export const Namespace = 3; -// export const Package = 4; -// export const Class = 5; -// export const Method = 6; -// export const Property = 7; -// export const Field = 8; -// export const Constructor = 9; -// export const Enum = 10; -// export const Interface = 11; -// export const Function = 12; -// export const Variable = 13; -// export const Constant = 14; -// export const String = 15; -// export const Number = 16; -// export const Boolean = 17; -// export const Array = 18; -// export const Object = 19; -// export const Key = 20; -// export const Null = 21; -// export const EnumMember = 22; -// export const Struct = 23; -// export const Event = 24; -// export const Operator = 25; -// export const TypeParameter = 26; -// } - -use scip::types::{Descriptor, Document}; - -#[derive(Debug)] -pub enum TagKind { - Function, - Class, -} - -impl std::str::FromStr for TagKind { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "definition.function" => Ok(Self::Function), - "definition.type" => Ok(Self::Function), - _ => anyhow::bail!("unknown tag kind: {}", s), - } - } -} - -#[derive(Debug)] -pub struct TagEntry { - pub descriptors: Vec, - pub kind: TagKind, - pub parent: Option>, - - pub line: usize, - // pub column: usize, -} - -impl TagEntry { - pub fn from_document(document: Document) -> Vec { - todo!("{:?}", document) - } -} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml index 236e2040054..ffb7a6caa2a 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml @@ -8,9 +8,12 @@ edition = "2021" [dependencies] anyhow.workspace = true clap.workspace = true +itertools.workspace = true protobuf.workspace = true scip.workspace = true tree-sitter.workspace = true +serde.workspace = true +serde_json.workspace = true scip-macros = { path = "../scip-macros" } scip-treesitter = { path = "../scip-treesitter" } @@ -19,3 +22,5 @@ scip-treesitter-languages = { path = "../scip-treesitter-languages" } rustc-hash = "1.1.0" walkdir = "2" insta = "*" +once_cell = "1.17.1" +bitvec = "1.0.1" diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/c/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/c/scip-tags.scm new file mode 100644 index 00000000000..d6a64365b12 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/c/scip-tags.scm @@ -0,0 +1,11 @@ +; Make use of @local + +(translation_unit (declaration (init_declarator declarator: (_) @descriptor.term))) + +(enum_specifier name: (_) @descriptor.type body: (enumerator_list (enumerator name: (_) @descriptor.term)) @descriptor.scope) + +(field_declaration declarator: [ + (pointer_declarator (field_identifier) @descriptor.term) + (field_identifier) @descriptor.term +]) +(function_definition (function_declarator declarator: (_) @descriptor.method)) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/c_sharp/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/c_sharp/scip-tags.scm new file mode 100644 index 00000000000..5449ed3485d --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/c_sharp/scip-tags.scm @@ -0,0 +1,11 @@ +(class_declaration name: (_) @descriptor.type) @scope +(interface_declaration name: (_) @descriptor.type) @scope +(enum_declaration name: (_) @descriptor.type) @scope +(namespace_declaration name: (_) @descriptor.type) @scope + +(method_declaration name: (_) @descriptor.method) +(constructor_declaration name: (_) @descriptor.method) + +(field_declaration (variable_declaration (variable_declarator (identifier) @descriptor.term))) +(property_declaration (identifier) @descriptor.term) +(enum_member_declaration name: (_) @descriptor.term) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/cpp/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/cpp/scip-tags.scm new file mode 100644 index 00000000000..7a6f9079d1e --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/cpp/scip-tags.scm @@ -0,0 +1,10 @@ +; Make use of @local + +(translation_unit (declaration (init_declarator declarator: (_) @descriptor.term))) + +(namespace_definition name: (_) @descriptor.type body: (_) @descriptor.scope) +(class_specifier name: (_) @descriptor.type body: (_) @descriptor.scope) +(enum_specifier name: (_) @descriptor.type body: (enumerator_list (enumerator name: (_) @descriptor.term)) @descriptor.scope) + +(field_declaration declarator: (_) @descriptor.term) +(function_definition (function_declarator declarator: (_) @descriptor.method)) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/go/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/go/scip-tags.scm index 348aa53bcf6..989b3d25111 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/queries/go/scip-tags.scm +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/go/scip-tags.scm @@ -1,4 +1,4 @@ -(source_file (package_clause (package_identifier) @descriptor.namespace) @scope) +(source_file (package_clause (package_identifier) @descriptor.namespace)) @scope (function_declaration name: (identifier) @descriptor.method) @@ -15,4 +15,9 @@ (parameter_declaration type: (type_identifier) @descriptor.type)) name: (field_identifier) @descriptor.method) -(type_declaration (type_spec name: (type_identifier) @descriptor.type)) +(type_declaration (type_spec name: (type_identifier) @descriptor.type)) @scope + +(field_declaration_list (field_declaration name: (_) @descriptor.term)) + +(const_spec name: (_) @descriptor.term) +(import_spec name: (_) @descriptor.term) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/java/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/java/scip-tags.scm new file mode 100644 index 00000000000..069c8f3f113 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/java/scip-tags.scm @@ -0,0 +1,16 @@ +(program + (package_declaration + (scoped_identifier) + @descriptor.namespace + ) +) @scope + +(class_declaration name: (_) @descriptor.type) @scope +(interface_declaration name: (_) @descriptor.type) @scope +(enum_declaration name: (_) @descriptor.type) @scope + +(method_declaration name: (_) @descriptor.method) @local +(constructor_declaration name: (_) @descriptor.method) @local + +(field_declaration (variable_declarator name: (_) @descriptor.term)) +(enum_constant name: (_) @descriptor.term) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/javascript/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/javascript/scip-tags.scm new file mode 100644 index 00000000000..ac568c47f6a --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/javascript/scip-tags.scm @@ -0,0 +1,22 @@ +(namespace_import (identifier) @descriptor.term) +(named_imports + [ + (import_specifier alias: (_) @descriptor.term) + (import_specifier name: (_) @descriptor.term !alias) + ] +) + +(function_declaration (identifier) @descriptor.method body: (_) @local) +(lexical_declaration (variable_declarator name: (identifier) @descriptor.term)) +(variable_declaration (variable_declarator name: (identifier) @descriptor.term)) + +(class_declaration name: (_) @descriptor.type body: (_) @scope) +(class_declaration + (class_body + [ + (method_definition name: (_) @descriptor.method body: (_) @local) + ] + ) +) + +[(if_statement) (while_statement) (for_statement) (do_statement)] @local diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/kotlin/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/kotlin/scip-tags.scm new file mode 100644 index 00000000000..9d2d38da4c9 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/kotlin/scip-tags.scm @@ -0,0 +1,14 @@ +(source_file + (package_header + (identifier) + @descriptor.namespace + ) +) @scope + +(function_declaration (simple_identifier) @descriptor.method) @local +(anonymous_function (_ (type_identifier) @descriptor.type . (type_identifier) @descriptor.method)) @local +(class_declaration (type_identifier) @descriptor.type) +(object_declaration (type_identifier) @descriptor.type) +(class_parameter (simple_identifier) @descriptor.term) +(enum_entry (simple_identifier) @descriptor.term) +(property_declaration (variable_declaration (simple_identifier) @descriptor.term)) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/python/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/python/scip-tags.scm new file mode 100644 index 00000000000..96cd64aca7e --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/python/scip-tags.scm @@ -0,0 +1,8 @@ +(import_statement name: (_) @descriptor.term) +(import_from_statement name: (_) @descriptor.term) + +(class_definition name: (_) @descriptor.type body: (_) @scope) +(function_definition name: (_) @descriptor.method body: (_) @local) +(expression_statement (assignment left: (identifier) @descriptor.term)) + +[(if_statement) (while_statement) (for_statement) (with_statement)] @local diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/ruby/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/ruby/scip-tags.scm new file mode 100644 index 00000000000..fdc54b8d786 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/ruby/scip-tags.scm @@ -0,0 +1,4 @@ +(assignment left: [(identifier) (constant)] @descriptor.term) +(class name: (_) @descriptor.type) @scope +(method name: (_) @descriptor.method) @local +[(block) (unless)] @local diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/rust/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/rust/scip-tags.scm index 987566bf257..5fd77f6c4da 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/queries/rust/scip-tags.scm +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/rust/scip-tags.scm @@ -7,16 +7,15 @@ name: (_) @descriptor.type) @scope (impl_item - trait: (_) @descriptor.type + trait: (_)? @descriptor.type type: (_) @descriptor.type) @scope ;; TODO: @local to stop traversal (function_signature_item name: (identifier) @descriptor.method) -;; TODO: @local to stop traversal (function_item - name: (identifier) @descriptor.method) + name: (identifier) @descriptor.method body: (_) @local) (struct_item name: (type_identifier) @descriptor.type) @scope diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/scala/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/scala/scip-tags.scm new file mode 100644 index 00000000000..187a5c56dda --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/scala/scip-tags.scm @@ -0,0 +1,19 @@ +; TODO: Exclude functions in non-container blocks + +(compilation_unit + (package_clause + (package_identifier) + @descriptor.namespace + ) +) @scope + +(compilation_unit (val_definition (identifier) @descriptor.term)) +(compilation_unit (var_definition (identifier) @descriptor.term)) +(template_body (val_definition (identifier) @descriptor.term)) +(template_body (var_definition (identifier) @descriptor.term)) + +(function_definition (identifier) @descriptor.method) + +(class_definition (identifier) @descriptor.type (template_body) @scope) +(object_definition (identifier) @descriptor.type (template_body) @scope) +(trait_definition (identifier) @descriptor.type (template_body) @scope) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/typescript/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/typescript/scip-tags.scm new file mode 100644 index 00000000000..f9e178fe038 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/typescript/scip-tags.scm @@ -0,0 +1,42 @@ +(namespace_import (identifier) @descriptor.term) +(named_imports + [ + (import_specifier alias: (_) @descriptor.term) + (import_specifier name: (_) @descriptor.term !alias) + ] +) + +(function_declaration (identifier) @descriptor.method body: (_) @local) +(lexical_declaration (variable_declarator name: (identifier) @descriptor.term)) +(variable_declaration (variable_declarator name: (identifier) @descriptor.term)) + +(interface_declaration name: (_) @descriptor.type body: (_) @scope) +(interface_declaration + (object_type + [ + (method_signature (property_identifier) @descriptor.method) + (property_signature (property_identifier) @descriptor.term) + ] + ) +) + +(class_declaration name: (_) @descriptor.type body: (_) @scope) +(class_declaration + (class_body + [ + (public_field_definition name: (_) @descriptor.term) + (method_definition name: (_) @descriptor.method body: (_) @local) + ] + ) +) + +(enum_declaration name: (_) @descriptor.type body: (_) @scope) +(enum_declaration + (enum_body + (property_identifier) @descriptor.term + ) +) + +(module name: (string (string_fragment) @descriptor.namespace) body: (_) @scope) + +[(if_statement) (while_statement) (for_statement) (do_statement)] @local diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/queries/zig/scip-tags.scm b/docker-images/syntax-highlighter/crates/scip-syntax/queries/zig/scip-tags.scm new file mode 100644 index 00000000000..e140f747742 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/queries/zig/scip-tags.scm @@ -0,0 +1,17 @@ +(Decl + (VarDecl + variable_type_function: (_) + @descriptor.term + ( + (ErrorUnionExpr + (SuffixExpr + (ContainerDecl) + ) + ) + @scope + )? + ) +) + +(Decl (FnProto function: (_) @descriptor.method)) @local +(ContainerField field_member: (IDENTIFIER) @descriptor.term) diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs new file mode 100644 index 00000000000..2d22f143b18 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/ctags.rs @@ -0,0 +1,307 @@ +use std::{ + collections::HashMap, + io::{BufRead, BufReader, BufWriter, Read, Write}, + ops::Not, + path, +}; + +use anyhow::{Context, Result}; +use itertools::intersperse; +use scip::types::{descriptor::Suffix, Descriptor}; +use scip_treesitter_languages::parsers::BundledParser; +use serde::{Deserialize, Serialize}; + +use crate::{get_globals, globals::Scope}; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(tag = "command", rename_all = "kebab-case")] +pub enum Request { + GenerateTags { filename: String, size: usize }, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(tag = "_type", rename_all = "kebab-case")] +pub enum Reply<'a> { + Program { + name: String, + version: String, + }, + Completed { + command: String, + }, + Error { + message: String, + fatal: bool, + }, + Tag { + name: String, + path: &'a str, + language: &'a str, + /// Starts at 1 + line: usize, + kind: &'a str, + scope: Option<&'a str>, + // Can't find any uses of these. If someone reports a bug, we can support this + // scope_kind: Option, + // signature: Option, + }, +} + +impl<'a> Reply<'a> { + pub fn write(self, writer: &mut W) { + writer + .write_all(serde_json::to_string(&self).unwrap().as_bytes()) + .unwrap(); + writer.write_all("\n".as_bytes()).unwrap(); + } + + pub fn write_tag( + writer: &mut W, + scope: &Scope, + path: &'a str, + language: &'a str, + tag_scope: Option<&'a str>, + scope_deduplicator: &mut HashMap, + ) { + let descriptors = &scope.descriptors; + let names = descriptors.iter().map(|d| d.name.as_str()); + let name = intersperse(names, ".").collect::(); + + let mut dedup = match tag_scope { + Some(ts) => vec![ts], + None => vec![], + }; + dedup.push(&name); + let dedup = dedup.join("."); + if scope_deduplicator.contains_key(&dedup) { + return; + } + scope_deduplicator.insert(dedup, ()); + + let tag = Self::Tag { + name, + path, + language, + line: scope.scope_range.start_line as usize + 1, + kind: descriptors_to_kind(&scope.descriptors), + scope: tag_scope, + }; + + tag.write(writer); + } +} + +fn descriptors_to_kind(descriptors: &[Descriptor]) -> &'static str { + match descriptors + .last() + .unwrap_or_default() + .suffix + .enum_value_or_default() + { + Suffix::Namespace => "namespace", + Suffix::Package => "package", + Suffix::Method => "method", + Suffix::Type => "type", + _ => "variable", + } +} + +fn emit_tags_for_scope( + buf_writer: &mut BufWriter, + path: &str, + parent_scopes: Vec, + scope: &Scope, + language: &str, + scope_deduplicator: &mut HashMap, +) { + let curr_scopes = { + let mut curr_scopes = parent_scopes.clone(); + for desc in &scope.descriptors { + curr_scopes.push(desc.name.clone()); + } + curr_scopes + }; + + if !scope.descriptors.is_empty() { + let tag_scope = parent_scopes + .is_empty() + .not() + .then(|| parent_scopes.join(".")); + let tag_scope = tag_scope.as_deref(); + + Reply::write_tag( + &mut *buf_writer, + scope, + path, + language, + tag_scope, + scope_deduplicator, + ); + } + + for subscope in &scope.children { + emit_tags_for_scope( + buf_writer, + path, + curr_scopes.clone(), + subscope, + language, + scope_deduplicator, + ); + } + + for global in &scope.globals { + let mut scope_name = curr_scopes.clone(); + scope_name.extend( + global + .descriptors + .iter() + .take(global.descriptors.len() - 1) + .map(|d| d.name.clone()), + ); + + Reply::Tag { + name: global.descriptors.last().unwrap().name.clone(), + path, + language, + line: global.range.start_line as usize + 1, + kind: descriptors_to_kind(&global.descriptors), + scope: scope_name + .is_empty() + .not() + .then(|| scope_name.join(".")) + .as_deref(), + } + .write(buf_writer); + } +} + +pub fn generate_tags( + buf_writer: &mut BufWriter, + filename: String, + file_data: &[u8], +) -> Option<()> { + let path = path::Path::new(&filename); + let extension = path.extension()?.to_str()?; + let filepath = path.file_name()?.to_str()?; + + let parser = BundledParser::get_parser_from_extension(extension)?; + let (root_scope, _) = match get_globals(&parser, file_data)? { + Ok(vals) => vals, + Err(err) => { + // TODO: Not sure I want to keep this or not + #[cfg(debug_assertions)] + if true { + panic!("Could not parse file: {}", err); + } + + return None; + } + }; + + let mut scope_deduplicator = HashMap::new(); + emit_tags_for_scope( + buf_writer, + filepath, + vec![], + &root_scope, + // I don't believe the language name is actually used anywhere but we'll + // keep it to be compliant with the ctags spec + parser.get_language_name(), + &mut scope_deduplicator, + ); + Some(()) +} + +pub fn ctags_runner( + input: &mut BufReader, + output: &mut std::io::BufWriter, +) -> Result<()> { + Reply::Program { + name: "SCIP Ctags".to_string(), + version: "5.9.0".to_string(), + } + .write(output); + output.flush().unwrap(); + + loop { + let mut line = String::new(); + input.read_line(&mut line)?; + + if line.is_empty() { + break; + } + + let request = serde_json::from_str::(&line); + let request = match request { + Ok(request) => request, + Err(_) => { + eprintln!("Could not parse request: {}", line); + continue; + } + }; + + match request { + Request::GenerateTags { filename, size } => { + let mut file_data = vec![0; size]; + input + .read_exact(&mut file_data) + .expect("Could not fill file data exactly"); + + generate_tags(output, filename, &file_data); + } + } + + Reply::Completed { + command: "generate-tags".to_string(), + } + .write(output); + + output.flush().unwrap(); + } + + Ok(()) +} + +pub fn helper_execute_one_file(name: &str, contents: &str) -> Result { + let command = format!( + r#" +{{ "command":"generate-tags","filename":"{}","size":{} }} +{} +"#, + name, + contents.len(), + contents + ) + .trim() + .to_string(); + + let mut input = BufReader::new(command.as_bytes()); + let mut output = BufWriter::new(Vec::new()); + ctags_runner(&mut input, &mut output)?; + + String::from_utf8(output.get_ref().to_vec()).context("Could not parse output") +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_ctags_runner_basic() -> Result<()> { + let file = r#" +fn main() { + println!("Hello, world!"); +} + +fn something() -> bool { true } +fn other() -> bool { false } +"# + .trim(); + + let output = helper_execute_one_file("main.rs", file)?; + insta::assert_snapshot!(output); + + Ok(()) + } +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs new file mode 100644 index 00000000000..bcb8eb22691 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/globals.rs @@ -0,0 +1,305 @@ +use anyhow::Result; +use bitvec::prelude::*; +use protobuf::Enum; +use scip::types::{Descriptor, Occurrence}; +use scip_treesitter::types::PackedRange; + +use crate::languages::TagConfiguration; + +#[derive(Debug)] +pub struct Scope { + pub ident_range: PackedRange, + pub scope_range: PackedRange, + pub globals: Vec, + pub children: Vec, + pub descriptors: Vec, +} + +#[derive(Debug)] +pub struct Global { + pub range: PackedRange, + pub descriptors: Vec, +} + +impl Scope { + pub fn insert_scope(&mut self, scope: Scope) { + if let Some(child) = self + .children + .iter_mut() + .find(|child| child.scope_range.contains(&scope.scope_range)) + { + child.insert_scope(scope); + } else { + self.children.push(scope); + } + } + + pub fn insert_global(&mut self, global: Global) { + if let Some(child) = self + .children + .iter_mut() + .find(|child| child.scope_range.contains(&global.range)) + { + child.insert_global(global) + } else { + self.globals.push(global); + } + } + + pub fn into_occurrences( + &mut self, + hint: usize, + base_descriptors: Vec, + ) -> Vec { + let mut descriptor_stack = base_descriptors; + let mut occs = Vec::with_capacity(hint); + self.rec_into_occurrences(true, &mut occs, &mut descriptor_stack); + occs + } + + fn rec_into_occurrences( + &self, + is_root: bool, + occurrences: &mut Vec, + descriptor_stack: &mut Vec, + ) { + descriptor_stack.extend(self.descriptors.clone()); + + if !is_root { + occurrences.push(scip::types::Occurrence { + range: self.ident_range.to_vec(), + symbol: scip::symbol::format_symbol(scip::types::Symbol { + scheme: "scip-ctags".into(), + // TODO: Package? + package: None.into(), + descriptors: descriptor_stack.clone(), + ..Default::default() + }), + symbol_roles: scip::types::SymbolRole::Definition.value(), + // TODO: + // syntax_kind: todo!(), + ..Default::default() + }); + } + + for global in &self.globals { + let mut global_descriptors = descriptor_stack.clone(); + global_descriptors.extend(global.descriptors.clone()); + + let symbol = scip::symbol::format_symbol(scip::types::Symbol { + scheme: "scip-ctags".into(), + // TODO: Package? + package: None.into(), + descriptors: global_descriptors, + ..Default::default() + }); + + let symbol_roles = scip::types::SymbolRole::Definition.value(); + occurrences.push(scip::types::Occurrence { + range: global.range.to_vec(), + symbol, + symbol_roles, + // TODO: + // syntax_kind: todo!(), + ..Default::default() + }); + } + + self.children + .iter() + .for_each(|c| c.rec_into_occurrences(false, occurrences, descriptor_stack)); + + self.descriptors.iter().for_each(|_| { + descriptor_stack.pop(); + }); + } +} + +pub fn parse_tree<'a>( + config: &TagConfiguration, + tree: &'a tree_sitter::Tree, + source_bytes: &'a [u8], +) -> Result<(Scope, usize)> { + let mut cursor = tree_sitter::QueryCursor::new(); + + let root_node = tree.root_node(); + let capture_names = config.query.capture_names(); + + let mut scopes = vec![]; + let mut globals = vec![]; + + let mut local_ranges = BitVec::::repeat(false, source_bytes.len()); + + let matches = cursor.matches(&config.query, root_node, source_bytes); + + for m in matches { + // eprintln!("\n==== NEW MATCH ===="); + + let mut node = None; + let mut scope = None; + let mut local_range = None; + let mut descriptors = vec![]; + + for capture in m.captures { + let capture_name = capture_names + .get(capture.index as usize) + .expect("capture indexes should always work"); + + if capture_name.starts_with("descriptor") { + descriptors.push((capture_name, capture.node.utf8_text(source_bytes)?)); + node = Some(capture.node); + } + + if capture_name.starts_with("scope") { + assert!(scope.is_none(), "declare only one scope per match"); + scope = Some(capture); + } + + if capture_name.starts_with("local") { + local_range = Some(capture.node.byte_range()); + } + + // eprintln!( + // "{}: {}", + // capture_name, + // capture.node.utf8_text(source_bytes).unwrap() + // ); + } + + match node { + Some(node) => { + if local_ranges[node.start_byte()] { + continue; + } + + let descriptors = descriptors + .iter() + .map(|(capture, name)| { + crate::ts_scip::capture_name_to_descriptor(capture, name.to_string()) + }) + .collect(); + + // dbg!(node); + + match scope { + Some(scope_ident) => scopes.push(Scope { + ident_range: node.into(), + scope_range: scope_ident.node.into(), + globals: vec![], + children: vec![], + descriptors, + }), + None => globals.push(Global { + range: node.into(), + descriptors, + }), + } + } + None => { + if local_range.is_none() { + panic!("there must always be at least one descriptor (except for @local)"); + } + } + } + + if let Some(local_range) = local_range { + local_ranges.get_mut(local_range).unwrap().fill(true); + } + } + + let mut root = Scope { + ident_range: root_node.into(), + scope_range: root_node.into(), + globals: vec![], + children: vec![], + descriptors: vec![], + }; + + scopes.sort_by_key(|m| { + std::cmp::Reverse(( + m.scope_range.start_line, + m.scope_range.end_line, + m.scope_range.start_col, + )) + }); + + // Add all the scopes to our tree + while let Some(m) = scopes.pop() { + root.insert_scope(m); + } + + while let Some(m) = globals.pop() { + root.insert_global(m); + } + + Ok((root, globals.len())) +} + +#[cfg(test)] +mod test { + use scip::types::Document; + use scip_treesitter::snapshot::dump_document; + use tree_sitter::Parser; + + use super::*; + + fn parse_file_for_lang(config: &TagConfiguration, source_code: &str) -> Result { + let source_bytes = source_code.as_bytes(); + + let mut parser = Parser::new(); + parser.set_language(config.language).unwrap(); + let tree = parser.parse(source_bytes, None).unwrap(); + + let mut occ = parse_tree(config, &tree, source_bytes)?; + let mut doc = Document::new(); + doc.occurrences = occ.0.into_occurrences(occ.1, vec![]); + doc.symbols = doc + .occurrences + .iter() + .map(|o| scip::types::SymbolInformation { + symbol: o.symbol.clone(), + ..Default::default() + }) + .collect(); + + Ok(doc) + } + + #[test] + fn test_can_parse_rust_tree() -> Result<()> { + let config = crate::languages::rust(); + let source_code = include_str!("../testdata/scopes.rs"); + let doc = parse_file_for_lang(config, source_code)?; + + let dumped = dump_document(&doc, source_code)?; + insta::assert_snapshot!(dumped); + + Ok(()) + } + + #[test] + fn test_can_parse_go_tree() -> Result<()> { + let config = crate::languages::go(); + let source_code = include_str!("../testdata/example.go"); + let doc = parse_file_for_lang(config, source_code)?; + // dbg!(doc); + + let dumped = dump_document(&doc, source_code)?; + insta::assert_snapshot!(dumped); + + Ok(()) + } + + #[test] + fn test_can_parse_go_internal_tree() -> Result<()> { + let config = crate::languages::go(); + let source_code = include_str!("../testdata/internal_go.go"); + let doc = parse_file_for_lang(config, source_code)?; + // dbg!(doc); + + let dumped = dump_document(&doc, source_code)?; + insta::assert_snapshot!(dumped); + + Ok(()) + } +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs index 4e74ae001d5..73618f29887 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/languages.rs @@ -1,3 +1,4 @@ +use once_cell::sync::OnceCell; use scip_macros::include_scip_query; use scip_treesitter_languages::parsers::BundledParser; use tree_sitter::{Language, Parser, Query}; @@ -5,35 +6,227 @@ use tree_sitter::{Language, Parser, Query}; pub struct TagConfiguration { pub language: Language, pub query: Query, - pub parser: Parser, } -pub fn rust() -> TagConfiguration { - let language = BundledParser::Rust.get_language(); - let query = include_scip_query!("rust", "scip-tags"); +pub fn c() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); - let mut parser = Parser::new(); - parser.set_language(language).unwrap(); + INSTANCE.get_or_init(|| { + let language = BundledParser::C.get_language(); + let query = include_scip_query!("c", "scip-tags"); - TagConfiguration { - language, - parser, - query: Query::new(language, query).unwrap(), - } + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) } -pub fn go() -> TagConfiguration { - let language = BundledParser::Go.get_language(); - let query = include_scip_query!("go", "scip-tags"); +pub fn javascript() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); - let mut parser = Parser::new(); - parser.set_language(language).unwrap(); + INSTANCE.get_or_init(|| { + let language = BundledParser::Javascript.get_language(); + let query = include_scip_query!("javascript", "scip-tags"); - TagConfiguration { - language, - parser, - query: Query::new(language, query).unwrap(), - } + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn kotlin() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Kotlin.get_language(); + let query = include_scip_query!("kotlin", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn ruby() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Ruby.get_language(); + let query = include_scip_query!("ruby", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn python() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Python.get_language(); + let query = include_scip_query!("python", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn cpp() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Cpp.get_language(); + let query = include_scip_query!("cpp", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn typescript() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Typescript.get_language(); + let query = include_scip_query!("typescript", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn scala() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Scala.get_language(); + let query = include_scip_query!("scala", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn c_sharp() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::C_Sharp.get_language(); + let query = include_scip_query!("c_sharp", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn java() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Java.get_language(); + let query = include_scip_query!("java", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn rust() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Rust.get_language(); + let query = include_scip_query!("rust", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn go() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Go.get_language(); + let query = include_scip_query!("go", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) +} + +pub fn zig() -> &'static TagConfiguration { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + let language = BundledParser::Zig.get_language(); + let query = include_scip_query!("zig", "scip-tags"); + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + + TagConfiguration { + language, + query: Query::new(language, query).unwrap(), + } + }) } pub struct LocalConfiguration { @@ -70,6 +263,25 @@ fn perl_locals() -> LocalConfiguration { } } +pub fn get_tag_configuration(parser: &BundledParser) -> Option<&'static TagConfiguration> { + match parser { + BundledParser::C => Some(c()), + BundledParser::Javascript => Some(javascript()), + BundledParser::Kotlin => Some(kotlin()), + BundledParser::Ruby => Some(ruby()), + BundledParser::Python => Some(python()), + BundledParser::Cpp => Some(cpp()), + BundledParser::Typescript => Some(typescript()), + BundledParser::Scala => Some(scala()), + BundledParser::C_Sharp => Some(c_sharp()), + BundledParser::Java => Some(java()), + BundledParser::Rust => Some(rust()), + BundledParser::Go => Some(go()), + BundledParser::Zig => Some(zig()), + _ => None, + } +} + pub fn get_local_configuration(parser: BundledParser) -> Option { match parser { BundledParser::Go => Some(go_locals()), diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs index eca0f698073..f5051e65eaf 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/lib.rs @@ -1,14 +1,75 @@ use anyhow::Result; use scip::types::Occurrence; use scip_treesitter_languages::parsers::BundledParser; +use tree_sitter::Parser; +pub mod ctags; +pub mod globals; pub mod languages; pub mod locals; -pub mod matches; pub mod ts_scip; +pub fn get_globals( + parser: &BundledParser, + source_bytes: &[u8], +) -> Option> { + let config = languages::get_tag_configuration(parser)?; + let mut parser = Parser::new(); + parser.set_language(config.language).unwrap(); + let tree = parser.parse(source_bytes, None).unwrap(); + Some(globals::parse_tree(config, &tree, source_bytes)) +} + pub fn get_locals(parser: BundledParser, source_bytes: &[u8]) -> Option>> { let mut config = languages::get_local_configuration(parser)?; let tree = config.parser.parse(source_bytes, None).unwrap(); Some(locals::parse_tree(&mut config, &tree, source_bytes)) } + +#[macro_export] +macro_rules! generate_tags_and_snapshot { + ($a:literal) => {{ + let mut buffer = vec![0u8; 1024]; + let mut buf_writer = BufWriter::new(&mut buffer); + + generate_tags(&mut buf_writer, $a.to_string(), include_bytes!($a)); + insta::assert_snapshot!(String::from_utf8_lossy(buf_writer.buffer())); + }}; +} + +#[cfg(test)] +mod test { + use std::io::BufWriter; + + use crate::ctags::generate_tags; + + #[test] + fn test_generate_ctags_go_globals() { + generate_tags_and_snapshot!("../testdata/go-globals.go"); + } + + #[test] + fn test_generate_ctags_empty_scope() { + generate_tags_and_snapshot!("../testdata/ctags-empty-scope.rs"); + } + + #[test] + fn test_generate_ctags_zig_globals() { + generate_tags_and_snapshot!("../testdata/globals.zig"); + } + + #[test] + fn test_generate_ctags_python_globals() { + generate_tags_and_snapshot!("../testdata/globals.py"); + } + + #[test] + fn test_generate_ctags_java_globals() { + generate_tags_and_snapshot!("../testdata/globals.java"); + } + + #[test] + fn test_generate_ctags_typescript_globals() { + generate_tags_and_snapshot!("../testdata/globals.ts"); + } +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs index 3781f83e13c..56f964866a5 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/locals.rs @@ -5,27 +5,15 @@ use scip::{ symbol::format_symbol, types::{Occurrence, Symbol}, }; -use scip_treesitter::prelude::*; +use scip_treesitter::{prelude::*, types::PackedRange}; use tree_sitter::Node; use crate::languages::LocalConfiguration; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct ByteRange { - start: usize, - end: usize, -} - -impl ByteRange { - pub fn contains(&self, other: &Self) -> bool { - self.start <= other.start && self.end >= other.end - } -} - #[derive(Debug)] pub struct Scope<'a> { pub scope: Node<'a>, - pub range: ByteRange, + pub range: PackedRange, pub definitions: HashMap<&'a str, Definition<'a>>, pub references: HashMap<&'a str, Vec>>, pub children: Vec>, @@ -55,10 +43,7 @@ impl<'a> Scope<'a> { pub fn new(scope: Node<'a>) -> Self { Self { scope, - range: ByteRange { - start: scope.start_byte(), - end: scope.end_byte(), - }, + range: scope.into(), definitions: HashMap::default(), references: HashMap::default(), children: vec![], @@ -102,10 +87,10 @@ impl<'a> Scope<'a> { } } - match self - .children - .binary_search_by_key(&reference.range.start, |r| r.range.start) - { + match self.children.binary_search_by_key( + &(reference.range.start_line, reference.range.start_col), + |r| (r.range.start_line, r.range.start_col), + ) { Ok(_) => { // self.children[idx].insert_reference(reference); todo!("I'm not sure what to do yet, think more now"); @@ -155,7 +140,8 @@ impl<'a> Scope<'a> { self.children.extend(child.children); } - self.children.sort_by_key(|s| s.range.start); + self.children + .sort_by_key(|s| (s.range.start_line, s.range.end_line, s.range.start_col)); } pub fn into_occurrences(&mut self, hint: usize) -> Vec { @@ -169,7 +155,7 @@ impl<'a> Scope<'a> { // We could probably make this a runtime option, where `self` has a `sorted` value // that decides whether we need to or not. But on a huge file, this made no difference. let mut values = self.definitions.values().collect::>(); - values.sort_by_key(|d| d.range.start); + values.sort_by_key(|d| &d.range); for definition in values { *id += 1; @@ -270,7 +256,7 @@ pub struct Definition<'a> { pub group: &'a str, pub identifier: &'a str, pub node: Node<'a>, - pub range: ByteRange, + pub range: PackedRange, pub scope_modifier: ScopeModifier, } @@ -279,7 +265,7 @@ pub struct Reference<'a> { pub group: &'a str, pub identifier: &'a str, pub node: Node<'a>, - pub range: ByteRange, + pub range: PackedRange, } pub fn parse_tree<'a>( @@ -356,10 +342,7 @@ pub fn parse_tree<'a>( let scope_modifier = scope_modifier.unwrap_or_default(); definitions.push(Definition { - range: ByteRange { - start: node.start_byte(), - end: node.end_byte(), - }, + range: node.into(), group, identifier, node, @@ -372,10 +355,7 @@ pub fn parse_tree<'a>( }; references.push(Reference { - range: ByteRange { - start: node.start_byte(), - end: node.end_byte(), - }, + range: node.into(), group, identifier, node, @@ -395,8 +375,9 @@ pub fn parse_tree<'a>( // Sort smallest to largest, so we can pop off the end of the list for the largest, first scope scopes.sort_by_key(|m| { ( - std::cmp::Reverse(m.range.start), - m.range.end - m.range.start, + std::cmp::Reverse(m.range.start_line), + m.range.end_line - m.range.start_line, + m.range.end_col - m.range.start_col, ) }); diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/matches.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/matches.rs deleted file mode 100644 index d61424611b3..00000000000 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/matches.rs +++ /dev/null @@ -1,345 +0,0 @@ -use anyhow::Result; -use protobuf::Enum; -use scip::types::Descriptor; -use scip_treesitter::prelude::*; -use tree_sitter::Node; - -use crate::languages::TagConfiguration; - -#[derive(Debug)] -pub struct Root<'a> { - pub root: Node<'a>, - pub children: Vec>, -} - -// #[derive(Debug)] -// pub struct Namespace {} - -pub struct Scope<'a> { - pub definer: Node<'a>, - pub scope: Node<'a>, - pub descriptors: Vec, - pub children: Vec>, -} - -impl<'a> std::fmt::Debug for Scope<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let descriptors = dbg_format_descriptors(&self.descriptors); - - write!( - f, - "({}, {}, {:?}) -> {:?}", - self.scope.kind(), - self.scope.start_position(), - descriptors, - self.children - ) - } -} - -pub struct Global<'a> { - pub node: Node<'a>, - pub descriptors: Vec, -} - -impl<'a> std::fmt::Debug for Global<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let descriptors = dbg_format_descriptors(&self.descriptors); - - write!( - f, - "({}, {}, {:?})", - self.node.kind(), - self.node.start_position(), - descriptors - ) - } -} - -#[derive(Debug)] -// TODO: Root as it's own type? -pub enum Matched<'a> { - /// The root node of a file - Root(Root<'a>), - - /// Does not generate a definition, simply a place ot add new descriptors - /// TODO: Haven't done this one for real yet - // Namespace(Namespace), - - /// Generates a new definition, and is itself a place to add additional descriptors - Scope(Scope<'a>), - - /// Generates a new definition, but does not generate a new scope - Global(Global<'a>), -} - -impl<'a> ContainsNode for Matched<'a> { - fn contains_node(&self, node: &Node) -> bool { - self.node().contains_node(node) - } -} - -impl<'a> Matched<'a> { - pub fn node(&self) -> &Node<'a> { - match self { - Matched::Root(m) => &m.root, - Matched::Scope(m) => &m.scope, - Matched::Global(m) => &m.node, - } - } - - // pub fn children(&self) -> &Vec> { - // match self { - // Matched::Root(m) => &m.children, - // Matched::Scope(s) => &s.children, - // Matched::Global(_) => todo!(), - // } - // } - - pub fn insert(&mut self, m: Matched<'a>) { - match self { - Matched::Root(root) => { - if let Some(child) = root - .children - .iter_mut() - .find(|child| child.contains_node(m.node())) - { - child.insert(m); - } else { - root.children.push(m); - } - } - Matched::Scope(scope) => { - if let Some(child) = scope - .children - .iter_mut() - .find(|child| child.contains_node(m.node())) - { - child.insert(m); - } else { - scope.children.push(m); - } - } - Matched::Global(_) => unreachable!(), - } - } - - pub fn into_occurences(&self) -> Vec { - self.rec_into_occurrences(&[]) - } - - // TODO: Could we use a dequeue for this to pop on and off quickly? - // TODO: Could we use a way to format the symbol w/out all the preamble? - // Perhaps just a "format_descriptors" function in the lib, that I didn't expose beforehand - fn rec_into_occurrences(&self, descriptors: &[Descriptor]) -> Vec { - match self { - Matched::Root(root) => { - assert!(descriptors.is_empty(), "root should not have descriptors"); - root.children - .iter() - .flat_map(|c| c.rec_into_occurrences(descriptors)) - .collect() - } - Matched::Scope(scope) => { - let mut these_descriptors = descriptors.to_vec(); - these_descriptors.extend(scope.descriptors.iter().cloned()); - - let symbol = scip::symbol::format_symbol(scip::types::Symbol { - scheme: "scip-ctags".into(), - // TODO: Package? - package: None.into(), - descriptors: these_descriptors, - ..Default::default() - }); - - let symbol_roles = scip::types::SymbolRole::Definition.value(); - let mut children = vec![scip::types::Occurrence { - range: vec![ - scope.definer.start_position().row as i32, - scope.definer.start_position().column as i32, - scope.definer.end_position().column as i32, - ], - symbol, - symbol_roles, - // TODO: - // syntax_kind: todo!(), - ..Default::default() - }]; - - children.extend(scope.children.iter().flat_map(|c| { - let mut descriptors = descriptors.to_vec(); - descriptors.extend(scope.descriptors.iter().cloned()); - c.rec_into_occurrences(&descriptors) - })); - - children - } - Matched::Global(global) => { - let mut these_descriptors = descriptors.to_vec(); - these_descriptors.extend(global.descriptors.iter().cloned()); - - let symbol = scip::symbol::format_symbol(scip::types::Symbol { - scheme: "scip-ctags".into(), - // TODO: Package? - package: None.into(), - descriptors: these_descriptors, - ..Default::default() - }); - - let symbol_roles = scip::types::SymbolRole::Definition.value(); - vec![scip::types::Occurrence { - range: vec![ - global.node.start_position().row as i32, - global.node.start_position().column as i32, - global.node.end_position().column as i32, - ], - symbol, - symbol_roles, - // TODO: - // syntax_kind: todo!(), - ..Default::default() - }] - } - } - } -} - -pub fn parse_tree<'a>( - config: &mut TagConfiguration, - tree: &'a tree_sitter::Tree, - source_bytes: &'a [u8], -) -> Result> { - let mut cursor = tree_sitter::QueryCursor::new(); - - let root_node = tree.root_node(); - let capture_names = config.query.capture_names(); - - let mut matched = vec![]; - for m in cursor.matches(&config.query, root_node, source_bytes) { - println!("\n==== NEW MATCH ===="); - - let mut node = None; - let mut scope = None; - let mut descriptors = vec![]; - - for capture in m.captures { - let capture_name = capture_names - .get(capture.index as usize) - .expect("capture indexes should always work"); - - if capture_name.starts_with("descriptor") { - descriptors.push((capture_name, capture.node.utf8_text(source_bytes)?)); - node = Some(capture.node); - } - - if capture_name.starts_with("scope") { - assert!(scope.is_none(), "declare only one scope per match"); - scope = Some(capture); - } - - println!( - "{}: {}", - capture_name, - capture.node.utf8_text(source_bytes).unwrap() - ); - } - - let descriptors = descriptors - .into_iter() - .map(|(capture, name)| { - crate::ts_scip::capture_name_to_descriptor(capture, name.to_string()) - }) - .collect::>(); - - let node = node.expect("there must always be at least one descriptor"); - dbg!(node); - - matched.push(match scope { - Some(scope) => Matched::Scope(Scope { - definer: node, - scope: scope.node, - descriptors, - children: vec![], - }), - None => Matched::Global(Global { node, descriptors }), - }) - } - - dbg!(&matched); - - let mut root = Matched::Root(Root { - root: root_node, - children: vec![], - }); - - matched.sort_by_key(|m| { - let node = m.node(); - node.end_byte() - node.start_byte() - }); - - while let Some(m) = matched.pop() { - root.insert(m); - } - dbg!(&root); - - let tags = root.into_occurences(); - Ok(dbg!(tags)) -} - -fn dbg_format_descriptors(descriptors: &[Descriptor]) -> Vec { - descriptors - .iter() - .map(|d| format!("{} ({:?})", d.name, d.suffix)) - .collect::>() -} - -#[cfg(test)] -mod test { - use scip::types::Document; - use scip_treesitter::snapshot::dump_document; - - use super::*; - - fn parse_file_for_lang(config: &mut TagConfiguration, source_code: &str) -> Result { - let source_bytes = source_code.as_bytes(); - let tree = config.parser.parse(source_bytes, None).unwrap(); - - let occ = parse_tree(config, &tree, source_bytes)?; - let mut doc = Document::new(); - doc.occurrences = occ; - doc.symbols = doc - .occurrences - .iter() - .map(|o| scip::types::SymbolInformation { - symbol: o.symbol.clone(), - ..Default::default() - }) - .collect(); - - Ok(doc) - } - - #[test] - fn test_can_parse_rust_tree() -> Result<()> { - let mut config = crate::languages::rust(); - let source_code = include_str!("../testdata/scopes.rs"); - let doc = parse_file_for_lang(&mut config, source_code)?; - - let dumped = dump_document(&doc, source_code)?; - insta::assert_snapshot!(dumped); - - Ok(()) - } - - #[test] - fn test_can_parse_go_tree() -> Result<()> { - let mut config = crate::languages::go(); - let source_code = include_str!("../testdata/example.go"); - let doc = dbg!(parse_file_for_lang(&mut config, source_code)?); - - let dumped = dump_document(&doc, source_code)?; - insta::assert_snapshot!(dumped); - - Ok(()) - } -} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_functions.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_functions.snap deleted file mode 100644 index 303e2b37b0a..00000000000 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_functions.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: src/locals.rs -assertion_line: 469 -expression: dumped ---- - package example -// ^^^^^^^ definition local 1 - - import ( - f "fmt" -// ^ definition local 2 - "github.com/sourcegraph/" - ) - - func Something() { -// ^^^^^^^^^ definition local 3 - y := ", world" -// ^ definition local 5 - f.Println("hello", y) -// ^ reference local 2 -// ^ reference local 5 - } - - func Another() { -// ^^^^^^^ definition local 4 - Something() -// ^^^^^^^^^ reference local 3 - if true { - x := true -// ^ definition local 6 - } - if true { - x := true -// ^ definition local 7 - if true { - x := true -// ^ definition local 8 - } - } - if true { - x := true -// ^ definition local 9 - } - } - diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_go.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_go.snap deleted file mode 100644 index 72ed5edfb21..00000000000 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_go.snap +++ /dev/null @@ -1,31 +0,0 @@ ---- -source: src/locals.rs -assertion_line: 463 -expression: dumped ---- - package main -// ^^^^ definition local 1 - - func main() { -// ^^^^ reference local 1 - local := true -// ^^^^^ definition local 3 - something := func(local int) int { -// ^^^^^^^^^ definition local 4 -// ^^^^^ definition local 5 - return local -// ^^^^^ reference local 5 - } - - println(local, something) -// ^^^^^ reference local 3 -// ^^^^^^^^^ reference local 4 - } - - func Another(local int) int { -// ^^^^^^^ definition local 2 -// ^^^^^ definition local 6 - return local -// ^^^^^ reference local 6 - } - diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_nested_locals.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_nested_locals.snap deleted file mode 100644 index e36c9cd6771..00000000000 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__locals__test__can_do_nested_locals.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: src/locals.rs -assertion_line: 502 -expression: dumped ---- - package main -// ^^^^ definition local 1 - - func main() { -// ^^^^ reference local 1 - local := 5 -// ^^^^^ definition local 2 - something := func(unrelated int) int { -// ^^^^^^^^^ definition local 3 -// ^^^^^^^^^ definition local 4 - superNested := func(deeplyNested int) int { -// ^^^^^^^^^^^ definition local 5 -// ^^^^^^^^^^^^ definition local 7 - return local + unrelated + deeplyNested -// ^^^^^ reference local 2 -// ^^^^^^^^^ reference local 4 -// ^^^^^^^^^^^^ reference local 7 - } - - overwriteName := func(local int) int { -// ^^^^^^^^^^^^^ definition local 6 -// ^^^^^ definition local 8 - return local + unrelated -// ^^^^^ reference local 8 -// ^^^^^^^^^ reference local 4 - } - - return superNested(1) + overwriteName(1) -// ^^^^^^^^^^^ reference local 5 -// ^^^^^^^^^^^^^ reference local 6 - } - - println(local, something) -// ^^^^^ reference local 2 -// ^^^^^^^^^ reference local 3 - } - diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_go_tree.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_go_tree.snap deleted file mode 100644 index 44f0f96b74d..00000000000 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_go_tree.snap +++ /dev/null @@ -1,29 +0,0 @@ ---- -source: src/matches.rs -assertion_line: 341 -expression: dumped ---- - package example -// ^^^^^^^ definition scip-ctags example/ - - import ( - f "fmt" - ) - - func Something() { -// ^^^^^^^^^ definition scip-ctags Something(). - x := true - f.Println(x) - } - - func Another() float64 { return 5 / 3 } -// ^^^^^^^ definition scip-ctags Another(). - - type MyThing struct{} -// ^^^^^^^ definition scip-ctags MyThing# - - func (m *MyThing) DoSomething() {} -// ^^^^^^^^^^^ definition scip-ctags MyThing#DoSomething(). - func (m MyThing) DoSomethingElse() {} -// ^^^^^^^^^^^^^^^ definition scip-ctags MyThing#DoSomethingElse(). - diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_rust_tree.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_rust_tree.snap deleted file mode 100644 index 2c2efdf5ff8..00000000000 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_semantic__matches__test__can_parse_rust_tree.snap +++ /dev/null @@ -1,34 +0,0 @@ ---- -source: src/matches.rs -assertion_line: 334 -expression: dumped ---- - pub trait Tag { -// ^^^ definition scip-ctags Tag# - // This is a pretty big thing - // And some more things here - fn name(&self) -> &str; -// ^^^^ definition scip-ctags Tag#name(). - } - - mod namespace { -// ^^^^^^^^^ definition scip-ctags namespace/ - mod nested { -// ^^^^^^ definition scip-ctags namespace/nested/ - mod even_more_nested { -// ^^^^^^^^^^^^^^^^ definition scip-ctags namespace/nested/even_more_nested/ - pub struct CoolStruct {} -// ^^^^^^^^^^ definition scip-ctags namespace/nested/even_more_nested/CoolStruct# - - impl Tag for CoolStruct { -// ^^^^^^^^^^ definition scip-ctags namespace/nested/even_more_nested/Tag#CoolStruct# - fn name(&self) -> &str {} -// ^^^^ definition scip-ctags namespace/nested/even_more_nested/Tag#CoolStruct#name(). - } - } - } - } - - fn something() {} -// ^^^^^^^^^ definition scip-ctags something(). - diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__ctags__test__ctags_runner_basic.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__ctags__test__ctags_runner_basic.snap new file mode 100644 index 00000000000..76e6f2897c0 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__ctags__test__ctags_runner_basic.snap @@ -0,0 +1,10 @@ +--- +source: crates/scip-syntax/src/ctags.rs +expression: output +--- +{"_type":"program","name":"SCIP Ctags","version":"5.9.0"} +{"_type":"tag","name":"other","path":"main.rs","language":"rust","line":6,"kind":"method","scope":null} +{"_type":"tag","name":"something","path":"main.rs","language":"rust","line":5,"kind":"method","scope":null} +{"_type":"tag","name":"main","path":"main.rs","language":"rust","line":1,"kind":"method","scope":null} +{"_type":"completed","command":"generate-tags"} + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_internal_tree.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_internal_tree.snap new file mode 100644 index 00000000000..ae4acfc604a --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_internal_tree.snap @@ -0,0 +1,41 @@ +--- +source: crates/scip-syntax/src/globals.rs +expression: dumped +--- + package memo +// ^^^^ definition scip-ctags memo/ + + import "sync" + + // MemoizedConstructorWithArg wraps a function returning taking a + // single argument value and returning a value and an error, memoizing + // its result. Multiple calls to Init will result in the underlying + // constructor being called once. The arguments to the call will be the + // first call to occur. All callers will receive the same return values. + type MemoizedConstructorWithArg[A, T any] struct { +// ^^^^^^^^^^^^^^^^^^^^^^^^^^ definition scip-ctags memo/MemoizedConstructorWithArg# + ctor func(A) (T, error) +// ^^^^ definition scip-ctags memo/MemoizedConstructorWithArg#ctor. + value T +// ^^^^^ definition scip-ctags memo/MemoizedConstructorWithArg#value. + err error +// ^^^ definition scip-ctags memo/MemoizedConstructorWithArg#err. + once sync.Once +// ^^^^ definition scip-ctags memo/MemoizedConstructorWithArg#once. + } + + // NewMemoizedConstructor memoizes the given constructor + func NewMemoizedConstructorWithArg[A, T any](ctor func(A) (T, error)) *MemoizedConstructorWithArg[A, T] { +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition scip-ctags memo/NewMemoizedConstructorWithArg(). + return &MemoizedConstructorWithArg[A, T]{ctor: ctor} + } + + // Init ensures that the given constructor has been called exactly + // once, then returns the constructor's result value and error. + func (m *MemoizedConstructorWithArg[A, T]) Init(arg A) (T, error) { +// ^^^^ definition scip-ctags memo/T#Init(). +// ^^^^ definition scip-ctags memo/A#Init(). + m.once.Do(func() { m.value, m.err = m.ctor(arg) }) + return m.value, m.err + } + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_tree.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_tree.snap new file mode 100644 index 00000000000..c678829f276 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_go_tree.snap @@ -0,0 +1,29 @@ +--- +source: crates/scip-syntax/src/globals.rs +expression: dumped +--- + package example +// ^^^^^^^ definition scip-ctags example/ + + import ( + f "fmt" +// ^ definition scip-ctags example/f. + ) + + func Something() { +// ^^^^^^^^^ definition scip-ctags example/Something(). + x := true + f.Println(x) + } + + func Another() float64 { return 5 / 3 } +// ^^^^^^^ definition scip-ctags example/Another(). + + type MyThing struct{} +// ^^^^^^^ definition scip-ctags example/MyThing# + + func (m *MyThing) DoSomething() {} +// ^^^^^^^^^^^ definition scip-ctags example/MyThing#DoSomething(). + func (m MyThing) DoSomethingElse() {} +// ^^^^^^^^^^^^^^^ definition scip-ctags example/MyThing#DoSomethingElse(). + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_rust_tree.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_rust_tree.snap similarity index 94% rename from docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_rust_tree.snap rename to docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_rust_tree.snap index 460ae95445f..90916ec3260 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_rust_tree.snap +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__globals__test__can_parse_rust_tree.snap @@ -1,6 +1,6 @@ --- -source: crates/scip-syntax/src/matches.rs -assertion_line: 329 +source: crates/scip-syntax/src/globals.rs +assertion_line: 318 expression: dumped --- pub trait Tag { diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_go_tree.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_go_tree.snap deleted file mode 100644 index 434123b06d8..00000000000 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__matches__test__can_parse_go_tree.snap +++ /dev/null @@ -1,29 +0,0 @@ ---- -source: crates/scip-syntax/src/matches.rs -assertion_line: 341 -expression: dumped ---- - package example -// ^^^^^^^ definition scip-ctags example/ - - import ( - f "fmt" - ) - - func Something() { -// ^^^^^^^^^ definition scip-ctags Something(). - x := true - f.Println(x) - } - - func Another() float64 { return 5 / 3 } -// ^^^^^^^ definition scip-ctags Another(). - - type MyThing struct{} -// ^^^^^^^ definition scip-ctags MyThing# - - func (m *MyThing) DoSomething() {} -// ^^^^^^^^^^^ definition scip-ctags MyThing#DoSomething(). - func (m MyThing) DoSomethingElse() {} -// ^^^^^^^^^^^^^^^ definition scip-ctags MyThing#DoSomethingElse(). - diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_empty_scope.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_empty_scope.snap new file mode 100644 index 00000000000..c5a09a1a269 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_empty_scope.snap @@ -0,0 +1,10 @@ +--- +source: crates/scip-syntax/src/lib.rs +expression: "String::from_utf8_lossy(buf_writer.buffer())" +--- +{"_type":"tag","name":"Arguments","path":"ctags-empty-scope.rs","language":"rust","line":10,"kind":"type","scope":null} +{"_type":"tag","name":"ParseTiming","path":"ctags-empty-scope.rs","language":"rust","line":15,"kind":"type","scope":null} +{"_type":"tag","name":"main","path":"ctags-empty-scope.rs","language":"rust","line":28,"kind":"method","scope":null} +{"_type":"tag","name":"measure_parsing","path":"ctags-empty-scope.rs","language":"rust","line":24,"kind":"method","scope":null} +{"_type":"tag","name":"parse_files","path":"ctags-empty-scope.rs","language":"rust","line":20,"kind":"method","scope":null} + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_go_globals.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_go_globals.snap new file mode 100644 index 00000000000..562c0c6a10f --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_go_globals.snap @@ -0,0 +1,12 @@ +--- +source: crates/scip-syntax/src/lib.rs +expression: "String::from_utf8_lossy(buf_writer.buffer())" +--- +{"_type":"tag","name":"multierror","path":"go-globals.go","language":"go","line":1,"kind":"namespace","scope":null} +{"_type":"tag","name":"Group","path":"go-globals.go","language":"go","line":7,"kind":"type","scope":"multierror"} +{"_type":"tag","name":"wg","path":"go-globals.go","language":"go","line":10,"kind":"variable","scope":"multierror.Group"} +{"_type":"tag","name":"err","path":"go-globals.go","language":"go","line":9,"kind":"variable","scope":"multierror.Group"} +{"_type":"tag","name":"mutex","path":"go-globals.go","language":"go","line":8,"kind":"variable","scope":"multierror.Group"} +{"_type":"tag","name":"Wait","path":"go-globals.go","language":"go","line":33,"kind":"method","scope":"multierror.Group"} +{"_type":"tag","name":"Go","path":"go-globals.go","language":"go","line":17,"kind":"method","scope":"multierror.Group"} + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_java_globals.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_java_globals.snap new file mode 100644 index 00000000000..5c7ec282dd5 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_java_globals.snap @@ -0,0 +1,31 @@ +--- +source: crates/scip-syntax/src/lib.rs +expression: "String::from_utf8_lossy(buf_writer.buffer())" +--- +{"_type":"tag","name":"globals","path":"globals.java","language":"java","line":1,"kind":"type","scope":null} +{"_type":"tag","name":"ClassInAClass","path":"globals.java","language":"java","line":18,"kind":"type","scope":"globals"} +{"_type":"tag","name":"Enum","path":"globals.java","language":"java","line":21,"kind":"type","scope":"globals.ClassInAClass"} +{"_type":"tag","name":"terms","path":"globals.java","language":"java","line":27,"kind":"variable","scope":"globals.ClassInAClass.Enum"} +{"_type":"tag","name":"as","path":"globals.java","language":"java","line":26,"kind":"variable","scope":"globals.ClassInAClass.Enum"} +{"_type":"tag","name":"recognized","path":"globals.java","language":"java","line":25,"kind":"variable","scope":"globals.ClassInAClass.Enum"} +{"_type":"tag","name":"be","path":"globals.java","language":"java","line":24,"kind":"variable","scope":"globals.ClassInAClass.Enum"} +{"_type":"tag","name":"should","path":"globals.java","language":"java","line":23,"kind":"variable","scope":"globals.ClassInAClass.Enum"} +{"_type":"tag","name":"these","path":"globals.java","language":"java","line":22,"kind":"variable","scope":"globals.ClassInAClass.Enum"} +{"_type":"tag","name":"Goated","path":"globals.java","language":"java","line":30,"kind":"type","scope":"globals.ClassInAClass"} +{"_type":"tag","name":"withTheSauce","path":"globals.java","language":"java","line":31,"kind":"method","scope":"globals.ClassInAClass.Goated"} +{"_type":"tag","name":"myCoolMethod","path":"globals.java","language":"java","line":34,"kind":"method","scope":"globals.ClassInAClass"} +{"_type":"tag","name":"classy","path":"globals.java","language":"java","line":19,"kind":"variable","scope":"globals.ClassInAClass"} +{"_type":"tag","name":"COOLEST_STRING","path":"globals.java","language":"java","line":16,"kind":"variable","scope":"globals"} +{"_type":"tag","name":"method6","path":"globals.java","language":"java","line":14,"kind":"method","scope":"globals"} +{"_type":"tag","name":"method5","path":"globals.java","language":"java","line":13,"kind":"method","scope":"globals"} +{"_type":"tag","name":"method4","path":"globals.java","language":"java","line":12,"kind":"method","scope":"globals"} +{"_type":"tag","name":"method3","path":"globals.java","language":"java","line":11,"kind":"method","scope":"globals"} +{"_type":"tag","name":"method2","path":"globals.java","language":"java","line":10,"kind":"method","scope":"globals"} +{"_type":"tag","name":"method1","path":"globals.java","language":"java","line":9,"kind":"method","scope":"globals"} +{"_type":"tag","name":"field6","path":"globals.java","language":"java","line":7,"kind":"variable","scope":"globals"} +{"_type":"tag","name":"field5","path":"globals.java","language":"java","line":6,"kind":"variable","scope":"globals"} +{"_type":"tag","name":"field4","path":"globals.java","language":"java","line":5,"kind":"variable","scope":"globals"} +{"_type":"tag","name":"field3","path":"globals.java","language":"java","line":4,"kind":"variable","scope":"globals"} +{"_type":"tag","name":"field2","path":"globals.java","language":"java","line":3,"kind":"variable","scope":"globals"} +{"_type":"tag","name":"field1","path":"globals.java","language":"java","line":2,"kind":"variable","scope":"globals"} + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_python_globals.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_python_globals.snap new file mode 100644 index 00000000000..ae44d387a35 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_python_globals.snap @@ -0,0 +1,10 @@ +--- +source: crates/scip-syntax/src/lib.rs +expression: "String::from_utf8_lossy(buf_writer.buffer())" +--- +{"_type":"tag","name":"Bruh","path":"globals.py","language":"python","line":6,"kind":"type","scope":null} +{"_type":"tag","name":"dab","path":"globals.py","language":"python","line":11,"kind":"method","scope":"Bruh"} +{"_type":"tag","name":"__init__","path":"globals.py","language":"python","line":8,"kind":"method","scope":"Bruh"} +{"_type":"tag","name":"a","path":"globals.py","language":"python","line":6,"kind":"variable","scope":"Bruh"} +{"_type":"tag","name":"bruh","path":"globals.py","language":"python","line":3,"kind":"variable","scope":null} + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_typescript_globals.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_typescript_globals.snap new file mode 100644 index 00000000000..39629a7d7ac --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_typescript_globals.snap @@ -0,0 +1,22 @@ +--- +source: crates/scip-syntax/src/lib.rs +expression: "String::from_utf8_lossy(buf_writer.buffer())" +--- +{"_type":"tag","name":"MyClass","path":"globals.ts","language":"typescript","line":1,"kind":"type","scope":null} +{"_type":"tag","name":"also_private_method","path":"globals.ts","language":"typescript","line":8,"kind":"method","scope":"MyClass"} +{"_type":"tag","name":"#private_method","path":"globals.ts","language":"typescript","line":7,"kind":"method","scope":"MyClass"} +{"_type":"tag","name":"public_method","path":"globals.ts","language":"typescript","line":6,"kind":"method","scope":"MyClass"} +{"_type":"tag","name":"also_private_field","path":"globals.ts","language":"typescript","line":4,"kind":"variable","scope":"MyClass"} +{"_type":"tag","name":"#private_field","path":"globals.ts","language":"typescript","line":3,"kind":"variable","scope":"MyClass"} +{"_type":"tag","name":"public_field","path":"globals.ts","language":"typescript","line":2,"kind":"variable","scope":"MyClass"} +{"_type":"tag","name":"MyInterface","path":"globals.ts","language":"typescript","line":11,"kind":"type","scope":null} +{"_type":"tag","name":"sayBruh","path":"globals.ts","language":"typescript","line":13,"kind":"method","scope":"MyInterface"} +{"_type":"tag","name":"bruh","path":"globals.ts","language":"typescript","line":12,"kind":"variable","scope":"MyInterface"} +{"_type":"tag","name":"MyEnum","path":"globals.ts","language":"typescript","line":16,"kind":"type","scope":null} +{"_type":"tag","name":"go","path":"globals.ts","language":"typescript","line":19,"kind":"variable","scope":"MyEnum"} +{"_type":"tag","name":"rust","path":"globals.ts","language":"typescript","line":18,"kind":"variable","scope":"MyEnum"} +{"_type":"tag","name":"zig","path":"globals.ts","language":"typescript","line":17,"kind":"variable","scope":"MyEnum"} +{"_type":"tag","name":"func","path":"globals.ts","language":"typescript","line":25,"kind":"method","scope":null} +{"_type":"tag","name":"global2","path":"globals.ts","language":"typescript","line":23,"kind":"variable","scope":null} +{"_type":"tag","name":"global1","path":"globals.ts","language":"typescript","line":22,"kind":"variable","scope":null} + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_zig_globals.snap b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_zig_globals.snap new file mode 100644 index 00000000000..da726e1be7b --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/snapshots/scip_syntax__test__generate_ctags_zig_globals.snap @@ -0,0 +1,27 @@ +--- +source: crates/scip-syntax/src/lib.rs +expression: "String::from_utf8_lossy(buf_writer.buffer())" +--- +{"_type":"tag","name":"Bruh","path":"globals.zig","language":"zig","line":1,"kind":"variable","scope":null} +{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":4,"kind":"method","scope":"Bruh"} +{"_type":"tag","name":"zig_is_cool","path":"globals.zig","language":"zig","line":2,"kind":"variable","scope":"Bruh"} +{"_type":"tag","name":"MyUnion","path":"globals.zig","language":"zig","line":10,"kind":"variable","scope":null} +{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":16,"kind":"method","scope":"MyUnion"} +{"_type":"tag","name":"b","path":"globals.zig","language":"zig","line":14,"kind":"variable","scope":"MyUnion"} +{"_type":"tag","name":"a","path":"globals.zig","language":"zig","line":13,"kind":"variable","scope":"MyUnion"} +{"_type":"tag","name":"decl","path":"globals.zig","language":"zig","line":11,"kind":"variable","scope":"MyUnion"} +{"_type":"tag","name":"MyEnum","path":"globals.zig","language":"zig","line":19,"kind":"variable","scope":null} +{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":25,"kind":"method","scope":"MyEnum"} +{"_type":"tag","name":"b","path":"globals.zig","language":"zig","line":23,"kind":"variable","scope":"MyEnum"} +{"_type":"tag","name":"a","path":"globals.zig","language":"zig","line":22,"kind":"variable","scope":"MyEnum"} +{"_type":"tag","name":"decl","path":"globals.zig","language":"zig","line":20,"kind":"variable","scope":"MyEnum"} +{"_type":"tag","name":"MyUnionEnum","path":"globals.zig","language":"zig","line":28,"kind":"variable","scope":null} +{"_type":"tag","name":"init","path":"globals.zig","language":"zig","line":34,"kind":"method","scope":"MyUnionEnum"} +{"_type":"tag","name":"b","path":"globals.zig","language":"zig","line":32,"kind":"variable","scope":"MyUnionEnum"} +{"_type":"tag","name":"a","path":"globals.zig","language":"zig","line":31,"kind":"variable","scope":"MyUnionEnum"} +{"_type":"tag","name":"decl","path":"globals.zig","language":"zig","line":29,"kind":"variable","scope":"MyUnionEnum"} +{"_type":"tag","name":"Ahh","path":"globals.zig","language":"zig","line":37,"kind":"variable","scope":null} +{"_type":"tag","name":"opaqueFn","path":"globals.zig","language":"zig","line":38,"kind":"method","scope":"Ahh"} +{"_type":"tag","name":"complex","path":"globals.zig","language":"zig","line":47,"kind":"method","scope":null} +{"_type":"tag","name":"bruh","path":"globals.zig","language":"zig","line":41,"kind":"method","scope":null} + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/ts_scip.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/ts_scip.rs index 025b02ea6d0..6e0032874c4 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/ts_scip.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/ts_scip.rs @@ -6,6 +6,7 @@ pub fn capture_name_to_descriptor(capture: &str, name: String) -> Descriptor { "descriptor.method" => Suffix::Method, "descriptor.namespace" => Suffix::Namespace, "descriptor.type" => Suffix::Type, + "descriptor.term" => Suffix::Term, // TODO: Should consider moving to result here. _ => Suffix::UnspecifiedSuffix, diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/ctags-empty-scope.rs b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/ctags-empty-scope.rs new file mode 100644 index 00000000000..7548f3915c1 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/ctags-empty-scope.rs @@ -0,0 +1,30 @@ +use std::{path::Path, time::Instant}; + +use clap::Parser; +use scip_syntax::locals::parse_tree; +use scip_treesitter_languages::parsers::BundledParser; +use walkdir::WalkDir; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Arguments { + /// Root directory to run local navigation over + root_dir: String, +} + +struct ParseTiming { + pub filepath: String, + pub duration: std::time::Duration, +} + +fn parse_files(dir: &Path) -> Vec { + // TODO +} + +fn measure_parsing() { + // TODO +} + +fn main() { + // TODO +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.java b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.java new file mode 100644 index 00000000000..f7d705afed3 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.java @@ -0,0 +1,39 @@ +public class globals { + private static int field1; + protected static int field2; + public static int field3; + private int field4; + protected int field5; + public int field6; + + private static void method1() {} + protected static void method2() {} + public static void method3() {} + private void method4() {} + protected void method5() {} + public void method6() {} + + public static final String COOLEST_STRING = "probably this one"; + + public class ClassInAClass { + boolean classy = true; + + public static enum Enum { + these, + should, + be, + recognized, + as, + terms + } + + public interface Goated { + boolean withTheSauce(); + } + + public void myCoolMethod() { + class WhatIsGoingOn {} + boolean iThinkThisIsAllowedButWeDontReallyCare = true; + } + } +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.py b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.py new file mode 100644 index 00000000000..a29d3e7d0b2 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.py @@ -0,0 +1,28 @@ +# TODO: Deal with duplicates (bruh = 10; bruh = 10;) being marked as definitions + +bruh = 10 + +class Bruh(object): + a: int + + def __init__(self) -> None: + pass + + def dab(): + print("yay!") + def more(): + print("a function in a function!!") + pass + more() + +if 1 == 1: + notHere = False + +while False: + notHereEither = False + +for i in range(0, 0): + definitelyNotInHere = False + +with 1: + what = "is this even allowed in Python anymore?" \ No newline at end of file diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.ts b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.ts new file mode 100644 index 00000000000..bd166a262b1 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.ts @@ -0,0 +1,33 @@ +class MyClass { + public_field: number + #private_field: number + private also_private_field: number + + public_method() {} + #private_method() {} + private also_private_method() {} +} + +interface MyInterface { + bruh: number, + sayBruh(): void, +} + +enum MyEnum { + zig, + rust, + go, +} + +var global1 = 0; +var global2; + +function func() { + var c; + function inAnotherFunc() { + var b; + function inAnother() { + var a; + } + } +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.zig b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.zig new file mode 100644 index 00000000000..a40639210bb --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/globals.zig @@ -0,0 +1,49 @@ +pub const Bruh = struct { + zig_is_cool: bool = true, + + pub fn init() Bruh { + var aaa = false; + return .{}; + } +}; + +const MyUnion = union { + const decl = 10; + + a: u8, + b: u40, + + pub fn init() void {}; +}; + +const MyEnum = enum { + const decl = 10; + + a, + b, + + pub fn init() void {}; +}; + +const MyUnionEnum = union(enum) { + const decl = 10; + + a: u8, + b: u40, + + pub fn init() void {}; +}; + +const Ahh = opaque { + pub fn opaqueFn() void {} +} + +fn bruh() void { + const ThisShouldntBeRegistered = struct { + fn bruh2() void {} + } +} + +fn complex(a: struct {bruh: bool}) struct {dab: u8} { + return .{.dab = if (a.bruh) 10 else 20}; +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/go-globals.go b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/go-globals.go new file mode 100644 index 00000000000..9c29efb7f87 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/go-globals.go @@ -0,0 +1,38 @@ +package multierror + +import "sync" + +// Group is a collection of goroutines which return errors that need to be +// coalesced. +type Group struct { + mutex sync.Mutex + err *Error + wg sync.WaitGroup +} + +// Go calls the given function in a new goroutine. +// +// If the function returns an error it is added to the group multierror which +// is returned by Wait. +func (g *Group) Go(f func() error) { + g.wg.Add(1) + + go func() { + defer g.wg.Done() + + if err := f(); err != nil { + g.mutex.Lock() + g.err = Append(g.err, err) + g.mutex.Unlock() + } + }() +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the multierror. +func (g *Group) Wait() *Error { + g.wg.Wait() + g.mutex.Lock() + defer g.mutex.Unlock() + return g.err +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/internal_go.go b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/internal_go.go new file mode 100644 index 00000000000..c62a580b782 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/internal_go.go @@ -0,0 +1,27 @@ +package memo + +import "sync" + +// MemoizedConstructorWithArg wraps a function returning taking a +// single argument value and returning a value and an error, memoizing +// its result. Multiple calls to Init will result in the underlying +// constructor being called once. The arguments to the call will be the +// first call to occur. All callers will receive the same return values. +type MemoizedConstructorWithArg[A, T any] struct { + ctor func(A) (T, error) + value T + err error + once sync.Once +} + +// NewMemoizedConstructor memoizes the given constructor +func NewMemoizedConstructorWithArg[A, T any](ctor func(A) (T, error)) *MemoizedConstructorWithArg[A, T] { + return &MemoizedConstructorWithArg[A, T]{ctor: ctor} +} + +// Init ensures that the given constructor has been called exactly +// once, then returns the constructor's result value and error. +func (m *MemoizedConstructorWithArg[A, T]) Init(arg A) (T, error) { + m.once.Do(func() { m.value, m.err = m.ctor(arg) }) + return m.value, m.err +} diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs b/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs index 61aa8a181c0..6ec95387bbc 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter-languages/src/parsers.rs @@ -78,4 +78,56 @@ impl BundledParser { _ => None, } } + + pub fn get_language_name(&self) -> &str { + match self { + BundledParser::C => "c", + BundledParser::Cpp => "cpp", + BundledParser::C_Sharp => "c_sharp", + BundledParser::Go => "go", + BundledParser::Java => "java", + BundledParser::Javascript => "javascript", + BundledParser::Jsonnet => "jsonnet", + BundledParser::Kotlin => "kotlin", + BundledParser::Nickel => "nickel", + BundledParser::Perl => "perl", + BundledParser::Pod => "pod", + BundledParser::Python => "python", + BundledParser::Ruby => "ruby", + BundledParser::Rust => "rust", + BundledParser::Scala => "scala", + BundledParser::Sql => "sql", + BundledParser::Typescript => "typescript", + BundledParser::Tsx => "tsx", + BundledParser::Xlsg => "xlsg", + BundledParser::Zig => "zig", + } + } + + // TODO(SuperAuguste): language detection library + pub fn get_parser_from_extension(name: &str) -> Option { + match name { + "c" => Some(BundledParser::C), + "cpp" => Some(BundledParser::Cpp), + "cs" => Some(BundledParser::C_Sharp), + "go" => Some(BundledParser::Go), + "java" => Some(BundledParser::Java), + "js" => Some(BundledParser::Javascript), + "jsonnet" => Some(BundledParser::Jsonnet), + "kt" => Some(BundledParser::Kotlin), + "ncl" => Some(BundledParser::Nickel), + "pl" => Some(BundledParser::Perl), + "pod" => Some(BundledParser::Pod), + "py" => Some(BundledParser::Python), + "rb" => Some(BundledParser::Ruby), + "rs" => Some(BundledParser::Rust), + "scala" => Some(BundledParser::Scala), + "sql" => Some(BundledParser::Sql), + "ts" => Some(BundledParser::Typescript), + "tsx" => Some(BundledParser::Tsx), + "xlsg" => Some(BundledParser::Xlsg), + "zig" => Some(BundledParser::Zig), + _ => None, + } + } } diff --git a/docker-images/syntax-highlighter/crates/scip-treesitter/src/types.rs b/docker-images/syntax-highlighter/crates/scip-treesitter/src/types.rs index 9e47efb18b8..dbaec91e69c 100644 --- a/docker-images/syntax-highlighter/crates/scip-treesitter/src/types.rs +++ b/docker-images/syntax-highlighter/crates/scip-treesitter/src/types.rs @@ -1,3 +1,5 @@ +use tree_sitter::Node; + #[derive(Debug, PartialEq, Eq, Default)] pub struct PackedRange { pub start_line: i32, @@ -29,6 +31,14 @@ impl PackedRange { } } + pub fn to_vec(&self) -> Vec { + if self.start_line == self.end_line { + vec![self.start_line, self.start_col, self.end_col] + } else { + vec![self.start_line, self.start_col, self.end_line, self.end_col] + } + } + /// Checks if the range is equal to the given vector. /// If the other vector is not a valid PackedRange then it returns false pub fn eq_vec(&self, v: &[i32]) -> bool { @@ -48,6 +58,13 @@ impl PackedRange { _ => false, } } + + pub fn contains(&self, other: &PackedRange) -> bool { + other.start_line >= self.start_line + && other.end_line <= self.end_line + && (other.start_line != self.start_line || other.start_col >= self.start_col) + && (other.end_line != self.end_line || other.end_col <= self.end_col) + } } impl PartialOrd for PackedRange { @@ -69,3 +86,64 @@ impl Ord for PackedRange { )) } } + +impl<'a> From> for PackedRange { + fn from(node: Node<'a>) -> Self { + let start = node.start_position(); + let end = node.end_position(); + Self { + start_line: start.row as i32, + start_col: start.column as i32, + end_line: end.row as i32, + end_col: end.column as i32, + } + } +} + +#[cfg(test)] +mod tests { + use super::PackedRange; + + #[test] + fn test_packed_range_contains() { + let outer = PackedRange { + start_line: 1, + start_col: 1, + end_line: 4, + end_col: 4, + }; + + let inner = PackedRange { + start_line: 2, + start_col: 2, + end_line: 3, + end_col: 3, + }; + + let overlapping = PackedRange { + start_line: 3, + start_col: 3, + end_line: 5, + end_col: 5, + }; + + let outside = PackedRange { + start_line: 5, + start_col: 5, + end_line: 6, + end_col: 6, + }; + + let same = PackedRange { + start_line: 1, + start_col: 1, + end_line: 4, + end_col: 4, + }; + + assert!(outer.contains(&inner)); + assert!(!outer.contains(&overlapping)); + assert!(!outer.contains(&outside)); + assert!(outer.contains(&same)); + } +} diff --git a/docker-images/syntax-highlighter/insta.yaml b/docker-images/syntax-highlighter/insta.yaml new file mode 100644 index 00000000000..ef228bf23b3 --- /dev/null +++ b/docker-images/syntax-highlighter/insta.yaml @@ -0,0 +1,2 @@ +review: + warn_undiscovered: false diff --git a/docker-images/syntax-highlighter/justfile b/docker-images/syntax-highlighter/justfile new file mode 100644 index 00000000000..820c13548f4 --- /dev/null +++ b/docker-images/syntax-highlighter/justfile @@ -0,0 +1,7 @@ +download-bench: + mkdir -p bench_data/ + test -f bench_data/event.rs || curl -o bench_data/event.rs https://raw.githubusercontent.com/alacritty/alacritty/ead65221ebe06ff5689e65b866d735d4365d0e9e/alacritty/src/event.rs + test -f bench_data/big.cpp || curl -o bench_data/big.cpp https://raw.githubusercontent.com/llvm/llvm-project/ff2e6199b23525b06947785368cc3e2e93eab381/llvm/lib/Target/X86/X86ISelLowering.cpp + +bench: download-bench + cargo bench diff --git a/docker-images/syntax-highlighter/src/bin/scip-ctags.rs b/docker-images/syntax-highlighter/src/bin/scip-ctags.rs new file mode 100644 index 00000000000..a49ee4a85e4 --- /dev/null +++ b/docker-images/syntax-highlighter/src/bin/scip-ctags.rs @@ -0,0 +1,12 @@ +use std::io::{BufReader, BufWriter}; + +use scip_syntax::ctags::ctags_runner; + +fn main() { + let mut stdin = BufReader::new(std::io::stdin()); + let mut stdout = BufWriter::new(std::io::stdout()); + + if let Err(err) = ctags_runner(&mut stdin, &mut stdout) { + eprintln!("Error while executing: {}", err); + } +} diff --git a/docker-images/syntax-highlighter/src/main.rs b/docker-images/syntax-highlighter/src/main.rs index dd4b5ee0928..a1b582adf4f 100644 --- a/docker-images/syntax-highlighter/src/main.rs +++ b/docker-images/syntax-highlighter/src/main.rs @@ -3,7 +3,14 @@ #[macro_use] extern crate rocket; +use std::path; + +use ::scip::types::Document; +use protobuf::Message; use rocket::serde::json::{json, Json, Value as JsonValue}; +use scip_syntax::get_globals; +use scip_treesitter_languages::parsers::BundledParser; +use serde::Deserialize; use sg_syntax::{ScipHighlightQuery, SourcegraphQuery}; #[post("/", format = "application/json", data = "")] @@ -38,6 +45,60 @@ fn scip(q: Json) -> JsonValue { } } +#[derive(Deserialize, Default, Debug)] +pub struct SymbolQuery { + filename: String, + content: String, +} + +pub fn jsonify_err(e: impl ToString) -> JsonValue { + json!({"error": e.to_string()}) +} + +#[post("/symbols", format = "application/json", data = "")] +fn symbols(q: Json) -> JsonValue { + let path = path::Path::new(&q.filename); + let extension = match match path.extension() { + Some(vals) => vals, + None => { + return json!({"error": "Extensionless file"}); + } + } + .to_str() + { + Some(vals) => vals, + None => { + return json!({"error": "Invalid codepoint"}); + } + }; + let parser = match BundledParser::get_parser_from_extension(extension) { + Some(parser) => parser, + None => return json!({"error": "Could not infer parser from extension"}), + }; + + let (mut scope, hint) = match match get_globals(&parser, q.content.as_bytes()) { + Some(globals) => globals, + None => return json!({"error": "Failed to get globals"}), + } { + Ok(vals) => vals, + Err(err) => { + return jsonify_err(err); + } + }; + + let mut document = Document::default(); + + document.occurrences = scope.into_occurrences(hint, vec![]); + + let encoded = match document.write_to_bytes() { + Ok(vals) => vals, + Err(err) => { + return jsonify_err(err); + } + }; + json!({"scip": base64::encode(encoded), "plaintext": false}) +} + #[get("/health")] fn health() -> &'static str { "OK" @@ -59,8 +120,8 @@ fn rocket() -> _ { Ok(v) if v == "true" => { println!("Sanity check passed, exiting without error"); std::process::exit(0) - }, - _ => {}, + } + _ => {} }; // load configurations on-startup instead of on-first-request. @@ -76,6 +137,6 @@ fn rocket() -> _ { }; rocket::build() - .mount("/", routes![syntect, lsif, scip, health]) + .mount("/", routes![syntect, lsif, scip, symbols, health]) .register("/", catchers![not_found]) } diff --git a/enterprise/cmd/symbols/build-bazel.sh b/enterprise/cmd/symbols/build-bazel.sh index 07bd0df42c3..32a8b4742d7 100755 --- a/enterprise/cmd/symbols/build-bazel.sh +++ b/enterprise/cmd/symbols/build-bazel.sh @@ -11,6 +11,8 @@ cleanup() { } trap cleanup EXIT +echo "--- :bazel: bazel build for targets //enterprise/cmd/symbols" + bazelrc=( --bazelrc=.bazelrc ) @@ -21,7 +23,6 @@ if [[ ${CI:-""} == "true" ]]; then ) fi -echo "--- bazel build" bazel "${bazelrc[@]}" \ build \ //enterprise/cmd/symbols \ @@ -30,18 +31,38 @@ bazel "${bazelrc[@]}" \ --config incompat-zig-linux-amd64 out=$( - bazel \ - "${bazelrc[@]}" \ - cquery \ - //enterprise/cmd/symbols \ + bazel "${bazelrc[@]}" \ + cquery //enterprise/cmd/symbols \ --stamp \ --workspace_status_command=./dev/bazel_stamp_vars.sh \ --config incompat-zig-linux-amd64 \ --output=files ) -cp "$out" "$OUTPUT" +cp -v "$out" "$OUTPUT" + +# we can't build scip-ctags with symbols since the platform args conflict +# NOTE: cmd/symbols/cargo-config.sh sets some specific config when running on arm64 +# since this bazel run typically runs on CI that config change isn't made +echo "--- :bazel: bazel build for target //docker-images/syntax-highlighter:scip-ctags" +bazel "${bazelrc[@]}" \ + build //docker-images/syntax-highlighter:scip-ctags \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh + +out=$( + bazel "${bazelrc[@]}" \ + cquery //docker-images/syntax-highlighter:scip-ctags \ + --stamp \ + --workspace_status_command=./dev/bazel_stamp_vars.sh \ + --output=files +) +cp -v "$out" "$OUTPUT" + cp cmd/symbols/ctags-install-alpine.sh "$OUTPUT" +echo ":docker: context directory contains the following:" +ls -lah "$OUTPUT" +echo "--- :docker: docker build for symbols" docker build -f cmd/symbols/Dockerfile.bazel -t "$IMAGE" "$OUTPUT" \ --progress=plain \ --build-arg COMMIT_SHA \ diff --git a/enterprise/cmd/symbols/shared/BUILD.bazel b/enterprise/cmd/symbols/shared/BUILD.bazel index a5cae3766f5..f2abdf44398 100644 --- a/enterprise/cmd/symbols/shared/BUILD.bazel +++ b/enterprise/cmd/symbols/shared/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//enterprise/internal/rockskip", "//internal/conf", "//internal/conf/conftypes", + "//internal/ctags_config", "//internal/database", "//internal/database/connections/live", "//internal/debugserver", diff --git a/enterprise/cmd/symbols/shared/setup.go b/enterprise/cmd/symbols/shared/setup.go index 24f4b25b0e5..8847ace3b58 100644 --- a/enterprise/cmd/symbols/shared/setup.go +++ b/enterprise/cmd/symbols/shared/setup.go @@ -17,6 +17,7 @@ import ( "github.com/sourcegraph/sourcegraph/enterprise/internal/rockskip" "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" + "github.com/sourcegraph/sourcegraph/internal/ctags_config" "github.com/sourcegraph/sourcegraph/internal/database" connections "github.com/sourcegraph/sourcegraph/internal/database/connections/live" "github.com/sourcegraph/sourcegraph/internal/env" @@ -127,14 +128,14 @@ func setupRockskip(observationCtx *observation.Context, config rockskipConfig, g codeintelDB := mustInitializeCodeIntelDB(observationCtx) createParser := func() (ctags.Parser, error) { - return symbolsParser.SpawnCtags(log.Scoped("parser", "ctags parser"), config.Ctags) + return symbolsParser.SpawnCtags(log.Scoped("parser", "ctags parser"), config.Ctags, ctags_config.UniversalCtags) } server, err := rockskip.NewService(codeintelDB, gitserverClient, repositoryFetcher, createParser, config.MaxConcurrentlyIndexing, config.MaxRepos, config.LogQueries, config.IndexRequestsQueueSize, config.SymbolsCacheSize, config.PathSymbolsCacheSize, config.SearchLastIndexedCommit) if err != nil { - return nil, nil, config.Ctags.Command, err + return nil, nil, config.Ctags.UniversalCommand, err } - return server.Search, server.HandleStatus, config.Ctags.Command, nil + return server.Search, server.HandleStatus, config.Ctags.UniversalCommand, nil } func mustInitializeCodeIntelDB(observationCtx *observation.Context) *sql.DB { diff --git a/go.mod b/go.mod index 33c996dc05c..4280aaf7814 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,7 @@ require ( github.com/getsentry/sentry-go v0.21.0 github.com/ghodss/yaml v1.0.0 github.com/gitchander/permutation v0.0.0-20210517125447-a5d73722e1b1 - github.com/go-enry/go-enry/v2 v2.8.3 + github.com/go-enry/go-enry/v2 v2.8.4 github.com/go-git/go-git/v5 v5.5.2 github.com/go-openapi/strfmt v0.21.3 github.com/gobwas/glob v0.2.3 @@ -533,7 +533,7 @@ require ( go.opentelemetry.io/collector/pdata v1.0.0-rc5 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20221208152030-732eee02a75a + golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 golang.org/x/mod v0.8.0 golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 diff --git a/go.sum b/go.sum index 7b255b54c57..b4396664d24 100644 --- a/go.sum +++ b/go.sum @@ -793,8 +793,8 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-enry/go-enry/v2 v2.8.3 h1:BwvNrN58JqBJhyyVdZSl5QD3xoxEEGYUrRyPh31FGhw= -github.com/go-enry/go-enry/v2 v2.8.3/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ= +github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs= +github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -2476,8 +2476,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw= -golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/internal/ctags_config/BUILD.bazel b/internal/ctags_config/BUILD.bazel new file mode 100644 index 00000000000..01f6aeaa726 --- /dev/null +++ b/internal/ctags_config/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "ctags_config", + srcs = ["ctags_config.go"], + importpath = "github.com/sourcegraph/sourcegraph/internal/ctags_config", + visibility = ["//:__subpackages__"], + deps = ["//lib/errors"], +) diff --git a/internal/ctags_config/ctags_config.go b/internal/ctags_config/ctags_config.go new file mode 100644 index 00000000000..bf0a6b5ea9f --- /dev/null +++ b/internal/ctags_config/ctags_config.go @@ -0,0 +1,42 @@ +package ctags_config + +import "github.com/sourcegraph/sourcegraph/lib/errors" + +type ParserType = uint8 + +const ( + UnknownCtags ParserType = iota + NoCtags + UniversalCtags + ScipCtags +) + +func ParserNameToParserType(name string) (ParserType, error) { + switch name { + case "off": + return NoCtags, nil + case "universal-ctags": + return UniversalCtags, nil + case "scip-ctags": + return ScipCtags, nil + default: + return UnknownCtags, errors.Errorf("unknown parser type: %s", name) + } +} + +type ParserConfiguration struct { + Default ParserType + Engine map[string]ParserType +} + +var SupportLanguages = map[string]struct{}{ + "Zig": {}, +} + +var BaseParserConfig = ParserConfiguration{ + Engine: map[string]ParserType{ + // TODO: put our other languages here + // TODO: also list the languages we support + "Zig": ScipCtags, + }, +} diff --git a/lib/codeintel/languages/BUILD.bazel b/lib/codeintel/languages/BUILD.bazel new file mode 100644 index 00000000000..09c5795b3bb --- /dev/null +++ b/lib/codeintel/languages/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "languages", + srcs = ["languages.go"], + importpath = "github.com/sourcegraph/sourcegraph/lib/codeintel/languages", + visibility = ["//visibility:public"], + deps = [ + "@com_github_go_enry_go_enry_v2//:go-enry", + "@org_golang_x_exp//slices", + ], +) diff --git a/lib/codeintel/languages/languages.go b/lib/codeintel/languages/languages.go new file mode 100644 index 00000000000..560ef0bc26a --- /dev/null +++ b/lib/codeintel/languages/languages.go @@ -0,0 +1,53 @@ +package languages + +import ( + "github.com/go-enry/go-enry/v2" + "golang.org/x/exp/slices" +) + +// TODO: We probably want to move the config for language detection into here from the syntax highlighting part +// I didn't add that yet. + +// GetLanguage returns the language for the given path and contents. +func GetLanguage(path, contents string) (lang string, found bool) { + // Force the use of the shebang. + if shebangLang, ok := overrideViaShebang(path, contents); ok { + return shebangLang, true + } + + // Lastly, fall back to whatever enry decides is a useful algorithm for calculating. + lang = enry.GetLanguage(path, []byte(contents)) + if lang != "" { + return lang, true + } + + return lang, false +} + +// overrideViaShebang handles explicitly using the shebang whenever possible. +// +// It also covers some edge cases when enry eagerly returns more languages +// than necessary, which ends up overriding the shebang completely (which, +// IMO is the highest priority match we can have). +// +// For example, enry will return "Perl" and "Pod" for a shebang of `#!/usr/bin/env perl`. +// This is actually unhelpful, because then enry will *not* select "Perl" as the +// language (which is our desired behavior). +func overrideViaShebang(path, content string) (lang string, ok bool) { + shebangs := enry.GetLanguagesByShebang(path, []byte(content), []string{}) + if len(shebangs) == 0 { + return "", false + } + + if len(shebangs) == 1 { + return shebangs[0], true + } + + // There are some shebangs that enry returns that are not really + // useful for our syntax highlighters to distinguish between. + if slices.Equal(shebangs, []string{"Perl", "Pod"}) { + return "Perl", true + } + + return "", false +} diff --git a/lib/go.mod b/lib/go.mod index 988b42c4e63..14a7c9d08ec 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -9,6 +9,7 @@ require ( github.com/derision-test/go-mockgen v1.3.7 github.com/fatih/color v1.13.0 github.com/ghodss/yaml v1.0.0 + github.com/go-enry/go-enry/v2 v2.8.4 github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.5.9 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db @@ -27,7 +28,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/urfave/cli/v2 v2.23.7 github.com/xeipuuv/gojsonschema v1.2.0 - go.uber.org/atomic v1.10.0 + golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 golang.org/x/sync v0.1.0 golang.org/x/sys v0.3.0 golang.org/x/term v0.3.0 @@ -55,6 +56,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/envoyproxy/protoc-gen-validate v0.6.13 // indirect github.com/getsentry/sentry-go v0.15.0 // indirect + github.com/go-enry/go-oniguruma v1.2.1 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -102,6 +104,7 @@ require ( github.com/yuin/goldmark v1.5.2 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect go.opencensus.io v0.24.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/goleak v1.2.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect diff --git a/lib/go.sum b/lib/go.sum index 26ea22fd7fb..e10e1b4b967 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -151,6 +151,10 @@ github.com/getsentry/sentry-go v0.15.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2Ht github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs= +github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= +github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= +github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -572,6 +576,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/monitoring/go.mod b/monitoring/go.mod index 9cd82429036..ebfb1055770 100644 --- a/monitoring/go.mod +++ b/monitoring/go.mod @@ -61,7 +61,7 @@ require ( go.uber.org/goleak v1.2.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect + golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect golang.org/x/oauth2 v0.2.0 // indirect golang.org/x/sys v0.3.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/monitoring/go.sum b/monitoring/go.sum index ab3efeadbd2..23756eb85ec 100644 --- a/monitoring/go.sum +++ b/monitoring/go.sum @@ -448,8 +448,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw= -golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -645,7 +645,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/schema/schema.go b/schema/schema.go index 27eb991ac3f..283d5ad38d9 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -2772,10 +2772,18 @@ type SubRepoPermissions struct { UserCacheTTLSeconds int `json:"userCacheTTLSeconds,omitempty"` } +// SymbolConfiguration description: Configure symbol generation +type SymbolConfiguration struct { + // Engine description: Manually specify overrides for symbol generation engine per language + Engine map[string]string `json:"engine"` +} + // SyntaxHighlighting description: Syntax highlighting configuration type SyntaxHighlighting struct { Engine SyntaxHighlightingEngine `json:"engine"` Languages SyntaxHighlightingLanguage `json:"languages"` + // Symbols description: Configure symbol generation + Symbols SymbolConfiguration `json:"symbols"` } type SyntaxHighlightingEngine struct { // Default description: The default syntax highlighting engine to use diff --git a/schema/site.schema.json b/schema/site.schema.json index 532b2933d9b..685631fcf11 100644 --- a/schema/site.schema.json +++ b/schema/site.schema.json @@ -848,7 +848,7 @@ "title": "SyntaxHighlighting", "description": "Syntax highlighting configuration", "type": "object", - "required": ["engine", "languages"], + "required": ["engine", "languages", "symbols"], "properties": { "engine": { "title": "SyntaxHighlightingEngine", @@ -903,6 +903,22 @@ } } } + }, + "symbols": { + "title": "SymbolConfiguration", + "description": "Configure symbol generation", + "type": "object", + "required": ["engine"], + "properties": { + "engine": { + "description": "Manually specify overrides for symbol generation engine per language", + "type": "object", + "additionalProperties": { + "type": "string", + "enum": ["universal-ctags", "scip-ctags", "off"] + } + } + } } }, "examples": [ diff --git a/sg.config.yaml b/sg.config.yaml index f1c51d1558a..566e95a0fee 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -340,6 +340,7 @@ commands: checkBinary: .bin/oss-symbols env: CTAGS_COMMAND: dev/universal-ctags-dev + SCIP_CTAGS_COMMAND: dev/scip-ctags-dev CTAGS_PROCESSES: 2 watch: - lib @@ -358,6 +359,7 @@ commands: checkBinary: .bin/symbols env: CTAGS_COMMAND: dev/universal-ctags-dev + SCIP_CTAGS_COMMAND: dev/scip-ctags-dev CTAGS_PROCESSES: 2 USE_ROCKSKIP: "false" watch: @@ -544,6 +546,7 @@ commands: checkBinary: .bin/zoekt-sourcegraph-indexserver env: &zoektenv CTAGS_COMMAND: dev/universal-ctags-dev + SCIP_CTAGS_COMMAND: dev/scip-ctags-dev GRPC_ENABLED: true zoekt-index-0: @@ -956,6 +959,7 @@ bazelCommands: target: //cmd/symbols env: CTAGS_COMMAND: dev/universal-ctags-dev + SCIP_CTAGS_COMMAND: dev/scip-ctags-dev CTAGS_PROCESSES: 2 oss-gitserver-0: target: //cmd/gitserver @@ -1023,6 +1027,7 @@ bazelCommands: checkBinary: .bin/symbols env: CTAGS_COMMAND: dev/universal-ctags-dev + SCIP_CTAGS_COMMAND: dev/scip-ctags-dev CTAGS_PROCESSES: 2 USE_ROCKSKIP: "false" gitserver-template: &gitserver_bazel_template