artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 17:31:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

syntax-highlighter: Add C# Support (#31811)

Browse Source 
Add C# support and fix some multi-line bugs for tree-sitter highlighting
This commit is contained in:
TJ DeVries 2022-03-02 12:02:36 -05:00 committed by GitHub
parent 58434a8249
commit 6187cdea71
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1600 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitmodules vendored
View File

@ -6,3 +6,7 @@
path = docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-sql
url = https://github.com/m-novikov/tree-sitter-sql
shallow = true
[submodule "tree-sitter-c_sharp"]
path = docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-c_sharp
url = https://github.com/tree-sitter/tree-sitter-c-sharp
shallow = true

6
client/web/src/lsif/html.ts
View File

@ -1,3 +1,5 @@
import escape from 'escape-html'
import { JsonDocument, Occurrence, SyntaxKind } from './lsif-typed'
class HtmlBuilder {
@ -14,7 +16,7 @@ class HtmlBuilder {
}
public element(element: string, attributes: string, value: string): void {
this.openTag(element + ' ' + attributes)
this.raw(value)
this.raw(escape(value))
this.closeTag(element)
}
public raw(html: string): void {
@ -103,7 +105,7 @@ export function render(lsif_json: string, content: string): string {
// At this time, the syntax highlighter only returns non-overlapping ranges so this
// is OK.
if (start.line !== end.line) {
html.plaintext(line.slice(start.character))
highlightSlice(html, occ.kind, line.slice(start.character))
closeLine(html)
// Move to the next line

612
docker-images/syntax-highlighter/Cargo.lock generated
View File

@ -90,6 +90,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base-x"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b"
[[package]]
name = "base64"
version = "0.13.0"
@ -117,6 +123,12 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bumpalo"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "bytes"
version = "1.1.0"
@ -135,6 +147,17 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clipboard-win"
version = "4.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f3e1238132dc01f081e1cbb9dace14e5ef4c3a51ee244bd982275fb514605db"
dependencies = [
"error-code",
"str-buf",
"winapi",
]
[[package]]
name = "console"
version = "0.15.0"
@ -148,6 +171,23 @@ dependencies = [
"winapi",
]
[[package]]
name = "const_fn"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935"
[[package]]
name = "cookie"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d"
dependencies = [
"percent-encoding",
"time 0.2.27",
"version_check",
]
[[package]]
name = "cookie"
version = "0.16.0"
@ -155,7 +195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05"
dependencies = [
"percent-encoding",
"time",
"time 0.3.7",
"version_check",
]
@ -217,6 +257,33 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
[[package]]
name = "dirs-next"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
dependencies = [
"cfg-if",
"dirs-sys-next",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "discard"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0"
[[package]]
name = "either"
version = "1.6.1"
@ -238,6 +305,43 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "endian-type"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "errno"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
dependencies = [
"errno-dragonfly",
"libc",
"winapi",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "error-code"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21"
dependencies = [
"libc",
"str-buf",
]
[[package]]
name = "fastrand"
version = "1.7.0"
@ -247,6 +351,17 @@ dependencies = [
"instant",
]
[[package]]
name = "fd-lock"
version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02ecad9808e0596f8956d14f7fa868f996290bd01c8d7329d6e5bc2bb76adf8f"
dependencies = [
"cfg-if",
"rustix",
"windows-sys",
]
[[package]]
name = "figment"
version = "0.10.6"
@ -287,6 +402,7 @@ checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
@ -309,12 +425,34 @@ version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7"
[[package]]
name = "futures-executor"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2"
[[package]]
name = "futures-macro"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.19"
@ -336,6 +474,7 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
@ -506,6 +645,12 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "io-lifetimes"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec58677acfea8a15352d42fc87d11d63596ade9239e0a7c9352914417515dbe6"
[[package]]
name = "itoa"
version = "0.4.8"
@ -532,9 +677,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.115"
version = "0.2.119"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a8d982fa7a96a000f6ec4cfe966de9703eccde29750df2bb8949da91b0e818d"
checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
[[package]]
name = "libloading"
@ -561,6 +706,12 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
[[package]]
name = "linux-raw-sys"
version = "0.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5284f00d480e1c39af34e72f8ad60b94f47007e3481cd3b731c1d67190ddc7b7"
[[package]]
name = "lock_api"
version = "0.4.5"
@ -609,6 +760,15 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "memoffset"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.16"
@ -667,6 +827,28 @@ dependencies = [
"version_check",
]
[[package]]
name = "nibble_vec"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
dependencies = [
"smallvec",
]
[[package]]
name = "nix"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6"
dependencies = [
"bitflags",
"cc",
"cfg-if",
"libc",
"memoffset",
]
[[package]]
name = "ntapi"
version = "0.3.6"
@ -820,7 +1002,7 @@ dependencies = [
"indexmap",
"line-wrap",
"serde",
"time",
"time 0.3.7",
"xml-rs",
]
@ -842,6 +1024,12 @@ dependencies = [
"output_vt100",
]
[[package]]
name = "proc-macro-hack"
version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]]
name = "proc-macro2"
version = "1.0.36"
@ -879,6 +1067,16 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "radix_trie"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
dependencies = [
"endian-type",
"nibble_vec",
]
[[package]]
name = "rand"
version = "0.8.4"
@ -928,6 +1126,16 @@ dependencies = [
"bitflags",
]
[[package]]
name = "redox_users"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
dependencies = [
"getrandom",
"redox_syscall",
]
[[package]]
name = "ref-cast"
version = "1.0.6"
@ -983,6 +1191,45 @@ dependencies = [
"winapi",
]
[[package]]
name = "rocket"
version = "0.5.0-rc.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a71c18c42a0eb15bf3816831caf0dad11e7966f2a41aaf486a701979c4dd1f2"
dependencies = [
"async-stream",
"async-trait",
"atomic",
"atty",
"binascii",
"bytes",
"either",
"figment",
"futures",
"indexmap",
"log",
"memchr",
"multer",
"num_cpus",
"parking_lot",
"pin-project-lite",
"rand",
"ref-cast",
"rocket_codegen 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_http 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde",
"serde_json",
"state",
"tempfile",
"time 0.2.27",
"tokio",
"tokio-stream",
"tokio-util",
"ubyte",
"version_check",
"yansi",
]
[[package]]
name = "rocket"
version = "0.5.0-rc.1"
@ -1006,13 +1253,13 @@ dependencies = [
"pin-project-lite",
"rand",
"ref-cast",
"rocket_codegen",
"rocket_http",
"rocket_codegen 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)",
"rocket_http 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)",
"serde",
"serde_json",
"state",
"tempfile",
"time",
"time 0.3.7",
"tokio",
"tokio-stream",
"tokio-util",
@ -1021,6 +1268,22 @@ dependencies = [
"yansi",
]
[[package]]
name = "rocket_codegen"
version = "0.5.0-rc.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66f5fa462f7eb958bba8710c17c5d774bbbd59809fa76fb1957af7e545aea8bb"
dependencies = [
"devise",
"glob",
"indexmap",
"proc-macro2",
"quote",
"rocket_http 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)",
"syn",
"unicode-xid",
]
[[package]]
name = "rocket_codegen"
version = "0.5.0-rc.1"
@ -1031,17 +1294,45 @@ dependencies = [
"indexmap",
"proc-macro2",
"quote",
"rocket_http",
"rocket_http 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)",
"syn",
"unicode-xid",
]
[[package]]
name = "rocket_http"
version = "0.5.0-rc.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23c8b7d512d2fcac2316ebe590cde67573844b99e6cc9ee0f53375fa16e25ebd"
dependencies = [
"cookie 0.15.1",
"either",
"http",
"hyper",
"indexmap",
"log",
"memchr",
"mime",
"parking_lot",
"pear",
"percent-encoding",
"pin-project-lite",
"ref-cast",
"serde",
"smallvec",
"stable-pattern",
"state",
"time 0.2.27",
"tokio",
"uncased",
]
[[package]]
name = "rocket_http"
version = "0.5.0-rc.1"
source = "git+https://github.com/SergioBenitez/Rocket#8cae077ba1d54b92cdef3e171a730b819d5eeb8e"
dependencies = [
"cookie",
"cookie 0.16.0",
"either",
"http",
"hyper",
@ -1056,17 +1347,64 @@ dependencies = [
"smallvec",
"stable-pattern",
"state",
"time",
"time 0.3.7",
"tokio",
"uncased",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "0.33.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9466f25b92a648960ac1042fd3baa6b0bf285e60f754d7e5070770c813a177a"
dependencies = [
"bitflags",
"errno",
"io-lifetimes",
"libc",
"linux-raw-sys",
"winapi",
]
[[package]]
name = "rustversion"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f"
[[package]]
name = "rustyline"
version = "9.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039"
dependencies = [
"bitflags",
"cfg-if",
"clipboard-win",
"dirs-next",
"fd-lock",
"libc",
"log",
"memchr",
"nix",
"radix_trie",
"scopeguard",
"smallvec",
"unicode-segmentation",
"unicode-width",
"utf8parse",
"winapi",
]
[[package]]
name = "ryu"
version = "1.0.9"
@ -1100,6 +1438,21 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
"semver-parser",
]
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.136"
@ -1171,7 +1524,7 @@ dependencies = [
"libloading",
"paste",
"protobuf",
"rocket",
"rocket 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)",
"serde",
"serde_json",
"sg-lsif",
@ -1181,6 +1534,21 @@ dependencies = [
"tree-sitter-highlight",
]
[[package]]
name = "sha1"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770"
dependencies = [
"sha1_smol",
]
[[package]]
name = "sha1_smol"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "sharded-slab"
version = "0.1.4"
@ -1242,6 +1610,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "standback"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff"
dependencies = [
"version_check",
]
[[package]]
name = "state"
version = "0.5.2"
@ -1251,6 +1628,61 @@ dependencies = [
"loom",
]
[[package]]
name = "stdweb"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
dependencies = [
"discard",
"rustc_version",
"stdweb-derive",
"stdweb-internal-macros",
"stdweb-internal-runtime",
"wasm-bindgen",
]
[[package]]
name = "stdweb-derive"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef"
dependencies = [
"proc-macro2",
"quote",
"serde",
"serde_derive",
"syn",
]
[[package]]
name = "stdweb-internal-macros"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11"
dependencies = [
"base-x",
"proc-macro2",
"quote",
"serde",
"serde_derive",
"serde_json",
"sha1",
"syn",
]
[[package]]
name = "stdweb-internal-runtime"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
[[package]]
name = "str-buf"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a"
[[package]]
name = "syn"
version = "1.0.86"
@ -1287,7 +1719,8 @@ dependencies = [
name = "syntect_server"
version = "1.0.1"
dependencies = [
"rocket",
"rocket 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rustyline",
"serde",
"serde_json",
"sg-syntax",
@ -1347,6 +1780,21 @@ dependencies = [
"once_cell",
]
[[package]]
name = "time"
version = "0.2.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242"
dependencies = [
"const_fn",
"libc",
"standback",
"stdweb",
"time-macros 0.1.1",
"version_check",
"winapi",
]
[[package]]
name = "time"
version = "0.3.7"
@ -1356,7 +1804,17 @@ dependencies = [
"itoa 1.0.1",
"libc",
"num_threads",
"time-macros",
"time-macros 0.2.3",
]
[[package]]
name = "time-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1"
dependencies = [
"proc-macro-hack",
"time-macros-impl",
]
[[package]]
@ -1365,6 +1823,19 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25eb0ca3468fc0acc11828786797f6ef9aa1555e4a211a60d64cc8e4d1be47d6"
[[package]]
name = "time-macros-impl"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f"
dependencies = [
"proc-macro-hack",
"proc-macro2",
"quote",
"standback",
"syn",
]
[[package]]
name = "tokio"
version = "1.15.0"
@ -1541,12 +2012,30 @@ dependencies = [
"version_check",
]
[[package]]
name = "unicode-segmentation"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
[[package]]
name = "unicode-width"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]]
name = "unicode-xid"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "utf8parse"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
[[package]]
name = "version_check"
version = "0.9.4"
@ -1580,6 +2069,60 @@ version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
[[package]]
name = "winapi"
version = "0.3.9"
@ -1611,6 +2154,49 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-sys"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "030b7ff91626e57a05ca64a07c481973cbb2db774e4852c9c7ca342408c6a99a"
dependencies = [
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_msvc"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29277a4435d642f775f63c7d1faeb927adba532886ce0287bd985bffb16b6bca"
[[package]]
name = "windows_i686_gnu"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1145e1989da93956c68d1864f32fb97c8f561a8f89a5125f6a2b7ea75524e4b8"
[[package]]
name = "windows_i686_msvc"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4a09e3a0d4753b73019db171c1339cd4362c8c44baf1bcea336235e955954a6"
[[package]]
name = "windows_x86_64_gnu"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ca64fcb0220d58db4c119e050e7af03c69e6f4f415ef69ec1773d9aab422d5a"
[[package]]
name = "windows_x86_64_msvc"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08cabc9f0066848fef4bc6a1c1668e6efce38b661d2aeec75d18d8617eebb5f1"
[[package]]
name = "xml-rs"
version = "0.8.4"

3
docker-images/syntax-highlighter/Cargo.toml
View File

@ -10,10 +10,11 @@ license = "MIT"
[dependencies]
syntect = { git = "https://github.com/slimsag/syntect" }
rocket = { git = "https://github.com/SergioBenitez/Rocket", features = ["json"] }
rocket = { version = "0.5.0-rc.1", features = ["json"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sg-syntax = { path = "./crates/sg-syntax" }
rustyline = "9.1.2"
[workspace]
members = [

1
docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-c_sharp Submodule

@ -0,0 +1 @@
Subproject commit 352a4630c81a7a5cbd3bc67327743bd8d38f2dd2

262
docker-images/syntax-highlighter/crates/sg-syntax/queries/c_sharp/highlights.scm Normal file
View File

@ -0,0 +1,262 @@
(using_directive (identifier) @variable.module)
(qualified_name (identifier) @variable.module)
;; Methods
(method_declaration (identifier) @type (identifier) @function)
(invocation_expression
(member_access_expression
(generic_name
(identifier) @method)))
(invocation_expression
(member_access_expression
name: (identifier) @method))
(invocation_expression
function: (conditional_access_expression
(member_binding_expression
name: (identifier) @method)))
(invocation_expression
(identifier) @method)
(invocation_expression
function: (generic_name
. (identifier) @method))
;; Types
(interface_declaration name: (identifier) @type)
(class_declaration name: (identifier) @type)
(enum_declaration name: (identifier) @type)
(struct_declaration (identifier) @type)
(record_declaration (identifier) @type)
(namespace_declaration name: (identifier) @type)
(constructor_declaration name: (identifier) @type)
[
(implicit_type)
(nullable_type)
(pointer_type)
(function_pointer_type)
(predefined_type)
] @type.builtin
;; Enum
(enum_member_declaration (identifier) @property.definition)
;; Literals
[
(real_literal)
(integer_literal)
] @number
[
(character_literal)
(string_literal)
(verbatim_string_literal)
(interpolated_string_text)
(interpolated_verbatim_string_text)
"\""
"$\""
"@$\""
"$@\""
] @string
[
(boolean_literal)
(null_literal)
(void_keyword)
] @constant.builtin
;; Comments
(comment) @comment
;; Tokens
[
";"
"."
","
] @punctuation.delimiter
[
"--"
"-"
"-="
"&"
"&&"
"+"
"++"
"+="
"<"
"<<"
"="
"=="
"!"
"!="
"=>"
">"
">>"
"|"
"||"
"?"
"??"
"^"
"~"
"*"
"/"
"%"
":"
] @operator
[
"("
")"
"["
"]"
"{"
"}"
] @punctuation.bracket
;; Keywords
(modifier) @keyword
(this_expression) @keyword
(escape_sequence) @keyword
[
"as"
"base"
"break"
"case"
"catch"
"checked"
"class"
"continue"
"default"
"delegate"
"do"
"else"
"enum"
"event"
"explicit"
"finally"
"for"
"foreach"
"goto"
"if"
"implicit"
"interface"
"is"
"lock"
"namespace"
"operator"
"params"
"return"
"sizeof"
"stackalloc"
"struct"
"switch"
"throw"
"try"
"typeof"
"unchecked"
"using"
"while"
"new"
"await"
"in"
"yield"
"get"
"set"
"when"
"out"
"ref"
"from"
"where"
"select"
"record"
"init"
"with"
"let"
] @keyword
;; Linq
(from_clause (identifier) @variable)
(group_clause)
(order_by_clause)
(select_clause (identifier) @variable)
(query_continuation (identifier) @variable) @keyword
;; Record
(with_expression
(with_initializer_expression
(simple_assignment_expression
(identifier) @variable)))
;; Exprs
(binary_expression (identifier) @variable (identifier) @variable)
(binary_expression (identifier)* @variable)
(conditional_expression (identifier) @variable)
(prefix_unary_expression (identifier) @variable)
(postfix_unary_expression (identifier)* @variable)
(assignment_expression (identifier) @variable)
(cast_expression (identifier) @type (identifier) @variable)
;; Class
(base_list (identifier) @type)
(property_declaration (generic_name))
(property_declaration
type: (nullable_type) @type
name: (identifier) @variable)
(property_declaration
type: (predefined_type) @type
name: (identifier) @variable)
(property_declaration
type: (identifier) @type
name: (identifier) @variable)
;; Lambda
(lambda_expression) @variable
;; Attribute
(attribute) @type
;; Parameter
(parameter
type: (identifier) @type
name: (identifier) @variable.parameter)
(parameter (identifier) @variable.parameter)
(parameter_modifier) @keyword
;; Typeof
(type_of_expression (identifier) @type)
;; Variable
(variable_declaration (identifier) @type)
(variable_declarator (identifier) @variable)
;; Return
(return_statement (identifier) @variable)
(yield_statement (identifier) @variable)
;; Type
(generic_name (identifier) @type)
(type_parameter (identifier) @property.definition)
(type_argument_list (identifier) @type)
;; Type constraints
(type_parameter_constraints_clause (identifier) @property.definition)
(type_constraint (identifier) @type)
;; Exception
(catch_declaration (identifier) @type (identifier) @variable)
(catch_declaration (identifier) @type)
;; Switch
(switch_statement (identifier) @variable)
(switch_expression (identifier) @variable)
;; Lock statement
(lock_statement (identifier) @variable)

20
docker-images/syntax-highlighter/crates/sg-syntax/src/lib.rs
View File

@ -10,8 +10,12 @@ use syntect::{
mod sg_treesitter;
pub use sg_treesitter::dump_document;
pub use sg_treesitter::dump_document_range;
pub use sg_treesitter::index_language as lsif_index;
pub use sg_treesitter::index_language_with_config as lsif_index_with_config;
pub use sg_treesitter::lsif_highlight;
pub use sg_treesitter::make_highlight_config;
pub use sg_treesitter::FileRange as DocumentFileRange;
pub use sg_treesitter::PackedRange as LsifPackedRange;
mod sg_syntect;
@ -60,7 +64,21 @@ pub struct SourcegraphQuery {
pub code: String,
}
pub fn determine_language<'a>(
pub fn determine_filetype(q: &SourcegraphQuery) -> String {
let filetype = SYNTAX_SET.with(|syntax_set| match determine_language(q, syntax_set) {
Ok(language) => language.name.clone(),
Err(_) => "".to_owned(),
});
// We normalize all the filenames here
match filetype.as_str() {
"C#" => "c_sharp",
filetype => filetype,
}
.to_lowercase()
}
fn determine_language<'a>(
q: &SourcegraphQuery,
syntax_set: &'a SyntaxSet,
) -> Result<&'a SyntaxReference, JsonValue> {

120
docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs
View File

@ -43,7 +43,7 @@ const MATCHES_TO_SYNTAX_KINDS: &[(&str, SyntaxKind)] = &[
("keyword", SyntaxKind::IdentifierKeyword),
("keyword.function", SyntaxKind::IdentifierKeyword),
("keyword.return", SyntaxKind::IdentifierKeyword),
("method", SyntaxKind::Identifier),
("method", SyntaxKind::IdentifierFunction),
("number", SyntaxKind::NumericLiteral),
("operator", SyntaxKind::IdentifierOperator),
("property", SyntaxKind::Identifier),
@ -112,7 +112,7 @@ macro_rules! create_configurations {
lazy_static::lazy_static! {
static ref CONFIGURATIONS: HashMap<&'static str, HighlightConfiguration> = {
create_configurations!( go, sql )
create_configurations!( go, sql, c_sharp )
};
}
@ -139,17 +139,46 @@ pub fn lsif_highlight(q: SourcegraphQuery) -> Result<JsonValue, JsonValue> {
}
pub fn index_language(filetype: &str, code: &str) -> Result<Document, Error> {
let mut highlighter = TSHighlighter::new();
let lang_config = match CONFIGURATIONS.get(filetype) {
Some(lang_config) => lang_config,
None => return Err(Error::InvalidLanguage),
};
index_language_with_config(code, &lang_config)
}
pub fn make_highlight_config(name: &str, highlights: &str) -> Option<HighlightConfiguration> {
let config = match CONFIGURATIONS.get(name) {
Some(config) => config,
None => return None,
};
// Create HighlightConfiguration language
let mut lang = match HighlightConfiguration::new(config.language, highlights, "", "") {
Ok(lang) => lang,
Err(_) => return None,
};
// Associate highlights with configuration
let highlight_names = MATCHES_TO_SYNTAX_KINDS
.iter()
.map(|hl| hl.0)
.collect::<Vec<&str>>();
lang.configure(&highlight_names);
Some(lang)
}
pub fn index_language_with_config(
code: &str,
lang_config: &HighlightConfiguration,
) -> Result<Document, Error> {
// TODO: We should automatically apply no highlights when we are
// in an injected piece of code.
//
// Unfortunately, that information isn't currently available when
// we are iterating in the higlighter.
let mut highlighter = TSHighlighter::new();
let highlights = highlighter.highlight(lang_config, code.as_bytes(), None, |l| {
CONFIGURATIONS.get(l)
})?;
@ -313,14 +342,34 @@ impl LsifEmitter {
}
}
pub fn dump_document(doc: Document, source: &str) -> String {
pub fn dump_document(doc: &Document, source: &str) -> String {
dump_document_range(doc, source, &None)
}
pub struct FileRange {
pub start: usize,
pub end: usize,
}
pub fn dump_document_range(doc: &Document, source: &str, file_range: &Option<FileRange>) -> String {
let mut occurences = doc.get_occurrences().to_owned();
occurences.sort_by_key(|o| PackedRange::from_vec(&o.range));
let mut occurences = VecDeque::from(occurences);
let mut result = String::new();
for (idx, line) in source.lines().enumerate() {
let line_iterator: Box<dyn Iterator<Item = (usize, &str)>> = match file_range {
Some(range) => Box::new(
source
.lines()
.enumerate()
.skip(range.start - 1)
.take(range.end - range.start + 1),
),
None => Box::new(source.lines().enumerate()),
};
for (idx, line) in line_iterator {
result += " ";
result += &line.replace("\t", " ");
result += "\n";
@ -335,7 +384,9 @@ pub fn dump_document(doc: Document, source: &str) -> String {
continue;
}
if range.start_line != idx as i32 {
if range.start_line < idx as i32 {
continue;
} else if range.start_line > idx as i32 {
occurences.push_front(occ);
break;
}
@ -356,13 +407,20 @@ pub fn dump_document(doc: Document, source: &str) -> String {
#[cfg(test)]
mod test {
use std::{
fs::{read_dir, File},
io::Read,
};
use crate::determine_filetype;
use super::*;
#[test]
fn test_highlights_one_comment() -> Result<(), Error> {
let src = "// Hello World";
let document = index_language("go", src)?;
insta::assert_snapshot!(dump_document(document, src));
insta::assert_snapshot!(dump_document(&document, src));
Ok(())
}
@ -378,7 +436,7 @@ func main() {
"#;
let document = index_language("go", src)?;
insta::assert_snapshot!(dump_document(document, src));
insta::assert_snapshot!(dump_document(&document, src));
Ok(())
}
@ -393,7 +451,51 @@ SELECT * FROM my_table
"#;
let document = index_language("go", src)?;
insta::assert_snapshot!(dump_document(document, src));
insta::assert_snapshot!(dump_document(&document, src));
Ok(())
}
#[test]
fn test_highlight_csharp_file() -> Result<(), Error> {
let src = "using System;";
let document = index_language("c_sharp", src)?;
insta::assert_snapshot!(dump_document(&document, src));
Ok(())
}
#[test]
fn test_all_files() -> Result<(), std::io::Error> {
let dir = read_dir("./src/snapshots/files/")?;
for entry in dir.into_iter() {
let entry = entry?;
let filepath = entry.path();
let mut file = File::open(&filepath)?;
let mut contents = String::new();
file.read_to_string(&mut contents)?;
let filetype = &determine_filetype(&SourcegraphQuery {
extension: filepath.extension().unwrap().to_str().unwrap().to_string(),
filepath: filepath.to_str().unwrap().to_string(),
filetype: None,
css: false,
line_length_limit: None,
theme: "".to_string(),
code: contents.clone(),
});
println!("Filetype: {filetype}");
let document = index_language(filetype, &contents).unwrap();
insta::assert_snapshot!(
filepath
.to_str()
.unwrap()
.replace("/src/snapshots/files", ""),
dump_document(&document, &contents)
);
}
Ok(())
}

59
docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/files/csharp_timed_out_syntect.cs Normal file
View File

@ -0,0 +1,59 @@
// Original:
// repo:^github\.com/radzenhq/radzen-blazor$ file:^Radzen\.Blazor\.Tests/DataGridTests\.cs
using AngleSharp.Dom;
using Bunit;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Rendering;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Xunit;
namespace Radzen.Blazor.
{
public class DataGridTests
{
// Css classes tests
[Fact]
public void DataGrid_Renders_CssClass()
{
using var ctx = new TestContext();
ctx.JSInterop.Mode = JSRuntimeMode.Loose;
ctx.JSInterop.SetupModule("_content/Radzen.Blazor/Radzen.Blazor.js");
var component = ctx.RenderComponent<RadzenGrid<dynamic>>(parameterBuilder =>
{
parameterBuilder.Add<IEnumerable<dynamic>>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } });
parameterBuilder.Add<RenderFragment>(p => p.Columns, builder =>
{
builder.OpenComponent(0, typeof(RadzenGridColumn<dynamic>));
builder.AddAttribute(1, "Property", "Id");
builder.CloseComponent();
});
});
// Main
Assert.Contains(@$"rz-datatable-scrollable-wrapper", component.Markup);
Assert.Contains(@$"rz-datatable-scrollable-view", component.Markup);
var component = ctx.RenderComponent<RadzenGrid<dynamic>>(parameterBuilder =>
{
parameterBuilder.Add<IEnumerable<dynamic>>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } });
parameterBuilder.Add<RenderFragment>(p => p.Columns, builder =>
{
builder.OpenComponent(0, typeof(RadzenGridColumn<dynamic>));
builder.AddAttribute(1, "Property", "Id");
builder.CloseComponent();
});
});
var markup = new Regex(@"\s\s+").Replace(component.Markup, "").Trim();
Assert.Contains(@$"""rz-cell-data"">1", markup);
Assert.Contains(@$"""rz-cell-data"">2", markup);
Assert.Contains(@$"""rz-cell-data"">3", markup);
Assert.Contains(@"
Lorem Ipsum
", component.Markup);
}
}
}

417
docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__.__csharp_timed_out_syntect.cs.snap Normal file
View File

@ -0,0 +1,417 @@
---
source: crates/sg-syntax/src/sg_treesitter.rs
assertion_line: 491
expression: "dump_document(&document, &contents)"
---
// Original:
//^^^^^^^^^^^^ Comment
// repo:^github\.com/radzenhq/radzen-blazor$ file:^Radzen\.Blazor\.Tests/DataGridTests\.cs
//^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Comment
using AngleSharp.Dom;
//^^^^^ IdentifierKeyword
// ^^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^ IdentifierModule
// ^ PunctuationDelimiter
using Bunit;
//^^^^^ IdentifierKeyword
// ^^^^^ IdentifierModule
// ^ PunctuationDelimiter
using Microsoft.AspNetCore.Components;
//^^^^^ IdentifierKeyword
// ^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
using Microsoft.AspNetCore.Components.Rendering;
//^^^^^ IdentifierKeyword
// ^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
using System;
//^^^^^ IdentifierKeyword
// ^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
using System.Collections.Generic;
//^^^^^ IdentifierKeyword
// ^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
using System.Linq;
//^^^^^ IdentifierKeyword
// ^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^ IdentifierModule
// ^ PunctuationDelimiter
using System.Text.RegularExpressions;
//^^^^^ IdentifierKeyword
// ^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
using Xunit;
//^^^^^ IdentifierKeyword
// ^^^^^ IdentifierModule
// ^ PunctuationDelimiter
namespace Radzen.Blazor.
//^^^^^^^^^ IdentifierKeyword
// ^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
// ^^^^^^ IdentifierModule
// ^ PunctuationDelimiter
{
public class DataGridTests
// ^^^^^^ IdentifierKeyword
// ^^^^^ IdentifierKeyword
// ^^^^^^^^^^^^^ IdentifierType
{
// Css classes tests
// ^^^^^^^^^^^^^^^^^^^^ Comment
[Fact]
// ^^^^ IdentifierType
public void DataGrid_Renders_CssClass()
// ^^^^^^ IdentifierKeyword
// ^^^^ IdentifierBuiltin
{
using var ctx = new TestContext();
// ^^^^^ IdentifierKeyword
// ^^^ IdentifierType
// ^^^ Identifier
// ^ IdentifierOperator
// ^^^ IdentifierKeyword
// ^ PunctuationDelimiter
ctx.JSInterop.Mode = JSRuntimeMode.Loose;
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
// ^ IdentifierOperator
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
ctx.JSInterop.SetupModule("_content/Radzen.Blazor/Radzen.Blazor.js");
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
// ^^^^^^^^^^^ IdentifierFunction
// ^ StringLiteral
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
var component = ctx.RenderComponent<RadzenGrid<dynamic>>(parameterBuilder =>
// ^^^ IdentifierType
// ^^^^^^^^^ Identifier
// ^ IdentifierOperator
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^^^^ IdentifierFunction
// ^ IdentifierOperator
// ^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^ IdentifierOperator
// ^^^^^^^^^^^^^^^^^ Identifier
// ^^ IdentifierOperator
{
parameterBuilder.Add<IEnumerable<dynamic>>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } });
// ^ PunctuationDelimiter
// ^^^ IdentifierFunction
// ^ IdentifierOperator
// ^^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^ IdentifierOperator
// ^^ Identifier
// ^^ IdentifierOperator
// ^^ Identifier
// ^ PunctuationDelimiter
// ^^^^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^^^^ Identifier
// ^ IdentifierOperator
// ^ Identifier
// ^ NumericLiteral
// ^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^^^^ Identifier
// ^ IdentifierOperator
// ^ Identifier
// ^ NumericLiteral
// ^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^^^^ Identifier
// ^ IdentifierOperator
// ^ Identifier
// ^ NumericLiteral
// ^ Identifier
// ^ Identifier
// ^ PunctuationDelimiter
parameterBuilder.Add<RenderFragment>(p => p.Columns, builder =>
// ^ PunctuationDelimiter
// ^^^ IdentifierFunction
// ^ IdentifierOperator
// ^^^^^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^ Identifier
// ^^ IdentifierOperator
// ^^ Identifier
// ^ PunctuationDelimiter
// ^^^^^^^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^^^^^^ Identifier
// ^^ IdentifierOperator
{
builder.OpenComponent(0, typeof(RadzenGridColumn<dynamic>));
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^^ IdentifierFunction
// ^ NumericLiteral
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^^^^ IdentifierKeyword
// ^^^^^^^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^ PunctuationDelimiter
builder.AddAttribute(1, "Property", "Id");
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^ IdentifierFunction
// ^ NumericLiteral
// ^ PunctuationDelimiter
// ^ Identifier
// ^ StringLiteral
// ^^^^^^^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^ Identifier
// ^ StringLiteral
// ^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
builder.CloseComponent();
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^^^ IdentifierFunction
// ^ PunctuationDelimiter
});
// ^ PunctuationDelimiter
});
// ^ PunctuationDelimiter
// Main
// ^^^^^^^ Comment
Assert.Contains(@$"rz-datatable-scrollable-wrapper", component.Markup);
// ^ PunctuationDelimiter
// ^^^^^^^^ IdentifierFunction
// ^^^ StringLiteral
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
Assert.Contains(@$"rz-datatable-scrollable-view", component.Markup);
// ^ PunctuationDelimiter
// ^^^^^^^^ IdentifierFunction
// ^^^ StringLiteral
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
var component = ctx.RenderComponent<RadzenGrid<dynamic>>(parameterBuilder =>
// ^^^ IdentifierType
// ^^^^^^^^^ Identifier
// ^ IdentifierOperator
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^^^^ IdentifierFunction
// ^ IdentifierOperator
// ^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^ IdentifierOperator
// ^^^^^^^^^^^^^^^^^ Identifier
// ^^ IdentifierOperator
{
parameterBuilder.Add<IEnumerable<dynamic>>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } });
// ^ PunctuationDelimiter
// ^^^ IdentifierFunction
// ^ IdentifierOperator
// ^^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^ IdentifierOperator
// ^^ Identifier
// ^^ IdentifierOperator
// ^^ Identifier
// ^ PunctuationDelimiter
// ^^^^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^^^^ Identifier
// ^ IdentifierOperator
// ^ Identifier
// ^ NumericLiteral
// ^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^^^^ Identifier
// ^ IdentifierOperator
// ^ Identifier
// ^ NumericLiteral
// ^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^ IdentifierKeyword
// ^ Identifier
// ^^^^ Identifier
// ^ IdentifierOperator
// ^ Identifier
// ^ NumericLiteral
// ^ Identifier
// ^ Identifier
// ^ PunctuationDelimiter
parameterBuilder.Add<RenderFragment>(p => p.Columns, builder =>
// ^ PunctuationDelimiter
// ^^^ IdentifierFunction
// ^ IdentifierOperator
// ^^^^^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^ Identifier
// ^^ IdentifierOperator
// ^^ Identifier
// ^ PunctuationDelimiter
// ^^^^^^^ Identifier
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^^^^^^ Identifier
// ^^ IdentifierOperator
{
builder.OpenComponent(0, typeof(RadzenGridColumn<dynamic>));
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^^ IdentifierFunction
// ^ NumericLiteral
// ^ PunctuationDelimiter
// ^ Identifier
// ^^^^^^ IdentifierKeyword
// ^^^^^^^^^^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^^^^^^^ IdentifierType
// ^ IdentifierOperator
// ^ PunctuationDelimiter
builder.AddAttribute(1, "Property", "Id");
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^ IdentifierFunction
// ^ NumericLiteral
// ^ PunctuationDelimiter
// ^ Identifier
// ^ StringLiteral
// ^^^^^^^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^ Identifier
// ^ StringLiteral
// ^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
builder.CloseComponent();
// ^ PunctuationDelimiter
// ^^^^^^^^^^^^^^ IdentifierFunction
// ^ PunctuationDelimiter
});
// ^ PunctuationDelimiter
});
// ^ PunctuationDelimiter
var markup = new Regex(@"\s\s+").Replace(component.Markup, "").Trim();
// ^^^ IdentifierType
// ^^^^^^ Identifier
// ^ IdentifierOperator
// ^^^ IdentifierKeyword
// ^^^^^^^^ StringLiteral
// ^ PunctuationDelimiter
// ^^^^^^^ IdentifierFunction
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
// ^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^^^^ IdentifierFunction
// ^ PunctuationDelimiter
Assert.Contains(@$"""rz-cell-data"">1", markup);
// ^ PunctuationDelimiter
// ^^^^^^^^ IdentifierFunction
// ^^^ StringLiteral
// ^^ StringLiteral
// ^^^^^^^^^^^^ StringLiteral
// ^^ StringLiteral
// ^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
Assert.Contains(@$"""rz-cell-data"">2", markup);
// ^ PunctuationDelimiter
// ^^^^^^^^ IdentifierFunction
// ^^^ StringLiteral
// ^^ StringLiteral
// ^^^^^^^^^^^^ StringLiteral
// ^^ StringLiteral
// ^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
Assert.Contains(@$"""rz-cell-data"">3", markup);
// ^ PunctuationDelimiter
// ^^^^^^^^ IdentifierFunction
// ^^^ StringLiteral
// ^^ StringLiteral
// ^^^^^^^^^^^^ StringLiteral
// ^^ StringLiteral
// ^^ StringLiteral
// ^ StringLiteral
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
Assert.Contains(@"
// ^ PunctuationDelimiter
// ^^^^^^^^ IdentifierFunction
Lorem Ipsum
", component.Markup);
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
// ^ PunctuationDelimiter
}
}
}

11
docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__highlight_csharp_file.snap Normal file
View File

@ -0,0 +1,11 @@
---
source: crates/sg-syntax/src/sg_treesitter.rs
assertion_line: 461
expression: "dump_document(&document, src)"
---
using System;
//^^^^^ IdentifierKeyword
// ^^^^^^ IdentifierModule
// ^ PunctuationDelimiter

3
docker-images/syntax-highlighter/src/bin/lsif-snapshot.rs
View File

@ -21,8 +21,7 @@ fn main() {
}
};
println!("\n\n{}", dump_document(document, &contents));
// println!("{}", dump_document())
println!("\n\n{}", dump_document(&document, &contents));
} else {
panic!("Must pass a filepath");
}

83
docker-images/syntax-highlighter/src/bin/lsif-syntax-repl.rs Normal file
View File

@ -0,0 +1,83 @@
use std::fs;
use rustyline::config::Configurer;
use rustyline::Config;
use sg_syntax::dump_document_range;
use sg_syntax::lsif_index_with_config;
use sg_syntax::make_highlight_config;
use sg_syntax::DocumentFileRange;
fn main() {
println!("========================================");
println!(" Welcome to lsif-syntax-repl");
println!("========================================");
let contents = if let Some(path) = std::env::args().nth(1) {
match fs::read_to_string(&path) {
Ok(contents) => contents,
Err(err) => {
eprintln!("Failed to read path: {:?}. {}", path, err);
return;
}
}
} else {
let mut rl = rustyline::Editor::<()>::new();
match rl.readline("Contents: ") {
Ok(contents) => contents,
Err(err) => {
eprintln!("Failed to read path: {err}");
return;
}
}
};
let mut config = Config::builder();
config.set_max_history_size(100);
config.set_auto_add_history(true);
let mut rl = rustyline::Editor::<()>::with_config(config.build());
let range: Option<DocumentFileRange> = match rl.readline("Range (Optional, 1-Indexed): ") {
Ok(line) => {
if line.is_empty() {
None
} else {
let line_number = line.parse().unwrap();
Some(DocumentFileRange {
start: line_number,
end: line_number,
})
}
}
_ => return,
};
eprintln!("");
eprintln!("Usage Instructions:");
eprintln!("- <Up> / <Down> to cycle through history");
eprintln!("");
while let Ok(line) = rl.readline("Query >> ") {
if line.is_empty() {
break;
}
let config = match make_highlight_config("c_sharp", &line) {
Some(config) => config,
None => {
eprintln!("=> Error when constructing configuration, probably invalid query.");
continue;
}
};
let document = match lsif_index_with_config(&contents, &config) {
Ok(document) => document,
Err(err) => {
eprintln!("Failed to index document: {:?}", err);
return;
}
};
eprintln!("{}", dump_document_range(&document, &contents, &range));
}
}

22
internal/gosyntect/gosyntect.go
View File

@ -102,8 +102,14 @@ type response struct {
Code string `json:"code"`
}
// Make sure all names are lowercase here, since they are normalized
var enryLanguageMappings = map[string]string{
"c#": "c_sharp",
}
var supportedFiletypes = map[string]struct{}{
"go": {},
"go": {},
"c_sharp": {},
}
// Client represents a client connection to a syntect_server.
@ -113,8 +119,17 @@ type Client struct {
var client = &http.Client{Transport: &nethttp.Transport{}}
func normalizeFiletype(filetype string) string {
normalized := strings.ToLower(filetype)
if mapped, ok := enryLanguageMappings[normalized]; ok {
normalized = mapped
}
return normalized
}
func (c *Client) IsTreesitterSupported(filetype string) bool {
_, contained := supportedFiletypes[strings.ToLower(filetype)]
_, contained := supportedFiletypes[normalizeFiletype(filetype)]
return contained
}
@ -126,6 +141,9 @@ func (c *Client) IsTreesitterSupported(filetype string) bool {
// to be a separate param. But I need to clean up these other deprecated
// options later, so it's OK for the first iteration.
func (c *Client) Highlight(ctx context.Context, q *Query, useTreeSitter bool) (*Response, error) {
// Normalize filetype
q.Filetype = normalizeFiletype(q.Filetype)
if useTreeSitter && !c.IsTreesitterSupported(q.Filetype) {
return nil, errors.New("Not a valid treesitter filetype")
}

2
package.json
View File

@ -170,6 +170,7 @@
"@types/d3-selection": "1.4.1",
"@types/d3-shape": "1.3.2",
"@types/d3-time-format": "3.0.0",
"@types/escape-html": "^1.0.1",
"@types/expect": "24.3.0",
"@types/express": "4.17.11",
"@types/fancy-log": "1.3.1",
@ -374,6 +375,7 @@
"date-fns": "^2.16.1",
"delay": "^4.4.1",
"downshift": "^3.4.8",
"escape-html": "^1.0.3",
"focus-visible": "^5.2.0",
"fzy.js": "^0.4.1",
"got": "^11.5.2",

5
yarn.lock
View File

@ -4767,6 +4767,11 @@
dependencies:
"@types/node" "*"
"@types/escape-html@^1.0.1":
version "1.0.1"
resolved "https://registry.npmjs.org/@types/escape-html/-/escape-html-1.0.1.tgz#b19b4646915f0ae2c306bf984dc0a59c5cfc97ba"
integrity sha512-4mI1FuUUZiuT95fSVqvZxp/ssQK9zsa86S43h9x3zPOSU9BBJ+BfDkXwuaU7BfsD+e7U0/cUUfJFk3iW2M4okA==
"@types/eslint-scope@^3.7.0":
version "3.7.0"
resolved "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.0.tgz#4792816e31119ebd506902a482caec4951fabd86"