From 6187cdea71ce07fcea8a6c2ca301a0ad9597cd93 Mon Sep 17 00:00:00 2001 From: TJ DeVries Date: Wed, 2 Mar 2022 12:02:36 -0500 Subject: [PATCH] syntax-highlighter: Add C# Support (#31811) Add C# support and fix some multi-line bugs for tree-sitter highlighting --- .gitmodules | 4 + client/web/src/lsif/html.ts | 6 +- docker-images/syntax-highlighter/Cargo.lock | 612 +++++++++++++++++- docker-images/syntax-highlighter/Cargo.toml | 3 +- .../sg-syntax/languages/tree-sitter-c_sharp | 1 + .../sg-syntax/queries/c_sharp/highlights.scm | 262 ++++++++ .../crates/sg-syntax/src/lib.rs | 20 +- .../crates/sg-syntax/src/sg_treesitter.rs | 120 +++- .../files/csharp_timed_out_syntect.cs | 59 ++ ..._test__.__csharp_timed_out_syntect.cs.snap | 417 ++++++++++++ ...eesitter__test__highlight_csharp_file.snap | 11 + .../src/bin/lsif-snapshot.rs | 3 +- .../src/bin/lsif-syntax-repl.rs | 83 +++ internal/gosyntect/gosyntect.go | 22 +- package.json | 2 + yarn.lock | 5 + 16 files changed, 1600 insertions(+), 30 deletions(-) create mode 160000 docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-c_sharp create mode 100644 docker-images/syntax-highlighter/crates/sg-syntax/queries/c_sharp/highlights.scm create mode 100644 docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/files/csharp_timed_out_syntect.cs create mode 100644 docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__.__csharp_timed_out_syntect.cs.snap create mode 100644 docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__highlight_csharp_file.snap create mode 100644 docker-images/syntax-highlighter/src/bin/lsif-syntax-repl.rs diff --git a/.gitmodules b/.gitmodules index 2efea72312b..85cad4f8b1b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,7 @@ path = docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-sql url = https://github.com/m-novikov/tree-sitter-sql shallow = true +[submodule "tree-sitter-c_sharp"] + path = docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-c_sharp + url = https://github.com/tree-sitter/tree-sitter-c-sharp + shallow = true diff --git a/client/web/src/lsif/html.ts b/client/web/src/lsif/html.ts index d468226aff0..85b8fbaf69a 100644 --- a/client/web/src/lsif/html.ts +++ b/client/web/src/lsif/html.ts @@ -1,3 +1,5 @@ +import escape from 'escape-html' + import { JsonDocument, Occurrence, SyntaxKind } from './lsif-typed' class HtmlBuilder { @@ -14,7 +16,7 @@ class HtmlBuilder { } public element(element: string, attributes: string, value: string): void { this.openTag(element + ' ' + attributes) - this.raw(value) + this.raw(escape(value)) this.closeTag(element) } public raw(html: string): void { @@ -103,7 +105,7 @@ export function render(lsif_json: string, content: string): string { // At this time, the syntax highlighter only returns non-overlapping ranges so this // is OK. if (start.line !== end.line) { - html.plaintext(line.slice(start.character)) + highlightSlice(html, occ.kind, line.slice(start.character)) closeLine(html) // Move to the next line diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index 7a94d4389af..25fecc0e4c3 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -90,6 +90,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + [[package]] name = "base64" version = "0.13.0" @@ -117,6 +123,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "bytes" version = "1.1.0" @@ -135,6 +147,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clipboard-win" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3e1238132dc01f081e1cbb9dace14e5ef4c3a51ee244bd982275fb514605db" +dependencies = [ + "error-code", + "str-buf", + "winapi", +] + [[package]] name = "console" version = "0.15.0" @@ -148,6 +171,23 @@ dependencies = [ "winapi", ] +[[package]] +name = "const_fn" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" + +[[package]] +name = "cookie" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d" +dependencies = [ + "percent-encoding", + "time 0.2.27", + "version_check", +] + [[package]] name = "cookie" version = "0.16.0" @@ -155,7 +195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" dependencies = [ "percent-encoding", - "time", + "time 0.3.7", "version_check", ] @@ -217,6 +257,33 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "either" version = "1.6.1" @@ -238,6 +305,43 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "error-code" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" +dependencies = [ + "libc", + "str-buf", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -247,6 +351,17 @@ dependencies = [ "instant", ] +[[package]] +name = "fd-lock" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02ecad9808e0596f8956d14f7fa868f996290bd01c8d7329d6e5bc2bb76adf8f" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys", +] + [[package]] name = "figment" version = "0.10.6" @@ -287,6 +402,7 @@ checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -309,12 +425,34 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" +[[package]] +name = "futures-executor" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2" +[[package]] +name = "futures-macro" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.19" @@ -336,6 +474,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -506,6 +645,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-lifetimes" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec58677acfea8a15352d42fc87d11d63596ade9239e0a7c9352914417515dbe6" + [[package]] name = "itoa" version = "0.4.8" @@ -532,9 +677,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.115" +version = "0.2.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a8d982fa7a96a000f6ec4cfe966de9703eccde29750df2bb8949da91b0e818d" +checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" [[package]] name = "libloading" @@ -561,6 +706,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "linux-raw-sys" +version = "0.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5284f00d480e1c39af34e72f8ad60b94f47007e3481cd3b731c1d67190ddc7b7" + [[package]] name = "lock_api" version = "0.4.5" @@ -609,6 +760,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -667,6 +827,28 @@ dependencies = [ "version_check", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + +[[package]] +name = "nix" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +dependencies = [ + "bitflags", + "cc", + "cfg-if", + "libc", + "memoffset", +] + [[package]] name = "ntapi" version = "0.3.6" @@ -820,7 +1002,7 @@ dependencies = [ "indexmap", "line-wrap", "serde", - "time", + "time 0.3.7", "xml-rs", ] @@ -842,6 +1024,12 @@ dependencies = [ "output_vt100", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.36" @@ -879,6 +1067,16 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.4" @@ -928,6 +1126,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom", + "redox_syscall", +] + [[package]] name = "ref-cast" version = "1.0.6" @@ -983,6 +1191,45 @@ dependencies = [ "winapi", ] +[[package]] +name = "rocket" +version = "0.5.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a71c18c42a0eb15bf3816831caf0dad11e7966f2a41aaf486a701979c4dd1f2" +dependencies = [ + "async-stream", + "async-trait", + "atomic", + "atty", + "binascii", + "bytes", + "either", + "figment", + "futures", + "indexmap", + "log", + "memchr", + "multer", + "num_cpus", + "parking_lot", + "pin-project-lite", + "rand", + "ref-cast", + "rocket_codegen 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rocket_http 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", + "serde_json", + "state", + "tempfile", + "time 0.2.27", + "tokio", + "tokio-stream", + "tokio-util", + "ubyte", + "version_check", + "yansi", +] + [[package]] name = "rocket" version = "0.5.0-rc.1" @@ -1006,13 +1253,13 @@ dependencies = [ "pin-project-lite", "rand", "ref-cast", - "rocket_codegen", - "rocket_http", + "rocket_codegen 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)", + "rocket_http 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)", "serde", "serde_json", "state", "tempfile", - "time", + "time 0.3.7", "tokio", "tokio-stream", "tokio-util", @@ -1021,6 +1268,22 @@ dependencies = [ "yansi", ] +[[package]] +name = "rocket_codegen" +version = "0.5.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66f5fa462f7eb958bba8710c17c5d774bbbd59809fa76fb1957af7e545aea8bb" +dependencies = [ + "devise", + "glob", + "indexmap", + "proc-macro2", + "quote", + "rocket_http 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "syn", + "unicode-xid", +] + [[package]] name = "rocket_codegen" version = "0.5.0-rc.1" @@ -1031,17 +1294,45 @@ dependencies = [ "indexmap", "proc-macro2", "quote", - "rocket_http", + "rocket_http 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)", "syn", "unicode-xid", ] +[[package]] +name = "rocket_http" +version = "0.5.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c8b7d512d2fcac2316ebe590cde67573844b99e6cc9ee0f53375fa16e25ebd" +dependencies = [ + "cookie 0.15.1", + "either", + "http", + "hyper", + "indexmap", + "log", + "memchr", + "mime", + "parking_lot", + "pear", + "percent-encoding", + "pin-project-lite", + "ref-cast", + "serde", + "smallvec", + "stable-pattern", + "state", + "time 0.2.27", + "tokio", + "uncased", +] + [[package]] name = "rocket_http" version = "0.5.0-rc.1" source = "git+https://github.com/SergioBenitez/Rocket#8cae077ba1d54b92cdef3e171a730b819d5eeb8e" dependencies = [ - "cookie", + "cookie 0.16.0", "either", "http", "hyper", @@ -1056,17 +1347,64 @@ dependencies = [ "smallvec", "stable-pattern", "state", - "time", + "time 0.3.7", "tokio", "uncased", ] +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9466f25b92a648960ac1042fd3baa6b0bf285e60f754d7e5070770c813a177a" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "winapi", +] + [[package]] name = "rustversion" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" +[[package]] +name = "rustyline" +version = "9.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039" +dependencies = [ + "bitflags", + "cfg-if", + "clipboard-win", + "dirs-next", + "fd-lock", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "scopeguard", + "smallvec", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi", +] + [[package]] name = "ryu" version = "1.0.9" @@ -1100,6 +1438,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.136" @@ -1171,7 +1524,7 @@ dependencies = [ "libloading", "paste", "protobuf", - "rocket", + "rocket 0.5.0-rc.1 (git+https://github.com/SergioBenitez/Rocket)", "serde", "serde_json", "sg-lsif", @@ -1181,6 +1534,21 @@ dependencies = [ "tree-sitter-highlight", ] +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sharded-slab" version = "0.1.4" @@ -1242,6 +1610,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + [[package]] name = "state" version = "0.5.2" @@ -1251,6 +1628,61 @@ dependencies = [ "loom", ] +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + +[[package]] +name = "str-buf" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a" + [[package]] name = "syn" version = "1.0.86" @@ -1287,7 +1719,8 @@ dependencies = [ name = "syntect_server" version = "1.0.1" dependencies = [ - "rocket", + "rocket 0.5.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustyline", "serde", "serde_json", "sg-syntax", @@ -1347,6 +1780,21 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros 0.1.1", + "version_check", + "winapi", +] + [[package]] name = "time" version = "0.3.7" @@ -1356,7 +1804,17 @@ dependencies = [ "itoa 1.0.1", "libc", "num_threads", - "time-macros", + "time-macros 0.2.3", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", ] [[package]] @@ -1365,6 +1823,19 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25eb0ca3468fc0acc11828786797f6ef9aa1555e4a211a60d64cc8e4d1be47d6" +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + [[package]] name = "tokio" version = "1.15.0" @@ -1541,12 +2012,30 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-segmentation" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + [[package]] name = "unicode-xid" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "utf8parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" + [[package]] name = "version_check" version = "0.9.4" @@ -1580,6 +2069,60 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + [[package]] name = "winapi" version = "0.3.9" @@ -1611,6 +2154,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030b7ff91626e57a05ca64a07c481973cbb2db774e4852c9c7ca342408c6a99a" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29277a4435d642f775f63c7d1faeb927adba532886ce0287bd985bffb16b6bca" + +[[package]] +name = "windows_i686_gnu" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145e1989da93956c68d1864f32fb97c8f561a8f89a5125f6a2b7ea75524e4b8" + +[[package]] +name = "windows_i686_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a09e3a0d4753b73019db171c1339cd4362c8c44baf1bcea336235e955954a6" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca64fcb0220d58db4c119e050e7af03c69e6f4f415ef69ec1773d9aab422d5a" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08cabc9f0066848fef4bc6a1c1668e6efce38b661d2aeec75d18d8617eebb5f1" + [[package]] name = "xml-rs" version = "0.8.4" diff --git a/docker-images/syntax-highlighter/Cargo.toml b/docker-images/syntax-highlighter/Cargo.toml index 071a08a8751..bbef2708d4d 100644 --- a/docker-images/syntax-highlighter/Cargo.toml +++ b/docker-images/syntax-highlighter/Cargo.toml @@ -10,10 +10,11 @@ license = "MIT" [dependencies] syntect = { git = "https://github.com/slimsag/syntect" } -rocket = { git = "https://github.com/SergioBenitez/Rocket", features = ["json"] } +rocket = { version = "0.5.0-rc.1", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sg-syntax = { path = "./crates/sg-syntax" } +rustyline = "9.1.2" [workspace] members = [ diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-c_sharp b/docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-c_sharp new file mode 160000 index 00000000000..352a4630c81 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/sg-syntax/languages/tree-sitter-c_sharp @@ -0,0 +1 @@ +Subproject commit 352a4630c81a7a5cbd3bc67327743bd8d38f2dd2 diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/queries/c_sharp/highlights.scm b/docker-images/syntax-highlighter/crates/sg-syntax/queries/c_sharp/highlights.scm new file mode 100644 index 00000000000..36722c7e09e --- /dev/null +++ b/docker-images/syntax-highlighter/crates/sg-syntax/queries/c_sharp/highlights.scm @@ -0,0 +1,262 @@ +(using_directive (identifier) @variable.module) +(qualified_name (identifier) @variable.module) + +;; Methods +(method_declaration (identifier) @type (identifier) @function) + +(invocation_expression + (member_access_expression + (generic_name + (identifier) @method))) + +(invocation_expression + (member_access_expression + name: (identifier) @method)) + +(invocation_expression + function: (conditional_access_expression + (member_binding_expression + name: (identifier) @method))) + +(invocation_expression + (identifier) @method) + +(invocation_expression + function: (generic_name + . (identifier) @method)) + +;; Types +(interface_declaration name: (identifier) @type) +(class_declaration name: (identifier) @type) +(enum_declaration name: (identifier) @type) +(struct_declaration (identifier) @type) +(record_declaration (identifier) @type) +(namespace_declaration name: (identifier) @type) + +(constructor_declaration name: (identifier) @type) + +[ + (implicit_type) + (nullable_type) + (pointer_type) + (function_pointer_type) + (predefined_type) +] @type.builtin + +;; Enum +(enum_member_declaration (identifier) @property.definition) + +;; Literals +[ + (real_literal) + (integer_literal) +] @number + +[ + (character_literal) + (string_literal) + (verbatim_string_literal) + (interpolated_string_text) + (interpolated_verbatim_string_text) + "\"" + "$\"" + "@$\"" + "$@\"" + ] @string + +[ + (boolean_literal) + (null_literal) + (void_keyword) +] @constant.builtin + +;; Comments +(comment) @comment + +;; Tokens +[ + ";" + "." + "," +] @punctuation.delimiter + +[ + "--" + "-" + "-=" + "&" + "&&" + "+" + "++" + "+=" + "<" + "<<" + "=" + "==" + "!" + "!=" + "=>" + ">" + ">>" + "|" + "||" + "?" + "??" + "^" + "~" + "*" + "/" + "%" + ":" +] @operator + +[ + "(" + ")" + "[" + "]" + "{" + "}" +] @punctuation.bracket + +;; Keywords +(modifier) @keyword +(this_expression) @keyword +(escape_sequence) @keyword + +[ + "as" + "base" + "break" + "case" + "catch" + "checked" + "class" + "continue" + "default" + "delegate" + "do" + "else" + "enum" + "event" + "explicit" + "finally" + "for" + "foreach" + "goto" + "if" + "implicit" + "interface" + "is" + "lock" + "namespace" + "operator" + "params" + "return" + "sizeof" + "stackalloc" + "struct" + "switch" + "throw" + "try" + "typeof" + "unchecked" + "using" + "while" + "new" + "await" + "in" + "yield" + "get" + "set" + "when" + "out" + "ref" + "from" + "where" + "select" + "record" + "init" + "with" + "let" +] @keyword + + +;; Linq +(from_clause (identifier) @variable) +(group_clause) +(order_by_clause) +(select_clause (identifier) @variable) +(query_continuation (identifier) @variable) @keyword + +;; Record +(with_expression + (with_initializer_expression + (simple_assignment_expression + (identifier) @variable))) + +;; Exprs +(binary_expression (identifier) @variable (identifier) @variable) +(binary_expression (identifier)* @variable) +(conditional_expression (identifier) @variable) +(prefix_unary_expression (identifier) @variable) +(postfix_unary_expression (identifier)* @variable) +(assignment_expression (identifier) @variable) +(cast_expression (identifier) @type (identifier) @variable) + +;; Class +(base_list (identifier) @type) +(property_declaration (generic_name)) +(property_declaration + type: (nullable_type) @type + name: (identifier) @variable) +(property_declaration + type: (predefined_type) @type + name: (identifier) @variable) +(property_declaration + type: (identifier) @type + name: (identifier) @variable) + +;; Lambda +(lambda_expression) @variable + +;; Attribute +(attribute) @type + +;; Parameter +(parameter + type: (identifier) @type + name: (identifier) @variable.parameter) +(parameter (identifier) @variable.parameter) +(parameter_modifier) @keyword + +;; Typeof +(type_of_expression (identifier) @type) + +;; Variable +(variable_declaration (identifier) @type) +(variable_declarator (identifier) @variable) + +;; Return +(return_statement (identifier) @variable) +(yield_statement (identifier) @variable) + +;; Type +(generic_name (identifier) @type) +(type_parameter (identifier) @property.definition) +(type_argument_list (identifier) @type) + +;; Type constraints +(type_parameter_constraints_clause (identifier) @property.definition) +(type_constraint (identifier) @type) + +;; Exception +(catch_declaration (identifier) @type (identifier) @variable) +(catch_declaration (identifier) @type) + +;; Switch +(switch_statement (identifier) @variable) +(switch_expression (identifier) @variable) + +;; Lock statement +(lock_statement (identifier) @variable) diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/src/lib.rs b/docker-images/syntax-highlighter/crates/sg-syntax/src/lib.rs index 4e0af1cd52a..ad683ec7cc7 100644 --- a/docker-images/syntax-highlighter/crates/sg-syntax/src/lib.rs +++ b/docker-images/syntax-highlighter/crates/sg-syntax/src/lib.rs @@ -10,8 +10,12 @@ use syntect::{ mod sg_treesitter; pub use sg_treesitter::dump_document; +pub use sg_treesitter::dump_document_range; pub use sg_treesitter::index_language as lsif_index; +pub use sg_treesitter::index_language_with_config as lsif_index_with_config; pub use sg_treesitter::lsif_highlight; +pub use sg_treesitter::make_highlight_config; +pub use sg_treesitter::FileRange as DocumentFileRange; pub use sg_treesitter::PackedRange as LsifPackedRange; mod sg_syntect; @@ -60,7 +64,21 @@ pub struct SourcegraphQuery { pub code: String, } -pub fn determine_language<'a>( +pub fn determine_filetype(q: &SourcegraphQuery) -> String { + let filetype = SYNTAX_SET.with(|syntax_set| match determine_language(q, syntax_set) { + Ok(language) => language.name.clone(), + Err(_) => "".to_owned(), + }); + + // We normalize all the filenames here + match filetype.as_str() { + "C#" => "c_sharp", + filetype => filetype, + } + .to_lowercase() +} + +fn determine_language<'a>( q: &SourcegraphQuery, syntax_set: &'a SyntaxSet, ) -> Result<&'a SyntaxReference, JsonValue> { diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs b/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs index 3a83b3f9bfb..1d7ce506a8b 100644 --- a/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs +++ b/docker-images/syntax-highlighter/crates/sg-syntax/src/sg_treesitter.rs @@ -43,7 +43,7 @@ const MATCHES_TO_SYNTAX_KINDS: &[(&str, SyntaxKind)] = &[ ("keyword", SyntaxKind::IdentifierKeyword), ("keyword.function", SyntaxKind::IdentifierKeyword), ("keyword.return", SyntaxKind::IdentifierKeyword), - ("method", SyntaxKind::Identifier), + ("method", SyntaxKind::IdentifierFunction), ("number", SyntaxKind::NumericLiteral), ("operator", SyntaxKind::IdentifierOperator), ("property", SyntaxKind::Identifier), @@ -112,7 +112,7 @@ macro_rules! create_configurations { lazy_static::lazy_static! { static ref CONFIGURATIONS: HashMap<&'static str, HighlightConfiguration> = { - create_configurations!( go, sql ) + create_configurations!( go, sql, c_sharp ) }; } @@ -139,17 +139,46 @@ pub fn lsif_highlight(q: SourcegraphQuery) -> Result { } pub fn index_language(filetype: &str, code: &str) -> Result { - let mut highlighter = TSHighlighter::new(); let lang_config = match CONFIGURATIONS.get(filetype) { Some(lang_config) => lang_config, None => return Err(Error::InvalidLanguage), }; + index_language_with_config(code, &lang_config) +} + +pub fn make_highlight_config(name: &str, highlights: &str) -> Option { + let config = match CONFIGURATIONS.get(name) { + Some(config) => config, + None => return None, + }; + + // Create HighlightConfiguration language + let mut lang = match HighlightConfiguration::new(config.language, highlights, "", "") { + Ok(lang) => lang, + Err(_) => return None, + }; + + // Associate highlights with configuration + let highlight_names = MATCHES_TO_SYNTAX_KINDS + .iter() + .map(|hl| hl.0) + .collect::>(); + lang.configure(&highlight_names); + + Some(lang) +} + +pub fn index_language_with_config( + code: &str, + lang_config: &HighlightConfiguration, +) -> Result { // TODO: We should automatically apply no highlights when we are // in an injected piece of code. // // Unfortunately, that information isn't currently available when // we are iterating in the higlighter. + let mut highlighter = TSHighlighter::new(); let highlights = highlighter.highlight(lang_config, code.as_bytes(), None, |l| { CONFIGURATIONS.get(l) })?; @@ -313,14 +342,34 @@ impl LsifEmitter { } } -pub fn dump_document(doc: Document, source: &str) -> String { +pub fn dump_document(doc: &Document, source: &str) -> String { + dump_document_range(doc, source, &None) +} + +pub struct FileRange { + pub start: usize, + pub end: usize, +} + +pub fn dump_document_range(doc: &Document, source: &str, file_range: &Option) -> String { let mut occurences = doc.get_occurrences().to_owned(); occurences.sort_by_key(|o| PackedRange::from_vec(&o.range)); let mut occurences = VecDeque::from(occurences); let mut result = String::new(); - for (idx, line) in source.lines().enumerate() { + let line_iterator: Box> = match file_range { + Some(range) => Box::new( + source + .lines() + .enumerate() + .skip(range.start - 1) + .take(range.end - range.start + 1), + ), + None => Box::new(source.lines().enumerate()), + }; + + for (idx, line) in line_iterator { result += " "; result += &line.replace("\t", " "); result += "\n"; @@ -335,7 +384,9 @@ pub fn dump_document(doc: Document, source: &str) -> String { continue; } - if range.start_line != idx as i32 { + if range.start_line < idx as i32 { + continue; + } else if range.start_line > idx as i32 { occurences.push_front(occ); break; } @@ -356,13 +407,20 @@ pub fn dump_document(doc: Document, source: &str) -> String { #[cfg(test)] mod test { + use std::{ + fs::{read_dir, File}, + io::Read, + }; + + use crate::determine_filetype; + use super::*; #[test] fn test_highlights_one_comment() -> Result<(), Error> { let src = "// Hello World"; let document = index_language("go", src)?; - insta::assert_snapshot!(dump_document(document, src)); + insta::assert_snapshot!(dump_document(&document, src)); Ok(()) } @@ -378,7 +436,7 @@ func main() { "#; let document = index_language("go", src)?; - insta::assert_snapshot!(dump_document(document, src)); + insta::assert_snapshot!(dump_document(&document, src)); Ok(()) } @@ -393,7 +451,51 @@ SELECT * FROM my_table "#; let document = index_language("go", src)?; - insta::assert_snapshot!(dump_document(document, src)); + insta::assert_snapshot!(dump_document(&document, src)); + + Ok(()) + } + + #[test] + fn test_highlight_csharp_file() -> Result<(), Error> { + let src = "using System;"; + let document = index_language("c_sharp", src)?; + insta::assert_snapshot!(dump_document(&document, src)); + + Ok(()) + } + + #[test] + fn test_all_files() -> Result<(), std::io::Error> { + let dir = read_dir("./src/snapshots/files/")?; + for entry in dir.into_iter() { + let entry = entry?; + let filepath = entry.path(); + let mut file = File::open(&filepath)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + let filetype = &determine_filetype(&SourcegraphQuery { + extension: filepath.extension().unwrap().to_str().unwrap().to_string(), + filepath: filepath.to_str().unwrap().to_string(), + filetype: None, + css: false, + line_length_limit: None, + theme: "".to_string(), + code: contents.clone(), + }); + + println!("Filetype: {filetype}"); + + let document = index_language(filetype, &contents).unwrap(); + insta::assert_snapshot!( + filepath + .to_str() + .unwrap() + .replace("/src/snapshots/files", ""), + dump_document(&document, &contents) + ); + } Ok(()) } diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/files/csharp_timed_out_syntect.cs b/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/files/csharp_timed_out_syntect.cs new file mode 100644 index 00000000000..dc90d358a23 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/files/csharp_timed_out_syntect.cs @@ -0,0 +1,59 @@ +// Original: +// repo:^github\.com/radzenhq/radzen-blazor$ file:^Radzen\.Blazor\.Tests/DataGridTests\.cs + +using AngleSharp.Dom; +using Bunit; +using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Rendering; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Xunit; +namespace Radzen.Blazor. +{ + public class DataGridTests + { + // Css classes tests + [Fact] + public void DataGrid_Renders_CssClass() + { + using var ctx = new TestContext(); + ctx.JSInterop.Mode = JSRuntimeMode.Loose; + ctx.JSInterop.SetupModule("_content/Radzen.Blazor/Radzen.Blazor.js"); + var component = ctx.RenderComponent>(parameterBuilder => + { + parameterBuilder.Add>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } }); + parameterBuilder.Add(p => p.Columns, builder => + { + builder.OpenComponent(0, typeof(RadzenGridColumn)); + builder.AddAttribute(1, "Property", "Id"); + builder.CloseComponent(); + }); + }); + + // Main + Assert.Contains(@$"rz-datatable-scrollable-wrapper", component.Markup); + Assert.Contains(@$"rz-datatable-scrollable-view", component.Markup); + + var component = ctx.RenderComponent>(parameterBuilder => + { + parameterBuilder.Add>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } }); + parameterBuilder.Add(p => p.Columns, builder => + { + builder.OpenComponent(0, typeof(RadzenGridColumn)); + builder.AddAttribute(1, "Property", "Id"); + builder.CloseComponent(); + }); + }); + + var markup = new Regex(@"\s\s+").Replace(component.Markup, "").Trim(); + Assert.Contains(@$"""rz-cell-data"">1", markup); + Assert.Contains(@$"""rz-cell-data"">2", markup); + Assert.Contains(@$"""rz-cell-data"">3", markup); + Assert.Contains(@" +Lorem Ipsum +", component.Markup); + } + } +} diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__.__csharp_timed_out_syntect.cs.snap b/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__.__csharp_timed_out_syntect.cs.snap new file mode 100644 index 00000000000..5b63cfd0115 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__.__csharp_timed_out_syntect.cs.snap @@ -0,0 +1,417 @@ +--- +source: crates/sg-syntax/src/sg_treesitter.rs +assertion_line: 491 +expression: "dump_document(&document, &contents)" + +--- + // Original: +//^^^^^^^^^^^^ Comment + // repo:^github\.com/radzenhq/radzen-blazor$ file:^Radzen\.Blazor\.Tests/DataGridTests\.cs +//^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Comment + + using AngleSharp.Dom; +//^^^^^ IdentifierKeyword +// ^^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^ IdentifierModule +// ^ PunctuationDelimiter + using Bunit; +//^^^^^ IdentifierKeyword +// ^^^^^ IdentifierModule +// ^ PunctuationDelimiter + using Microsoft.AspNetCore.Components; +//^^^^^ IdentifierKeyword +// ^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter + using Microsoft.AspNetCore.Components.Rendering; +//^^^^^ IdentifierKeyword +// ^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter + using System; +//^^^^^ IdentifierKeyword +// ^^^^^^ IdentifierModule +// ^ PunctuationDelimiter + using System.Collections.Generic; +//^^^^^ IdentifierKeyword +// ^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter + using System.Linq; +//^^^^^ IdentifierKeyword +// ^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^ IdentifierModule +// ^ PunctuationDelimiter + using System.Text.RegularExpressions; +//^^^^^ IdentifierKeyword +// ^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^^^^^^^ IdentifierModule +// ^ PunctuationDelimiter + using Xunit; +//^^^^^ IdentifierKeyword +// ^^^^^ IdentifierModule +// ^ PunctuationDelimiter + namespace Radzen.Blazor. +//^^^^^^^^^ IdentifierKeyword +// ^^^^^^ IdentifierModule +// ^ PunctuationDelimiter +// ^^^^^^ IdentifierModule +// ^ PunctuationDelimiter + { + public class DataGridTests +// ^^^^^^ IdentifierKeyword +// ^^^^^ IdentifierKeyword +// ^^^^^^^^^^^^^ IdentifierType + { + // Css classes tests +// ^^^^^^^^^^^^^^^^^^^^ Comment + [Fact] +// ^^^^ IdentifierType + public void DataGrid_Renders_CssClass() +// ^^^^^^ IdentifierKeyword +// ^^^^ IdentifierBuiltin + { + using var ctx = new TestContext(); +// ^^^^^ IdentifierKeyword +// ^^^ IdentifierType +// ^^^ Identifier +// ^ IdentifierOperator +// ^^^ IdentifierKeyword +// ^ PunctuationDelimiter + ctx.JSInterop.Mode = JSRuntimeMode.Loose; +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter +// ^ IdentifierOperator +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter + ctx.JSInterop.SetupModule("_content/Radzen.Blazor/Radzen.Blazor.js"); +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter +// ^^^^^^^^^^^ IdentifierFunction +// ^ StringLiteral +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter + var component = ctx.RenderComponent>(parameterBuilder => +// ^^^ IdentifierType +// ^^^^^^^^^ Identifier +// ^ IdentifierOperator +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^^^^ IdentifierFunction +// ^ IdentifierOperator +// ^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^ IdentifierOperator +// ^^^^^^^^^^^^^^^^^ Identifier +// ^^ IdentifierOperator + { + parameterBuilder.Add>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } }); +// ^ PunctuationDelimiter +// ^^^ IdentifierFunction +// ^ IdentifierOperator +// ^^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^ IdentifierOperator +// ^^ Identifier +// ^^ IdentifierOperator +// ^^ Identifier +// ^ PunctuationDelimiter +// ^^^^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^^^^ Identifier +// ^ IdentifierOperator +// ^ Identifier +// ^ NumericLiteral +// ^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^^^^ Identifier +// ^ IdentifierOperator +// ^ Identifier +// ^ NumericLiteral +// ^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^^^^ Identifier +// ^ IdentifierOperator +// ^ Identifier +// ^ NumericLiteral +// ^ Identifier +// ^ Identifier +// ^ PunctuationDelimiter + parameterBuilder.Add(p => p.Columns, builder => +// ^ PunctuationDelimiter +// ^^^ IdentifierFunction +// ^ IdentifierOperator +// ^^^^^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^ Identifier +// ^^ IdentifierOperator +// ^^ Identifier +// ^ PunctuationDelimiter +// ^^^^^^^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^^^^^^ Identifier +// ^^ IdentifierOperator + { + builder.OpenComponent(0, typeof(RadzenGridColumn)); +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^^ IdentifierFunction +// ^ NumericLiteral +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^^^^ IdentifierKeyword +// ^^^^^^^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^ PunctuationDelimiter + builder.AddAttribute(1, "Property", "Id"); +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^ IdentifierFunction +// ^ NumericLiteral +// ^ PunctuationDelimiter +// ^ Identifier +// ^ StringLiteral +// ^^^^^^^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^ Identifier +// ^ StringLiteral +// ^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter + builder.CloseComponent(); +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^^^ IdentifierFunction +// ^ PunctuationDelimiter + }); +// ^ PunctuationDelimiter + }); +// ^ PunctuationDelimiter + + // Main +// ^^^^^^^ Comment + Assert.Contains(@$"rz-datatable-scrollable-wrapper", component.Markup); +// ^ PunctuationDelimiter +// ^^^^^^^^ IdentifierFunction +// ^^^ StringLiteral +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter + Assert.Contains(@$"rz-datatable-scrollable-view", component.Markup); +// ^ PunctuationDelimiter +// ^^^^^^^^ IdentifierFunction +// ^^^ StringLiteral +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter + + var component = ctx.RenderComponent>(parameterBuilder => +// ^^^ IdentifierType +// ^^^^^^^^^ Identifier +// ^ IdentifierOperator +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^^^^ IdentifierFunction +// ^ IdentifierOperator +// ^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^ IdentifierOperator +// ^^^^^^^^^^^^^^^^^ Identifier +// ^^ IdentifierOperator + { + parameterBuilder.Add>(p => p.Data, new[] { new { Id = 1 }, new { Id = 2 }, new { Id = 3 } }); +// ^ PunctuationDelimiter +// ^^^ IdentifierFunction +// ^ IdentifierOperator +// ^^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^ IdentifierOperator +// ^^ Identifier +// ^^ IdentifierOperator +// ^^ Identifier +// ^ PunctuationDelimiter +// ^^^^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^^^^ Identifier +// ^ IdentifierOperator +// ^ Identifier +// ^ NumericLiteral +// ^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^^^^ Identifier +// ^ IdentifierOperator +// ^ Identifier +// ^ NumericLiteral +// ^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^ IdentifierKeyword +// ^ Identifier +// ^^^^ Identifier +// ^ IdentifierOperator +// ^ Identifier +// ^ NumericLiteral +// ^ Identifier +// ^ Identifier +// ^ PunctuationDelimiter + parameterBuilder.Add(p => p.Columns, builder => +// ^ PunctuationDelimiter +// ^^^ IdentifierFunction +// ^ IdentifierOperator +// ^^^^^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^ Identifier +// ^^ IdentifierOperator +// ^^ Identifier +// ^ PunctuationDelimiter +// ^^^^^^^ Identifier +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^^^^^^ Identifier +// ^^ IdentifierOperator + { + builder.OpenComponent(0, typeof(RadzenGridColumn)); +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^^ IdentifierFunction +// ^ NumericLiteral +// ^ PunctuationDelimiter +// ^ Identifier +// ^^^^^^ IdentifierKeyword +// ^^^^^^^^^^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^^^^^^^ IdentifierType +// ^ IdentifierOperator +// ^ PunctuationDelimiter + builder.AddAttribute(1, "Property", "Id"); +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^ IdentifierFunction +// ^ NumericLiteral +// ^ PunctuationDelimiter +// ^ Identifier +// ^ StringLiteral +// ^^^^^^^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^ Identifier +// ^ StringLiteral +// ^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter + builder.CloseComponent(); +// ^ PunctuationDelimiter +// ^^^^^^^^^^^^^^ IdentifierFunction +// ^ PunctuationDelimiter + }); +// ^ PunctuationDelimiter + }); +// ^ PunctuationDelimiter + + var markup = new Regex(@"\s\s+").Replace(component.Markup, "").Trim(); +// ^^^ IdentifierType +// ^^^^^^ Identifier +// ^ IdentifierOperator +// ^^^ IdentifierKeyword +// ^^^^^^^^ StringLiteral +// ^ PunctuationDelimiter +// ^^^^^^^ IdentifierFunction +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter +// ^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^^^^ IdentifierFunction +// ^ PunctuationDelimiter + Assert.Contains(@$"""rz-cell-data"">1", markup); +// ^ PunctuationDelimiter +// ^^^^^^^^ IdentifierFunction +// ^^^ StringLiteral +// ^^ StringLiteral +// ^^^^^^^^^^^^ StringLiteral +// ^^ StringLiteral +// ^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter + Assert.Contains(@$"""rz-cell-data"">2", markup); +// ^ PunctuationDelimiter +// ^^^^^^^^ IdentifierFunction +// ^^^ StringLiteral +// ^^ StringLiteral +// ^^^^^^^^^^^^ StringLiteral +// ^^ StringLiteral +// ^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter + Assert.Contains(@$"""rz-cell-data"">3", markup); +// ^ PunctuationDelimiter +// ^^^^^^^^ IdentifierFunction +// ^^^ StringLiteral +// ^^ StringLiteral +// ^^^^^^^^^^^^ StringLiteral +// ^^ StringLiteral +// ^^ StringLiteral +// ^ StringLiteral +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter + Assert.Contains(@" +// ^ PunctuationDelimiter +// ^^^^^^^^ IdentifierFunction + Lorem Ipsum + ", component.Markup); +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter +// ^ PunctuationDelimiter + } + } + } + diff --git a/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__highlight_csharp_file.snap b/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__highlight_csharp_file.snap new file mode 100644 index 00000000000..642782c8385 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/sg-syntax/src/snapshots/sg_syntax__sg_treesitter__test__highlight_csharp_file.snap @@ -0,0 +1,11 @@ +--- +source: crates/sg-syntax/src/sg_treesitter.rs +assertion_line: 461 +expression: "dump_document(&document, src)" + +--- + using System; +//^^^^^ IdentifierKeyword +// ^^^^^^ IdentifierModule +// ^ PunctuationDelimiter + diff --git a/docker-images/syntax-highlighter/src/bin/lsif-snapshot.rs b/docker-images/syntax-highlighter/src/bin/lsif-snapshot.rs index 5b2593a0bb8..e2a2f422ae7 100644 --- a/docker-images/syntax-highlighter/src/bin/lsif-snapshot.rs +++ b/docker-images/syntax-highlighter/src/bin/lsif-snapshot.rs @@ -21,8 +21,7 @@ fn main() { } }; - println!("\n\n{}", dump_document(document, &contents)); - // println!("{}", dump_document()) + println!("\n\n{}", dump_document(&document, &contents)); } else { panic!("Must pass a filepath"); } diff --git a/docker-images/syntax-highlighter/src/bin/lsif-syntax-repl.rs b/docker-images/syntax-highlighter/src/bin/lsif-syntax-repl.rs new file mode 100644 index 00000000000..14110720b5a --- /dev/null +++ b/docker-images/syntax-highlighter/src/bin/lsif-syntax-repl.rs @@ -0,0 +1,83 @@ +use std::fs; + +use rustyline::config::Configurer; +use rustyline::Config; +use sg_syntax::dump_document_range; +use sg_syntax::lsif_index_with_config; +use sg_syntax::make_highlight_config; +use sg_syntax::DocumentFileRange; + +fn main() { + println!("========================================"); + println!(" Welcome to lsif-syntax-repl"); + println!("========================================"); + + let contents = if let Some(path) = std::env::args().nth(1) { + match fs::read_to_string(&path) { + Ok(contents) => contents, + Err(err) => { + eprintln!("Failed to read path: {:?}. {}", path, err); + return; + } + } + } else { + let mut rl = rustyline::Editor::<()>::new(); + match rl.readline("Contents: ") { + Ok(contents) => contents, + Err(err) => { + eprintln!("Failed to read path: {err}"); + return; + } + } + }; + + let mut config = Config::builder(); + config.set_max_history_size(100); + config.set_auto_add_history(true); + + let mut rl = rustyline::Editor::<()>::with_config(config.build()); + + let range: Option = match rl.readline("Range (Optional, 1-Indexed): ") { + Ok(line) => { + if line.is_empty() { + None + } else { + let line_number = line.parse().unwrap(); + Some(DocumentFileRange { + start: line_number, + end: line_number, + }) + } + } + _ => return, + }; + + eprintln!(""); + eprintln!("Usage Instructions:"); + eprintln!("- / to cycle through history"); + eprintln!(""); + + while let Ok(line) = rl.readline("Query >> ") { + if line.is_empty() { + break; + } + + let config = match make_highlight_config("c_sharp", &line) { + Some(config) => config, + None => { + eprintln!("=> Error when constructing configuration, probably invalid query."); + continue; + } + }; + + let document = match lsif_index_with_config(&contents, &config) { + Ok(document) => document, + Err(err) => { + eprintln!("Failed to index document: {:?}", err); + return; + } + }; + + eprintln!("{}", dump_document_range(&document, &contents, &range)); + } +} diff --git a/internal/gosyntect/gosyntect.go b/internal/gosyntect/gosyntect.go index 9a5679cd205..e9b7b628509 100644 --- a/internal/gosyntect/gosyntect.go +++ b/internal/gosyntect/gosyntect.go @@ -102,8 +102,14 @@ type response struct { Code string `json:"code"` } +// Make sure all names are lowercase here, since they are normalized +var enryLanguageMappings = map[string]string{ + "c#": "c_sharp", +} + var supportedFiletypes = map[string]struct{}{ - "go": {}, + "go": {}, + "c_sharp": {}, } // Client represents a client connection to a syntect_server. @@ -113,8 +119,17 @@ type Client struct { var client = &http.Client{Transport: &nethttp.Transport{}} +func normalizeFiletype(filetype string) string { + normalized := strings.ToLower(filetype) + if mapped, ok := enryLanguageMappings[normalized]; ok { + normalized = mapped + } + + return normalized +} + func (c *Client) IsTreesitterSupported(filetype string) bool { - _, contained := supportedFiletypes[strings.ToLower(filetype)] + _, contained := supportedFiletypes[normalizeFiletype(filetype)] return contained } @@ -126,6 +141,9 @@ func (c *Client) IsTreesitterSupported(filetype string) bool { // to be a separate param. But I need to clean up these other deprecated // options later, so it's OK for the first iteration. func (c *Client) Highlight(ctx context.Context, q *Query, useTreeSitter bool) (*Response, error) { + // Normalize filetype + q.Filetype = normalizeFiletype(q.Filetype) + if useTreeSitter && !c.IsTreesitterSupported(q.Filetype) { return nil, errors.New("Not a valid treesitter filetype") } diff --git a/package.json b/package.json index c5a98880f5b..8e22d4db00c 100644 --- a/package.json +++ b/package.json @@ -170,6 +170,7 @@ "@types/d3-selection": "1.4.1", "@types/d3-shape": "1.3.2", "@types/d3-time-format": "3.0.0", + "@types/escape-html": "^1.0.1", "@types/expect": "24.3.0", "@types/express": "4.17.11", "@types/fancy-log": "1.3.1", @@ -374,6 +375,7 @@ "date-fns": "^2.16.1", "delay": "^4.4.1", "downshift": "^3.4.8", + "escape-html": "^1.0.3", "focus-visible": "^5.2.0", "fzy.js": "^0.4.1", "got": "^11.5.2", diff --git a/yarn.lock b/yarn.lock index af4061ae0a2..be4f581fbe1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4767,6 +4767,11 @@ dependencies: "@types/node" "*" +"@types/escape-html@^1.0.1": + version "1.0.1" + resolved "https://registry.npmjs.org/@types/escape-html/-/escape-html-1.0.1.tgz#b19b4646915f0ae2c306bf984dc0a59c5cfc97ba" + integrity sha512-4mI1FuUUZiuT95fSVqvZxp/ssQK9zsa86S43h9x3zPOSU9BBJ+BfDkXwuaU7BfsD+e7U0/cUUfJFk3iW2M4okA== + "@types/eslint-scope@^3.7.0": version "3.7.0" resolved "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.0.tgz#4792816e31119ebd506902a482caec4951fabd86"