From dfa60d6c9b77171d767fabc9d517aec5efcae492 Mon Sep 17 00:00:00 2001 From: Anton Sviridov Date: Thu, 6 Jun 2024 19:33:58 +0100 Subject: [PATCH] Syntactic Indexing: add TAR archive indexing mode to scip-syntax CLI (#63097) Fixes GRAPH-651 Fixes GRAPH-650 New features: - `index tar ` allows indexing tar archives when `input` is a file, and `stdin` when `input` is literal string `-` - **BREAKING: Instead of flags controlling the type of input source, we now have subcommands: `index files`, `index workspace`, `index tar`** Refactoring: - Tests were improved, with helper functions broken down into composable pieces, and producing 1 snapshot per test, making it easier to manage - Closures were removed from the indexing code, instead replaced with functions. - Calls to `.unwrap` were replaced with better error handling code - Path canonicalisation was replaced with path cleanup + absolutisation - to avoid following symlinks (which is what `canonicalize` would do) Most of the refactoring was triggered by the changes required to add more tests. ## Test plan - New integration tests --------- Co-authored-by: Christoph Hegemann --- .../syntax-highlighter/Cargo.Bazel.lock | 1130 ++++++++++++++--- docker-images/syntax-highlighter/Cargo.lock | 169 ++- docker-images/syntax-highlighter/Cargo.toml | 1 + .../crates/scip-syntax/BUILD.bazel | 1 + .../crates/scip-syntax/Cargo.toml | 5 + .../crates/scip-syntax/src/index.rs | 226 +++- .../crates/scip-syntax/src/main.rs | 189 +-- .../scip-syntax/testdata/package-info.java | 4 + .../scip-syntax/tests/integration_test.rs | 293 ++++- ...ntegration_test__java_files_indexing.snap} | 14 +- ...egration_test__java_tar_file_indexing.snap | 86 ++ ...ration_test__java_tar_stream_indexing.snap | 86 ++ ...gration_test__java_workspace_indexing.snap | 86 ++ 13 files changed, 1918 insertions(+), 372 deletions(-) create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/testdata/package-info.java rename docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/{integration_test__globals.java.snap => integration_test__java_files_indexing.snap} (93%) create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_file_indexing.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_stream_indexing.snap create mode 100644 docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_workspace_indexing.snap diff --git a/docker-images/syntax-highlighter/Cargo.Bazel.lock b/docker-images/syntax-highlighter/Cargo.Bazel.lock index 64427a1ae96..1e90448f6bc 100644 --- a/docker-images/syntax-highlighter/Cargo.Bazel.lock +++ b/docker-images/syntax-highlighter/Cargo.Bazel.lock @@ -1,5 +1,5 @@ { - "checksum": "296b539d4c595a40878a6f2db34cde2485d34792507243386f1f540fee4ab337", + "checksum": "3cd06aa5fd5108cee2e6e19b9e6809923b6c23e0bd51d36717b4c016dd136a8c", "crates": { "addr2line 0.20.0": { "name": "addr2line", @@ -832,7 +832,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -928,7 +928,7 @@ "target": "cfg_if" }, { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" }, { @@ -1433,7 +1433,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ] @@ -2092,7 +2092,7 @@ "target": "lazy_static" }, { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" }, { @@ -3012,7 +3012,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -3262,13 +3262,13 @@ }, "license": "Apache-2.0 OR MIT" }, - "errno 0.3.2": { + "errno 0.3.9": { "name": "errno", - "version": "0.3.2", + "version": "0.3.9", "repository": { "Http": { - "url": "https://static.crates.io/crates/errno/0.3.2/download", - "sha256": "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" + "url": "https://static.crates.io/crates/errno/0.3.9/download", + "sha256": "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" } }, "targets": [ @@ -3287,112 +3287,46 @@ "compile_data_glob": [ "**" ], + "crate_features": { + "common": [ + "std" + ], + "selects": {} + }, "deps": { "common": [], "selects": { - "cfg(target_os = \"dragonfly\")": [ - { - "id": "errno-dragonfly 0.1.2", - "target": "errno_dragonfly" - } - ], "cfg(target_os = \"hermit\")": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], "cfg(target_os = \"wasi\")": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], "cfg(windows)": [ { - "id": "windows-sys 0.48.0", + "id": "windows-sys 0.52.0", "target": "windows_sys" } ] } }, "edition": "2018", - "version": "0.3.2" + "version": "0.3.9" }, "license": "MIT OR Apache-2.0" }, - "errno-dragonfly 0.1.2": { - "name": "errno-dragonfly", - "version": "0.1.2", - "repository": { - "Http": { - "url": "https://static.crates.io/crates/errno-dragonfly/0.1.2/download", - "sha256": "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" - } - }, - "targets": [ - { - "Library": { - "crate_name": "errno_dragonfly", - "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs", - "srcs": [ - "**/*.rs" - ] - } - } - ], - "library_target_name": "errno_dragonfly", - "common_attrs": { - "compile_data_glob": [ - "**" - ], - "deps": { - "common": [ - { - "id": "errno-dragonfly 0.1.2", - "target": "build_script_build" - }, - { - "id": "libc 0.2.147", - "target": "libc" - } - ], - "selects": {} - }, - "edition": "2018", - "version": "0.1.2" - }, - "build_script_attrs": { - "data_glob": [ - "**" - ], - "deps": { - "common": [ - { - "id": "cc 1.0.83", - "target": "cc" - } - ], - "selects": {} - } - }, - "license": "MIT" - }, "error-code 2.3.1": { "name": "error-code", "version": "2.3.1", @@ -3421,7 +3355,7 @@ "deps": { "common": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" }, { @@ -3436,13 +3370,13 @@ }, "license": "BSL-1.0" }, - "fastrand 2.0.0": { + "fastrand 2.1.0": { "name": "fastrand", - "version": "2.0.0", + "version": "2.1.0", "repository": { "Http": { - "url": "https://static.crates.io/crates/fastrand/2.0.0/download", - "sha256": "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + "url": "https://static.crates.io/crates/fastrand/2.1.0/download", + "sha256": "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" } }, "targets": [ @@ -3470,7 +3404,7 @@ "selects": {} }, "edition": "2018", - "version": "2.0.0" + "version": "2.1.0" }, "license": "Apache-2.0 OR MIT" }, @@ -3509,7 +3443,7 @@ "selects": { "cfg(unix)": [ { - "id": "rustix 0.38.8", + "id": "rustix 0.38.34", "target": "rustix" } ], @@ -3620,6 +3554,64 @@ }, "license": "MIT OR Apache-2.0" }, + "filetime 0.2.23": { + "name": "filetime", + "version": "0.2.23", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/filetime/0.2.23/download", + "sha256": "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" + } + }, + "targets": [ + { + "Library": { + "crate_name": "filetime", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "filetime", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "cfg-if 1.0.0", + "target": "cfg_if" + } + ], + "selects": { + "cfg(target_os = \"redox\")": [ + { + "id": "redox_syscall 0.4.1", + "target": "syscall" + } + ], + "cfg(unix)": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(windows)": [ + { + "id": "windows-sys 0.52.0", + "target": "windows_sys" + } + ] + } + }, + "edition": "2018", + "version": "0.2.23" + }, + "license": "MIT/Apache-2.0" + }, "flate2 1.0.27": { "name": "flate2", "version": "1.0.27", @@ -4270,7 +4262,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -4357,7 +4349,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ] @@ -4719,7 +4711,7 @@ "deps": { "common": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -5478,7 +5470,7 @@ "selects": { "cfg(not(any(windows, target_os = \"hermit\", target_os = \"unknown\")))": [ { - "id": "rustix 0.38.8", + "id": "rustix 0.38.34", "target": "rustix" } ], @@ -5694,13 +5686,13 @@ }, "license": "MIT/Apache-2.0" }, - "libc 0.2.147": { + "libc 0.2.155": { "name": "libc", - "version": "0.2.147", + "version": "0.2.155", "repository": { "Http": { - "url": "https://static.crates.io/crates/libc/0.2.147/download", - "sha256": "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + "url": "https://static.crates.io/crates/libc/0.2.155/download", + "sha256": "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" } }, "targets": [ @@ -5832,14 +5824,14 @@ "deps": { "common": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "build_script_build" } ], "selects": {} }, "edition": "2015", - "version": "0.2.147" + "version": "0.2.155" }, "build_script_attrs": { "data_glob": [ @@ -5917,13 +5909,13 @@ }, "license": "MIT/Apache-2.0" }, - "linux-raw-sys 0.4.5": { + "linux-raw-sys 0.4.14": { "name": "linux-raw-sys", - "version": "0.4.5", + "version": "0.4.14", "repository": { "Http": { - "url": "https://static.crates.io/crates/linux-raw-sys/0.4.5/download", - "sha256": "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" + "url": "https://static.crates.io/crates/linux-raw-sys/0.4.14/download", + "sha256": "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" } }, "targets": [ @@ -5950,30 +5942,50 @@ ], "selects": { "aarch64-unknown-linux-gnu": [ - "errno" + "elf", + "errno", + "std" ], "aarch64-unknown-nixos-gnu": [ - "errno" + "elf", + "errno", + "std" ], "arm-unknown-linux-gnueabi": [ - "errno" + "elf", + "errno", + "std" ], "armv7-unknown-linux-gnueabi": [ - "errno" + "elf", + "errno", + "std" ], "i686-unknown-linux-gnu": [ - "errno" + "elf", + "errno", + "std" + ], + "powerpc-unknown-linux-gnu": [ + "std" + ], + "s390x-unknown-linux-gnu": [ + "std" ], "x86_64-unknown-linux-gnu": [ - "errno" + "elf", + "errno", + "std" ], "x86_64-unknown-nixos-gnu": [ - "errno" + "elf", + "errno", + "std" ] } }, "edition": "2021", - "version": "0.4.5" + "version": "0.4.14" }, "license": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" }, @@ -6497,7 +6509,7 @@ "selects": { "cfg(target_os = \"wasi\")": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" }, { @@ -6507,7 +6519,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -6714,7 +6726,7 @@ "target": "cfg_if" }, { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -6907,7 +6919,7 @@ "selects": { "cfg(not(windows))": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -7082,7 +7094,7 @@ "selects": { "cfg(windows)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ] @@ -7363,7 +7375,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -7438,6 +7450,36 @@ }, "license": "MIT OR Apache-2.0" }, + "path-clean 1.0.1": { + "name": "path-clean", + "version": "1.0.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/path-clean/1.0.1/download", + "sha256": "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" + } + }, + "targets": [ + { + "Library": { + "crate_name": "path_clean", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "path_clean", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.0.1" + }, + "license": "MIT OR Apache-2.0" + }, "pear 0.2.7": { "name": "pear", "version": "0.2.7", @@ -8701,7 +8743,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ] @@ -8999,6 +9041,45 @@ }, "license": "MIT" }, + "redox_syscall 0.4.1": { + "name": "redox_syscall", + "version": "0.4.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/redox_syscall/0.4.1/download", + "sha256": "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" + } + }, + "targets": [ + { + "Library": { + "crate_name": "syscall", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "syscall", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "bitflags 1.3.2", + "target": "bitflags" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "0.4.1" + }, + "license": "MIT" + }, "redox_users 0.4.3": { "name": "redox_users", "version": "0.4.3", @@ -9565,7 +9646,7 @@ "target": "state" }, { - "id": "tempfile 3.8.0", + "id": "tempfile 3.10.1", "target": "tempfile" }, { @@ -9882,13 +9963,13 @@ }, "license": "Apache-2.0/MIT" }, - "rustix 0.38.8": { + "rustix 0.38.34": { "name": "rustix", - "version": "0.38.8", + "version": "0.38.34", "repository": { "Http": { - "url": "https://static.crates.io/crates/rustix/0.38.8/download", - "sha256": "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" + "url": "https://static.crates.io/crates/rustix/0.38.34/download", + "sha256": "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" } }, "targets": [ @@ -9918,8 +9999,10 @@ ], "crate_features": { "common": [ + "alloc", "default", "libc", + "libc-extra-traits", "libc_errno", "std", "use-libc", @@ -10050,58 +10133,58 @@ "target": "bitflags" }, { - "id": "rustix 0.38.8", + "id": "rustix 0.38.34", "target": "build_script_build" } ], "selects": { "cfg(all(any(target_os = \"android\", target_os = \"linux\"), any(rustix_use_libc, miri, not(all(target_os = \"linux\", target_endian = \"little\", any(target_arch = \"arm\", all(target_arch = \"aarch64\", target_pointer_width = \"64\"), target_arch = \"riscv64\", all(rustix_use_experimental_asm, target_arch = \"powerpc64\"), all(rustix_use_experimental_asm, target_arch = \"mips\"), all(rustix_use_experimental_asm, target_arch = \"mips32r6\"), all(rustix_use_experimental_asm, target_arch = \"mips64\"), all(rustix_use_experimental_asm, target_arch = \"mips64r6\"), target_arch = \"x86\", all(target_arch = \"x86_64\", target_pointer_width = \"64\")))))))": [ { - "id": "linux-raw-sys 0.4.5", + "id": "linux-raw-sys 0.4.14", "target": "linux_raw_sys" } ], "cfg(all(not(rustix_use_libc), not(miri), target_os = \"linux\", target_endian = \"little\", any(target_arch = \"arm\", all(target_arch = \"aarch64\", target_pointer_width = \"64\"), target_arch = \"riscv64\", all(rustix_use_experimental_asm, target_arch = \"powerpc64\"), all(rustix_use_experimental_asm, target_arch = \"mips\"), all(rustix_use_experimental_asm, target_arch = \"mips32r6\"), all(rustix_use_experimental_asm, target_arch = \"mips64\"), all(rustix_use_experimental_asm, target_arch = \"mips64r6\"), target_arch = \"x86\", all(target_arch = \"x86_64\", target_pointer_width = \"64\"))))": [ { - "id": "errno 0.3.2", + "id": "errno 0.3.9", "target": "errno", "alias": "libc_errno" }, { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" }, { - "id": "linux-raw-sys 0.4.5", + "id": "linux-raw-sys 0.4.14", "target": "linux_raw_sys" } ], "cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \"linux\", target_endian = \"little\", any(target_arch = \"arm\", all(target_arch = \"aarch64\", target_pointer_width = \"64\"), target_arch = \"riscv64\", all(rustix_use_experimental_asm, target_arch = \"powerpc64\"), all(rustix_use_experimental_asm, target_arch = \"mips\"), all(rustix_use_experimental_asm, target_arch = \"mips32r6\"), all(rustix_use_experimental_asm, target_arch = \"mips64\"), all(rustix_use_experimental_asm, target_arch = \"mips64r6\"), target_arch = \"x86\", all(target_arch = \"x86_64\", target_pointer_width = \"64\")))))))": [ { - "id": "errno 0.3.2", + "id": "errno 0.3.9", "target": "errno", "alias": "libc_errno" }, { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], "cfg(windows)": [ { - "id": "errno 0.3.2", + "id": "errno 0.3.9", "target": "errno", "alias": "libc_errno" }, { - "id": "windows-sys 0.48.0", + "id": "windows-sys 0.52.0", "target": "windows_sys" } ] } }, "edition": "2021", - "version": "0.38.8" + "version": "0.38.34" }, "build_script_attrs": { "data_glob": [ @@ -10215,7 +10298,7 @@ "target": "fd_lock" }, { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" }, { @@ -10472,6 +10555,10 @@ "id": "lazy_static 1.4.0", "target": "lazy_static" }, + { + "id": "path-clean 1.0.1", + "target": "path_clean" + }, { "id": "predicates 3.0.4", "target": "predicates" @@ -10496,6 +10583,10 @@ "id": "string-interner 0.14.0", "target": "string_interner" }, + { + "id": "tar 0.4.40", + "target": "tar" + }, { "id": "walkdir 2.3.3", "target": "walkdir" @@ -10503,6 +10594,15 @@ ], "selects": {} }, + "deps_dev": { + "common": [ + { + "id": "tempfile 3.10.1", + "target": "tempfile" + } + ], + "selects": {} + }, "edition": "2021", "proc_macro_deps": { "common": [ @@ -10903,7 +11003,7 @@ "deps": { "common": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -11087,7 +11187,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -11140,7 +11240,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ], @@ -11769,7 +11869,7 @@ "target": "rocket" }, { - "id": "rustix 0.38.8", + "id": "rustix 0.38.34", "target": "rustix" }, { @@ -11847,13 +11947,70 @@ }, "license": "MIT" }, - "tempfile 3.8.0": { - "name": "tempfile", - "version": "3.8.0", + "tar 0.4.40": { + "name": "tar", + "version": "0.4.40", "repository": { "Http": { - "url": "https://static.crates.io/crates/tempfile/3.8.0/download", - "sha256": "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" + "url": "https://static.crates.io/crates/tar/0.4.40/download", + "sha256": "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" + } + }, + "targets": [ + { + "Library": { + "crate_name": "tar", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "tar", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "xattr" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "filetime 0.2.23", + "target": "filetime" + } + ], + "selects": { + "cfg(unix)": [ + { + "id": "libc 0.2.155", + "target": "libc" + }, + { + "id": "xattr 1.3.1", + "target": "xattr" + } + ] + } + }, + "edition": "2018", + "version": "0.4.40" + }, + "license": "MIT/Apache-2.0" + }, + "tempfile 3.10.1": { + "name": "tempfile", + "version": "3.10.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/tempfile/3.10.1/download", + "sha256": "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" } }, "targets": [ @@ -11879,33 +12036,27 @@ "target": "cfg_if" }, { - "id": "fastrand 2.0.0", + "id": "fastrand 2.1.0", "target": "fastrand" } ], "selects": { "cfg(any(unix, target_os = \"wasi\"))": [ { - "id": "rustix 0.38.8", + "id": "rustix 0.38.34", "target": "rustix" } ], - "cfg(target_os = \"redox\")": [ - { - "id": "redox_syscall 0.3.5", - "target": "syscall" - } - ], "cfg(windows)": [ { - "id": "windows-sys 0.48.0", + "id": "windows-sys 0.52.0", "target": "windows_sys" } ] } }, - "edition": "2018", - "version": "3.8.0" + "edition": "2021", + "version": "3.10.1" }, "license": "MIT OR Apache-2.0" }, @@ -12403,7 +12554,7 @@ ], "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" }, { @@ -15224,7 +15375,7 @@ "selects": { "cfg(unix)": [ { - "id": "libc 0.2.147", + "id": "libc 0.2.155", "target": "libc" } ] @@ -16086,8 +16237,6 @@ "common": [ "Win32", "Win32_Foundation", - "Win32_NetworkManagement", - "Win32_NetworkManagement_IpHelper", "Win32_Networking", "Win32_Networking_WinSock", "Win32_Security", @@ -16095,8 +16244,6 @@ "Win32_Storage_FileSystem", "Win32_System", "Win32_System_Console", - "Win32_System_Diagnostics", - "Win32_System_Diagnostics_Debug", "Win32_System_IO", "Win32_System_Pipes", "Win32_System_SystemServices", @@ -16120,6 +16267,63 @@ }, "license": "MIT OR Apache-2.0" }, + "windows-sys 0.52.0": { + "name": "windows-sys", + "version": "0.52.0", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows-sys/0.52.0/download", + "sha256": "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_sys", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_sys", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "Win32", + "Win32_Foundation", + "Win32_NetworkManagement", + "Win32_NetworkManagement_IpHelper", + "Win32_Networking", + "Win32_Networking_WinSock", + "Win32_Storage", + "Win32_Storage_FileSystem", + "Win32_System", + "Win32_System_Diagnostics", + "Win32_System_Diagnostics_Debug", + "Win32_System_Threading", + "default" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "windows-targets 0.52.5", + "target": "windows_targets" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.0" + }, + "license": "MIT OR Apache-2.0" + }, "windows-targets 0.42.2": { "name": "windows-targets", "version": "0.42.2", @@ -16304,6 +16508,89 @@ }, "license": "MIT OR Apache-2.0" }, + "windows-targets 0.52.5": { + "name": "windows-targets", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows-targets/0.52.5/download", + "sha256": "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_targets", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_targets", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [], + "selects": { + "aarch64-pc-windows-gnullvm": [ + { + "id": "windows_aarch64_gnullvm 0.52.5", + "target": "windows_aarch64_gnullvm" + } + ], + "cfg(all(any(target_arch = \"x86_64\", target_arch = \"arm64ec\"), target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_x86_64_msvc 0.52.5", + "target": "windows_x86_64_msvc" + } + ], + "cfg(all(target_arch = \"aarch64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_aarch64_msvc 0.52.5", + "target": "windows_aarch64_msvc" + } + ], + "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + { + "id": "windows_i686_gnu 0.52.5", + "target": "windows_i686_gnu" + } + ], + "cfg(all(target_arch = \"x86\", target_env = \"msvc\", not(windows_raw_dylib)))": [ + { + "id": "windows_i686_msvc 0.52.5", + "target": "windows_i686_msvc" + } + ], + "cfg(all(target_arch = \"x86_64\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + { + "id": "windows_x86_64_gnu 0.52.5", + "target": "windows_x86_64_gnu" + } + ], + "i686-pc-windows-gnullvm": [ + { + "id": "windows_i686_gnullvm 0.52.5", + "target": "windows_i686_gnullvm" + } + ], + "x86_64-pc-windows-gnullvm": [ + { + "id": "windows_x86_64_gnullvm 0.52.5", + "target": "windows_x86_64_gnullvm" + } + ] + } + }, + "edition": "2021", + "version": "0.52.5" + }, + "license": "MIT OR Apache-2.0" + }, "windows_aarch64_gnullvm 0.42.2": { "name": "windows_aarch64_gnullvm", "version": "0.42.2", @@ -16410,6 +16697,59 @@ }, "license": "MIT OR Apache-2.0" }, + "windows_aarch64_gnullvm 0.52.5": { + "name": "windows_aarch64_gnullvm", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_aarch64_gnullvm/0.52.5/download", + "sha256": "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_aarch64_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_aarch64_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_aarch64_gnullvm 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "windows_aarch64_msvc 0.42.2": { "name": "windows_aarch64_msvc", "version": "0.42.2", @@ -16516,6 +16856,59 @@ }, "license": "MIT OR Apache-2.0" }, + "windows_aarch64_msvc 0.52.5": { + "name": "windows_aarch64_msvc", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_aarch64_msvc/0.52.5/download", + "sha256": "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_aarch64_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_aarch64_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_aarch64_msvc 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "windows_i686_gnu 0.42.2": { "name": "windows_i686_gnu", "version": "0.42.2", @@ -16622,6 +17015,112 @@ }, "license": "MIT OR Apache-2.0" }, + "windows_i686_gnu 0.52.5": { + "name": "windows_i686_gnu", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_gnu/0.52.5/download", + "sha256": "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_gnu 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, + "windows_i686_gnullvm 0.52.5": { + "name": "windows_i686_gnullvm", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_gnullvm/0.52.5/download", + "sha256": "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_gnullvm 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "windows_i686_msvc 0.42.2": { "name": "windows_i686_msvc", "version": "0.42.2", @@ -16728,6 +17227,59 @@ }, "license": "MIT OR Apache-2.0" }, + "windows_i686_msvc 0.52.5": { + "name": "windows_i686_msvc", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_i686_msvc/0.52.5/download", + "sha256": "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_i686_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_i686_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_i686_msvc 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "windows_x86_64_gnu 0.42.2": { "name": "windows_x86_64_gnu", "version": "0.42.2", @@ -16834,6 +17386,59 @@ }, "license": "MIT OR Apache-2.0" }, + "windows_x86_64_gnu 0.52.5": { + "name": "windows_x86_64_gnu", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_gnu/0.52.5/download", + "sha256": "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_gnu", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_gnu", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_gnu 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "windows_x86_64_gnullvm 0.42.2": { "name": "windows_x86_64_gnullvm", "version": "0.42.2", @@ -16940,6 +17545,59 @@ }, "license": "MIT OR Apache-2.0" }, + "windows_x86_64_gnullvm 0.52.5": { + "name": "windows_x86_64_gnullvm", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_gnullvm/0.52.5/download", + "sha256": "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_gnullvm", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_gnullvm", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_gnullvm 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "windows_x86_64_msvc 0.42.2": { "name": "windows_x86_64_msvc", "version": "0.42.2", @@ -17046,6 +17704,59 @@ }, "license": "MIT OR Apache-2.0" }, + "windows_x86_64_msvc 0.52.5": { + "name": "windows_x86_64_msvc", + "version": "0.52.5", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/windows_x86_64_msvc/0.52.5/download", + "sha256": "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + } + }, + "targets": [ + { + "Library": { + "crate_name": "windows_x86_64_msvc", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "windows_x86_64_msvc", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "deps": { + "common": [ + { + "id": "windows_x86_64_msvc 0.52.5", + "target": "build_script_build" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.52.5" + }, + "build_script_attrs": { + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0" + }, "winnow 0.5.14": { "name": "winnow", "version": "0.5.14", @@ -17123,6 +17834,65 @@ }, "license": "MIT" }, + "xattr 1.3.1": { + "name": "xattr", + "version": "1.3.1", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/xattr/1.3.1/download", + "sha256": "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" + } + }, + "targets": [ + { + "Library": { + "crate_name": "xattr", + "crate_root": "src/lib.rs", + "srcs": [ + "**/*.rs" + ] + } + } + ], + "library_target_name": "xattr", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "unsupported" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "rustix 0.38.34", + "target": "rustix" + } + ], + "selects": { + "cfg(any(target_os = \"freebsd\", target_os = \"netbsd\"))": [ + { + "id": "libc 0.2.155", + "target": "libc" + } + ], + "cfg(target_os = \"linux\")": [ + { + "id": "linux-raw-sys 0.4.14", + "target": "linux_raw_sys" + } + ] + } + }, + "edition": "2021", + "version": "1.3.1" + }, + "license": "MIT/Apache-2.0" + }, "yaml-rust 0.4.5": { "name": "yaml-rust", "version": "0.4.5", @@ -17277,6 +18047,9 @@ "armv7-unknown-linux-gnueabi": [ "armv7-unknown-linux-gnueabi" ], + "cfg(all(any(target_arch = \"x86_64\", target_arch = \"arm64ec\"), target_env = \"msvc\", not(windows_raw_dylib)))": [ + "x86_64-pc-windows-msvc" + ], "cfg(all(any(target_os = \"android\", target_os = \"linux\"), any(rustix_use_libc, miri, not(all(target_os = \"linux\", target_endian = \"little\", any(target_arch = \"arm\", all(target_arch = \"aarch64\", target_pointer_width = \"64\"), target_arch = \"riscv64\", all(rustix_use_experimental_asm, target_arch = \"powerpc64\"), all(rustix_use_experimental_asm, target_arch = \"mips\"), all(rustix_use_experimental_asm, target_arch = \"mips32r6\"), all(rustix_use_experimental_asm, target_arch = \"mips64\"), all(rustix_use_experimental_asm, target_arch = \"mips64r6\"), target_arch = \"x86\", all(target_arch = \"x86_64\", target_pointer_width = \"64\")))))))": [ "aarch64-linux-android", "armv7-linux-androideabi", @@ -17326,6 +18099,9 @@ "cfg(all(target_arch = \"wasm32\", not(target_os = \"wasi\")))": [ "wasm32-unknown-unknown" ], + "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ + "i686-unknown-linux-gnu" + ], "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(windows_raw_dylib)))": [ "i686-unknown-linux-gnu" ], @@ -17339,6 +18115,10 @@ "cfg(all(target_arch = \"x86_64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ "x86_64-pc-windows-msvc" ], + "cfg(any(target_os = \"freebsd\", target_os = \"netbsd\"))": [ + "i686-unknown-freebsd", + "x86_64-unknown-freebsd" + ], "cfg(any(target_os = \"linux\", target_os = \"android\", target_os = \"windows\", target_os = \"macos\", target_os = \"ios\", target_os = \"freebsd\", target_os = \"openbsd\", target_os = \"netbsd\", target_os = \"dragonfly\", target_os = \"solaris\", target_os = \"illumos\", target_os = \"fuchsia\", target_os = \"redox\", target_os = \"cloudabi\", target_os = \"haiku\", target_os = \"vxworks\", target_os = \"emscripten\", target_os = \"wasi\"))": [ "aarch64-apple-darwin", "aarch64-apple-ios", @@ -17621,8 +18401,18 @@ "wasm32-unknown-unknown", "wasm32-wasi" ], - "cfg(target_os = \"dragonfly\")": [], "cfg(target_os = \"hermit\")": [], + "cfg(target_os = \"linux\")": [ + "aarch64-unknown-linux-gnu", + "aarch64-unknown-nixos-gnu", + "arm-unknown-linux-gnueabi", + "armv7-unknown-linux-gnueabi", + "i686-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-nixos-gnu" + ], "cfg(target_os = \"redox\")": [], "cfg(target_os = \"wasi\")": [ "wasm32-wasi" @@ -17672,6 +18462,7 @@ "i686-linux-android" ], "i686-pc-windows-gnu": [], + "i686-pc-windows-gnullvm": [], "i686-pc-windows-msvc": [ "i686-pc-windows-msvc" ], @@ -17757,18 +18548,20 @@ "lazy_static 1.4.0", "once_cell 1.18.0", "paste 1.0.14", + "path-clean 1.0.1", "predicates 3.0.4", "protobuf 3.2.0", "regex 1.9.3", "rocket 0.5.0-rc.3", "rustc-hash 1.1.0", - "rustix 0.38.8", + "rustix 0.38.34", "rustyline 9.1.2", "scip 0.3.2", "serde 1.0.194", "serde_json 1.0.99", "string-interner 0.14.0", "syntect 4.7.0", + "tar 0.4.40", "tree-sitter 0.20.10", "tree-sitter-c 0.20.6", "tree-sitter-c-sharp 0.20.0", @@ -17797,6 +18590,7 @@ ], "direct_dev_deps": [ "criterion 0.4.0", - "pretty_assertions 1.4.0" + "pretty_assertions 1.4.0", + "tempfile 3.10.1" ] } diff --git a/docker-images/syntax-highlighter/Cargo.lock b/docker-images/syntax-highlighter/Cargo.lock index 748576a796f..cd7c8787494 100644 --- a/docker-images/syntax-highlighter/Cargo.lock +++ b/docker-images/syntax-highlighter/Cargo.lock @@ -625,23 +625,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] @@ -656,9 +645,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fd-lock" @@ -685,6 +674,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] + [[package]] name = "flate2" version = "1.0.27" @@ -1068,9 +1069,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "line-wrap" @@ -1089,9 +1090,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" @@ -1350,6 +1351,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "path-clean" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" + [[package]] name = "pear" version = "0.2.7" @@ -1644,6 +1651,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -1815,15 +1831,15 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1898,6 +1914,7 @@ dependencies = [ "insta", "lazy_static", "paste", + "path-clean", "predicates", "protobuf", "scip", @@ -1905,6 +1922,8 @@ dependencies = [ "serde_json", "string-interner", "syntax-analysis", + "tar", + "tempfile", "tree-sitter-all-languages", "walkdir", ] @@ -2161,16 +2180,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] -name = "tempfile" -version = "3.8.0" +name = "tar" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2892,6 +2921,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -2922,6 +2960,22 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -2934,6 +2988,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -2946,6 +3006,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -2958,6 +3024,18 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -2970,6 +3048,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -2982,6 +3066,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -2994,6 +3084,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -3006,6 +3102,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + [[package]] name = "winnow" version = "0.5.14" @@ -3024,6 +3126,17 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/docker-images/syntax-highlighter/Cargo.toml b/docker-images/syntax-highlighter/Cargo.toml index f655e853bc6..41f288e2b24 100644 --- a/docker-images/syntax-highlighter/Cargo.toml +++ b/docker-images/syntax-highlighter/Cargo.toml @@ -59,6 +59,7 @@ syntect = { git = "https://github.com/sourcegraph/syntect", rev = "7e02c5b4085e6 tree-sitter = "0.20.9" tree-sitter-highlight = "0.20.1" walkdir = "2" +path-clean = "1" scip = "0.3.2" protobuf = "3" diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/BUILD.bazel b/docker-images/syntax-highlighter/crates/scip-syntax/BUILD.bazel index 535cef8ec30..d8fc746b409 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/BUILD.bazel +++ b/docker-images/syntax-highlighter/crates/scip-syntax/BUILD.bazel @@ -94,6 +94,7 @@ rust_test( tags = [TAG_PLATFORM_GRAPH], deps = all_crate_deps( normal = True, + normal_dev = True, ) + [ ":scip-syntax", ":scip_syntax_lib", diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml b/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml index 554964072fc..7edb932cf93 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml +++ b/docker-images/syntax-highlighter/crates/scip-syntax/Cargo.toml @@ -23,6 +23,11 @@ serde = { workspace = true } serde_json = { workspace = true } string-interner = { workspace = true } walkdir = { workspace = true } +path-clean = { workspace = true } syntax-analysis = { path = "../syntax-analysis" } tree-sitter-all-languages = { path = "../tree-sitter-all-languages" } +tar = "0.4.40" + +[dev-dependencies] +tempfile="3.10.1" diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/index.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/index.rs index cf56c82a6bf..bd1b57422bd 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/index.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/index.rs @@ -1,7 +1,13 @@ -use std::path::{Path, PathBuf}; +use std::{ + env, + fs::File, + io::{self, prelude::*}, + path::{Path, PathBuf}, +}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use clap::ValueEnum; +use path_clean; use scip::{types::Document, write_message_to_file}; use syntax_analysis::{get_globals, get_locals}; use tree_sitter_all_languages::ParserId; @@ -20,7 +26,7 @@ pub struct IndexOptions { pub fail_fast: bool, } -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Debug)] pub enum AnalysisMode { /// Only extract occurrences of local definitions Locals, @@ -39,12 +45,32 @@ impl AnalysisMode { } } +pub enum TarMode { + /// Data is streamed from STDIN + Stdin, + + /// Data is read from a .tar file + File { location: PathBuf }, +} + pub enum IndexMode { /// Index only this list of files, without checking file extensions Files { list: Vec }, /// Discover all files that can be handled by the chosen language /// in the passed location (which has to be a directory) Workspace { location: PathBuf }, + + /// Discover all files that can be handled by the chosen language + /// in either a .tar file, or from STDIN to which TAR data is streamed + TarArchive { input: TarMode }, +} + +fn make_absolute(cwd: &Path, path: &Path) -> PathBuf { + if path.is_absolute() { + path.to_owned() + } else { + path_clean::clean(cwd.join(path)) + } } pub fn index_command( @@ -55,20 +81,17 @@ pub fn index_command( evaluate_against: Option, options: IndexOptions, ) -> Result<()> { - let p = ParserId::from_name(&language).unwrap(); - let project_root = { - match index_mode { - IndexMode::Files { .. } => project_root, - IndexMode::Workspace { ref location } => location.clone(), - } - }; + let parser_id = ParserId::from_name(&language) + .context(format!("No parser found for language {language}"))?; - let canonical_project_root = project_root.canonicalize().with_context(|| { - format!( - "Failed to canonicalize project root: {}", - project_root.display() - ) - })?; + let cwd = env::current_dir().context("Failed to get the current working directory")?; + let absolute_project_root = make_absolute( + &cwd, + match &index_mode { + IndexMode::Workspace { location } => location, + _ => &project_root, + }, + ); let mut index = scip::types::Index { metadata: Some(scip::types::Metadata { @@ -79,68 +102,44 @@ pub fn index_command( ..Default::default() }) .into(), - project_root: format!("file://{}", canonical_project_root.display()), + project_root: format!("file://{}", absolute_project_root.display()), ..Default::default() }) .into(), ..Default::default() }; - let mut index_file = |filepath: &Path| -> Result<()> { - let contents = std::fs::read_to_string(filepath) - .with_context(|| format!("Failed to read file at {}", filepath.display()))?; - let filepath = if filepath.is_absolute() { - filepath.to_owned() - } else { - filepath.canonicalize().with_context(|| { - format!("Failed to canonicalize file path: {}", filepath.display()) - })? - }; - let relative_path = filepath - .strip_prefix(canonical_project_root.clone()) - .with_context(|| { - format!( - "Failed to strip project root prefix: root={} file={}", - canonical_project_root.display(), - filepath.display() - ) - })?; - - match index_content(&contents, p, &options) { - Ok(mut document) => { - document.relative_path = relative_path.display().to_string(); - index.documents.push(document); - Ok(()) - } - Err(error) => { - if options.fail_fast { - Err(anyhow!( - "Failed to index {}: {:?}", - filepath.display(), - error - )) - } else { - eprintln!("Failed to index {}: {:?}", filepath.display(), error); - Ok(()) - } - } - } - }; + let extensions = ParserId::language_extensions(&parser_id); match index_mode { IndexMode::Files { list } => { let bar = create_progress_bar(list.len() as u64); for filename in list { - let filepath = PathBuf::from(filename).canonicalize().unwrap(); - bar.set_message(filepath.display().to_string()); - index_file(&filepath)?; + bar.set_message(filename.clone()); + let filepath = make_absolute(&cwd, &PathBuf::from(filename)); + let document = index_file(&filepath, parser_id, &absolute_project_root, &options)?; + index.documents.push(document); bar.inc(1); } bar.finish(); } + IndexMode::TarArchive { input } => match input { + TarMode::File { location } => { + let mut ar = tar::Archive::new(File::open(location)?); + let entries = ar.entries()?; + let documents = index_tar_entries(entries, parser_id, &options)?; + index.documents.extend(documents); + } + TarMode::Stdin => { + let stdin = io::stdin(); + let mut ar: tar::Archive<_> = tar::Archive::new(stdin); + let entries = ar.entries()?; + let documents = index_tar_entries(entries, parser_id, &options)?; + index.documents.extend(documents); + } + }, IndexMode::Workspace { location } => { - let extensions = ParserId::language_extensions(&p); let bar = create_spinner(); for entry in walkdir::WalkDir::new(location) { @@ -153,17 +152,21 @@ pub fn index_command( }; if extensions.contains(extension) { bar.set_message(entry.path().display().to_string()); - index_file(&entry.into_path())?; + let document = index_file( + &entry.into_path(), + parser_id, + &absolute_project_root, + &options, + )?; + index.documents.push(document); bar.tick(); } } } } - eprintln!(); - eprintln!( - "Writing index for {} documents into {}", + "\nWriting index for {} documents into {}", index.documents.len(), out.display() ); @@ -179,11 +182,102 @@ pub fn index_command( .write_summary(&mut std::io::stdout(), Default::default())? } - write_message_to_file(out.clone(), index) + write_message_to_file(&out, index) .map_err(|err| anyhow!("{err:?}")) .with_context(|| format!("When writing index to {}", out.display())) } +fn index_file( + filepath: &Path, + parser_id: ParserId, + absolute_project_root: &Path, + options: &IndexOptions, +) -> Result { + let contents = std::fs::read_to_string(filepath) + .with_context(|| format!("Failed to read file at {}", filepath.display()))?; + + let relative_path = filepath + .strip_prefix(absolute_project_root) + .with_context(|| { + format!( + "Failed to strip project root prefix: root={} file={}", + absolute_project_root.display(), + filepath.display() + ) + })?; + + match index_content(&contents, parser_id, options) { + Ok(mut document) => { + document.relative_path = relative_path.display().to_string(); + Ok(document) + } + Err(error) => { + bail!("Failed to index {}: {:?}", filepath.display(), error) + } + } +} + +fn index_tar_entries( + entries: tar::Entries<'_, R>, + parser: ParserId, + options: &IndexOptions, +) -> anyhow::Result> { + let extensions = ParserId::language_extensions(&parser); + let mut contents = String::new(); + let mut documents: Vec = vec![]; + let mut progress = 0; + let spinner = create_spinner(); + for entry in entries { + let mut e = entry?; + let path = PathBuf::from(e.path()?); + + if matches!(path.extension().and_then(|e| e.to_str()), Some(ext) if extensions.contains(ext)) + { + match e.read_to_string(&mut contents) { + Ok(size) => { + match index_content(&contents, parser, options) { + Ok(mut document) => { + document.relative_path = path.display().to_string(); + documents.push(document); + } + Err(error) => { + if options.fail_fast { + anyhow::bail!("Failed to index {}: {:?}", path.display(), error); + } else { + eprintln!("Failed to index {}: {:?}", path.display(), error); + } + } + } + if size > 0 { + contents.clear(); + } + } + Err(error) => { + if options.fail_fast { + anyhow::bail!( + "Failed to read contents of path {}: {:?}", + path.display(), + error + ) + } else { + eprintln!( + "Failed to read contents of path {}: {:?}", + path.display(), + error + ); + } + } + } + + progress += 1; + spinner.set_message(format!("[{}]: {}", progress, path.display())); + spinner.tick(); + } + } + + Ok(documents) +} + fn index_content(contents: &str, parser: ParserId, options: &IndexOptions) -> Result { let mut document: Document; diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/src/main.rs b/docker-images/syntax-highlighter/crates/scip-syntax/src/main.rs index 43a55e91475..f1cb92f7065 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/src/main.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/src/main.rs @@ -1,7 +1,7 @@ use std::{path::PathBuf, process}; use clap::{Parser, Subcommand}; -use scip_syntax::index::{index_command, AnalysisMode, IndexMode, IndexOptions}; +use scip_syntax::index::{index_command, AnalysisMode, IndexMode, IndexOptions, TarMode}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -11,44 +11,78 @@ struct Cli { command: Commands, } +#[derive(Parser, Clone, Debug)] +struct IndexCommandOptions { + /// Which language parser to use to process the files + #[arg(short, long)] + language: String, + + /// Path where the SCIP index will be written + #[arg(short, long, default_value = "./index.scip")] + out: String, + + /// Analysis mode + #[arg(short, long, default_value = "full")] + mode: AnalysisMode, + + /// Fail on first error + #[arg(long, default_value_t = false)] + fail_fast: bool, + + /// Project root to write to SCIP index + #[arg(short, long, default_value = "./")] + project_root: String, + + /// Evaluate the build index against an index from a file + #[arg(long)] + evaluate: Option, +} + +#[derive(Subcommand, Debug)] +enum IndexCommand { + /// Index a folder, automatically detecting files + /// to be processed by the chosen language + Workspace { + /// Folder to index - will be chosen as project root, + /// and files will be discovered according to + /// configured extensions for the selected language + /// Has to be absolute path. + dir: String, + + #[command(flatten)] + options: IndexCommandOptions, + }, + + /// Index a list of files + Files { + /// List of files to analyse + filenames: Vec, + + #[command(flatten)] + options: IndexCommandOptions, + }, + + /// Index a .tar archive, either from a file or streaming from STDIN + Tar { + /// Either a path to .tar file, or "-" to read .tar data from STDIN + tar: String, + + #[command(flatten)] + options: IndexCommandOptions, + }, +} +#[derive(Parser, Debug)] +struct IndexCommandParser { + #[structopt(subcommand)] + index_command: IndexCommand, +} + #[derive(Subcommand)] enum Commands { /// Index source files using Tree Sitter parser for a given language /// and produce a SCIP file - Index { - /// Which language parser to use to process the files - #[arg(short, long)] - language: String, - - /// Path where the SCIP index will be written - #[arg(short, long, default_value = "./index.scip")] - out: String, - - /// Folder to index - will be chosen as project root, - /// and files will be discovered according to - /// configured extensions for the selected language - #[arg(long)] - workspace: Option, - - /// List of files to analyse - filenames: Vec, - - /// Analysis mode - #[arg(short, long, default_value = "full")] - mode: AnalysisMode, - - /// Fail on first error - #[arg(long, default_value_t = false)] - fail_fast: bool, - - /// Project root to write to SCIP index - #[arg(short, long, default_value = "./")] - project_root: String, - - /// Evaluate the build index against an index from a file - #[arg(long)] - evaluate: Option, - }, + #[clap(name = "index")] + Index(IndexCommandParser), /// Fuzzily evaluate candidate SCIP index against known ground truth ScipEvaluate { @@ -99,49 +133,44 @@ pub fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { - Commands::Index { - language, - out, - filenames, - workspace, - mode, - fail_fast, - project_root, - evaluate, - } => { - let index_mode = { - match workspace { - None => { - if filenames.is_empty() { - eprintln!("either specify --workspace or provide a list of files"); - process::exit(1) - } - IndexMode::Files { list: filenames } + Commands::Index(index1) => { + let result = match index1.index_command { + IndexCommand::Files { filenames, options } => { + if filenames.is_empty() { + eprintln!("List of files cannot be empty"); + process::exit(1) } - Some(location) => { - if !filenames.is_empty() { - eprintln!("--workspace option cannot be combined with a list of files"); - process::exit(1) - } else { - IndexMode::Workspace { - location: location.into(), - } - } + run_index_command(options, IndexMode::Files { list: filenames }) + } + IndexCommand::Workspace { dir, options } => run_index_command( + options, + IndexMode::Workspace { + location: dir.into(), + }, + ), + + IndexCommand::Tar { tar, options } => { + if tar == "-" { + run_index_command( + options, + IndexMode::TarArchive { + input: scip_syntax::index::TarMode::Stdin, + }, + ) + } else { + run_index_command( + options, + IndexMode::TarArchive { + input: TarMode::File { + location: PathBuf::from(tar), + }, + }, + ) } } }; - index_command( - language, - index_mode, - PathBuf::from(out), - PathBuf::from(project_root), - evaluate.map(PathBuf::from), - IndexOptions { - analysis_mode: mode, - fail_fast, - }, - )? + result.unwrap() } Commands::ScipEvaluate { @@ -166,3 +195,17 @@ pub fn main() -> anyhow::Result<()> { } Ok(()) } + +fn run_index_command(options: IndexCommandOptions, mode: IndexMode) -> anyhow::Result<()> { + index_command( + options.language, + mode, + PathBuf::from(options.out), + PathBuf::from(options.project_root), + options.evaluate.map(PathBuf::from), + IndexOptions { + analysis_mode: options.mode, + fail_fast: options.fail_fast, + }, + ) +} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/testdata/package-info.java b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/package-info.java new file mode 100644 index 00000000000..07ff6085cc0 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/testdata/package-info.java @@ -0,0 +1,4 @@ +@Deprecated +package foo.bar; + +class Baz {} diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/tests/integration_test.rs b/docker-images/syntax-highlighter/crates/scip-syntax/tests/integration_test.rs index a99fa2fec78..9314654b476 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/tests/integration_test.rs +++ b/docker-images/syntax-highlighter/crates/scip-syntax/tests/integration_test.rs @@ -1,11 +1,13 @@ use std::{ - collections::HashMap, - env::temp_dir, + collections::{HashMap, HashSet}, + io::Write, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, }; +use anyhow::{anyhow, bail, Context, Result}; use assert_cmd::{cargo::cargo_bin, prelude::*}; +use scip::types::Document; use scip_syntax::{ evaluate::Evaluator, index::{index_command, AnalysisMode, IndexMode, IndexOptions}, @@ -37,6 +39,7 @@ lazy_static::lazy_static! { } use syntax_analysis::snapshot::{dump_document_with_config, EmitSymbol, SnapshotOptions}; +use tar::{Builder, Header}; fn snapshot_syntax_document(doc: &scip::types::Document, source: &str) -> String { dump_document_with_config( @@ -54,7 +57,7 @@ fn snapshot_syntax_document(doc: &scip::types::Document, source: &str) -> String fn java_e2e_evaluation() { let dir = BASE.join("testdata/java"); - let out_dir = temp_dir(); + let out_dir = tempdir(); let candidate = out_dir.join("index-tree-sitter.scip"); @@ -94,55 +97,273 @@ fn java_e2e_evaluation() { } #[test] -fn java_e2e_indexing() { - let out_dir = temp_dir(); - let setup = HashMap::from([( - PathBuf::from("globals.java"), - include_str!("../testdata/globals.java").to_string(), - )]); +fn java_files_indexing() { + let out_dir = tempdir(); + let setup = indexing_data(); - run_index(&out_dir, &setup, vec!["--language", "java"]); + let mut cmd = command("index"); + let output_location = out_dir.join("index.scip"); + let paths = extract_paths(&setup); - let index = read_index_from_file(&out_dir.join("index.scip")).unwrap(); + prepare(&out_dir, &setup).unwrap(); - for doc in &index.documents { - let path = &doc.relative_path; - let dumped = snapshot_syntax_document(doc, setup.get(&PathBuf::from(&path)).expect("??")); + cmd.args(vec![ + "files", + "--language", + "java", + "--out", + output_location.to_str().unwrap(), + ]) + .current_dir(&out_dir) + .args(paths) + .assert() + .success(); - insta::assert_snapshot!(path.clone(), dumped); - } + let index = read_index_from_file(&output_location).unwrap(); + + assert_eq!(extract_paths(&setup), extract_indexed_paths(&index)); + + let index_snapshot = snapshot_from_files(&index.documents, &out_dir); + + insta::assert_snapshot!(index_snapshot); } -fn prepare(temp: &Path, files: &HashMap) { +#[test] +fn java_workspace_indexing() { + let out_dir = tempdir(); + let setup = indexing_data(); + + let mut cmd = command("index"); + let output_location = out_dir.join("index.scip"); + + prepare(&out_dir, &setup).unwrap(); + + cmd.args(vec![ + "workspace", + out_dir.to_str().unwrap(), + "--language", + "java", + "--out", + output_location.to_str().unwrap(), + ]) + .assert() + .success(); + + let index = read_index_from_file(&output_location).unwrap(); + + assert_eq!(extract_paths(&setup), extract_indexed_paths(&index)); + + let index_snapshot = snapshot_from_files(&index.documents, &out_dir); + + insta::assert_snapshot!(index_snapshot); +} + +#[test] +fn java_tar_file_indexing() { + let out_dir = tempdir(); + let setup = indexing_data(); + let tar_data = create_tar(&setup); + + let data = tar_data.unwrap(); + + let mut cmd = command("index"); + let tar_file = out_dir.join("test.tar"); + let output_location = out_dir.join("index.scip"); + + write_file_bytes(&tar_file, &data).unwrap(); + + cmd.args(vec![ + "tar", + tar_file.to_str().unwrap(), + "--language", + "java", + "--out", + output_location.to_str().unwrap(), + ]) + .assert() + .success(); + + let index = read_index_from_file(&output_location).unwrap(); + + assert_eq!(extract_paths(&setup), extract_indexed_paths(&index)); + + let index_snapshot = snapshot_from_data(&index.documents, &setup); + + insta::assert_snapshot!(index_snapshot); +} + +#[test] +fn java_tar_stream_indexing() { + let out_dir = tempdir(); + let setup = indexing_data(); + let tar_data = create_tar(&setup); + + let data = tar_data.unwrap(); + + let mut cmd = command("index"); + let tar_file = out_dir.join("test.tar"); + let output_location = out_dir.join("index.scip"); + + write_file_bytes(&tar_file, &data) + .context("Failed to write tar data") + .unwrap(); + + let mut spawned = cmd + .args(vec![ + "tar", + "-", + "--language", + "java", + "--out", + output_location.to_str().unwrap(), + ]) + .stdin(Stdio::piped()) + .spawn() + .unwrap(); + + spawned.stdin.take().unwrap().write_all(&data).unwrap(); + + let exit_status = spawned.wait().unwrap(); + + assert_eq!(exit_status.code(), Some(0)); + + let index = read_index_from_file(&output_location).unwrap(); + + assert_eq!(extract_paths(&setup), extract_indexed_paths(&index)); + + let index_snapshot = snapshot_from_data(&index.documents, &setup); + + insta::assert_snapshot!(index_snapshot); +} + +fn prepare(temp: &Path, files: &HashMap) -> Result<()> { for (path, contents) in files.iter() { let file_path = temp.join(path); - write_file(&file_path, contents); + write_file_string(&file_path, contents)?; } + + Ok(()) } -fn run_index(location: &PathBuf, files: &HashMap, extra_arguments: Vec<&str>) { - prepare(location, files); - - let mut base_args = vec!["index"]; - base_args.extend(extra_arguments); - +fn command(sub: &str) -> Command { let mut cmd = Command::new(BINARY_LOCATION.to_str().unwrap()); - cmd.args(base_args); + cmd.arg(sub); - for (path, _) in files.iter() { - cmd.arg(path.to_str().unwrap()); - } - - cmd.current_dir(location); - - cmd.assert().success(); + cmd } -fn write_file(path: &PathBuf, contents: &String) { +fn write_file_string(path: &PathBuf, contents: &String) -> Result<()> { + write_file_bytes(path, contents.as_bytes()) +} + +fn write_file_bytes(path: &PathBuf, contents: &[u8]) -> Result<()> { use std::io::Write; - let output = std::fs::File::create(path).unwrap(); + let Some(parent) = path.parent() else { + bail!("failed to find parent dir for {}", path.display()) + }; + + std::fs::create_dir_all(parent) + .with_context(|| anyhow!("Failed to create all parent folders for {}", path.display()))?; + + let output = std::fs::File::create(path) + .with_context(|| anyhow!("Failed to open file {} for writing", path.to_str().unwrap()))?; let mut writer = std::io::BufWriter::new(output); - writer.write_all(contents.as_bytes()).unwrap(); + writer.write_all(contents)?; + + Ok(()) +} + +fn tempdir() -> PathBuf { + tempfile::tempdir().unwrap().into_path() +} + +fn create_tar(files: &HashMap) -> Result, std::io::Error> { + let mut ar = Builder::new(Vec::new()); + + for (path, text) in files.iter() { + let mut header = Header::new_gnu(); + let bytes = text.as_bytes(); + + header + .set_path(path.to_str().unwrap()) + .expect("Failed to set path for archive entry"); + header.set_size(bytes.len() as u64); + header.set_cksum(); + ar.append(&header, bytes).unwrap(); + } + + ar.into_inner() +} + +fn indexing_data() -> HashMap { + HashMap::from([ + ( + PathBuf::from("src/main/java/globals.java"), + include_str!("../testdata/globals.java").to_string(), + ), + ( + PathBuf::from("package-info.java"), + include_str!("../testdata/package-info.java").to_string(), + ), + ]) +} + +fn extract_paths(setup: &HashMap) -> HashSet { + setup + .keys() + .map(|pb| pb.to_str().unwrap().to_string()) + .collect() +} + +fn extract_indexed_paths(index: &scip::types::Index) -> HashSet { + index + .documents + .iter() + .map(|pb| pb.relative_path.clone()) + .collect() +} + +fn snapshot_from_files(docs: &[Document], project_root: &Path) -> String { + let mut str = String::new(); + let mut docs = docs.to_owned(); + docs.sort_by_key(|doc| doc.relative_path.clone()); + + for doc in docs { + let path = project_root.join(doc.relative_path.clone()); + let contents = std::fs::read_to_string(path.clone()) + .with_context(|| anyhow!("Failed to read path {}", path.display())) + .unwrap(); + + str.push_str(&format_snapshot_document(&doc, &contents)); + } + + str +} + +fn format_snapshot_document(doc: &scip::types::Document, contents: &str) -> String { + let mut str = String::new(); + str.push_str(format!("//----FILE={}\n", doc.relative_path).as_str()); + str.push_str(&snapshot_syntax_document(doc, contents)); + str.push_str("\n\n"); + + str +} + +fn snapshot_from_data(docs: &[Document], data: &HashMap) -> String { + let mut str = String::new(); + let mut docs = docs.to_owned(); + docs.sort_by_key(|doc| doc.relative_path.clone()); + + for doc in docs { + let contents = data + .get(&PathBuf::from(&doc.relative_path)) + .context(format!("Failed to find {} in data", &doc.relative_path)) + .unwrap(); + + str.push_str(&format_snapshot_document(&doc, contents)); + } + + str } diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__globals.java.snap b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_files_indexing.snap similarity index 93% rename from docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__globals.java.snap rename to docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_files_indexing.snap index b03bfa08386..e72be905a46 100644 --- a/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__globals.java.snap +++ b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_files_indexing.snap @@ -1,7 +1,17 @@ --- source: crates/scip-syntax/tests/integration_test.rs -expression: dumped +expression: index_snapshot --- +//----FILE=package-info.java + @Deprecated + package foo.bar; +// ^^^^^^^ definition(Package) scip-ctags `foo.bar`/ + + class Baz {} +// ^^^ definition scip-ctags `foo.bar`/Baz# + + +//----FILE=src/main/java/globals.java package MyPackage; // ^^^^^^^^^ definition(Package) scip-ctags MyPackage/ @@ -72,3 +82,5 @@ expression: dumped } } + + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_file_indexing.snap b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_file_indexing.snap new file mode 100644 index 00000000000..e72be905a46 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_file_indexing.snap @@ -0,0 +1,86 @@ +--- +source: crates/scip-syntax/tests/integration_test.rs +expression: index_snapshot +--- +//----FILE=package-info.java + @Deprecated + package foo.bar; +// ^^^^^^^ definition(Package) scip-ctags `foo.bar`/ + + class Baz {} +// ^^^ definition scip-ctags `foo.bar`/Baz# + + +//----FILE=src/main/java/globals.java + package MyPackage; +// ^^^^^^^^^ definition(Package) scip-ctags MyPackage/ + + public class globals { +// ^^^^^^^ definition scip-ctags MyPackage/globals# + private static int field1; +// ^^^^^^ definition scip-ctags MyPackage/globals#field1. + protected static int field2; +// ^^^^^^ definition scip-ctags MyPackage/globals#field2. + public static int field3; +// ^^^^^^ definition scip-ctags MyPackage/globals#field3. + private int field4; +// ^^^^^^ definition scip-ctags MyPackage/globals#field4. + protected int field5; +// ^^^^^^ definition scip-ctags MyPackage/globals#field5. + public int field6; +// ^^^^^^ definition scip-ctags MyPackage/globals#field6. + + private static void method1() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method1(). + protected static void method2() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method2(). + public static void method3() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method3(). + private void method4() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method4(). + protected void method5() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method5(). + public void method6() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method6(). + + public static final String COOLEST_STRING = "probably this one"; +// ^^^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#COOLEST_STRING. + + public class ClassInAClass { +// ^^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass# + boolean classy = true; +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#classy. + + public static enum Enum { +// ^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum# + these, +// ^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#these. + should, +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#should. + be, +// ^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#be. + recognized, +// ^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#recognized. + as, +// ^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#as. + terms +// ^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#terms. + } + + public interface Goated { +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Goated# + boolean withTheSauce(); +// ^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Goated#withTheSauce(). + } + + public void myCoolMethod() { +// ^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#myCoolMethod(). + class WhatIsGoingOn {} + boolean iThinkThisIsAllowedButWeDontReallyCare = true; +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition local 1 + } + } + } + + + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_stream_indexing.snap b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_stream_indexing.snap new file mode 100644 index 00000000000..e72be905a46 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_tar_stream_indexing.snap @@ -0,0 +1,86 @@ +--- +source: crates/scip-syntax/tests/integration_test.rs +expression: index_snapshot +--- +//----FILE=package-info.java + @Deprecated + package foo.bar; +// ^^^^^^^ definition(Package) scip-ctags `foo.bar`/ + + class Baz {} +// ^^^ definition scip-ctags `foo.bar`/Baz# + + +//----FILE=src/main/java/globals.java + package MyPackage; +// ^^^^^^^^^ definition(Package) scip-ctags MyPackage/ + + public class globals { +// ^^^^^^^ definition scip-ctags MyPackage/globals# + private static int field1; +// ^^^^^^ definition scip-ctags MyPackage/globals#field1. + protected static int field2; +// ^^^^^^ definition scip-ctags MyPackage/globals#field2. + public static int field3; +// ^^^^^^ definition scip-ctags MyPackage/globals#field3. + private int field4; +// ^^^^^^ definition scip-ctags MyPackage/globals#field4. + protected int field5; +// ^^^^^^ definition scip-ctags MyPackage/globals#field5. + public int field6; +// ^^^^^^ definition scip-ctags MyPackage/globals#field6. + + private static void method1() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method1(). + protected static void method2() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method2(). + public static void method3() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method3(). + private void method4() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method4(). + protected void method5() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method5(). + public void method6() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method6(). + + public static final String COOLEST_STRING = "probably this one"; +// ^^^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#COOLEST_STRING. + + public class ClassInAClass { +// ^^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass# + boolean classy = true; +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#classy. + + public static enum Enum { +// ^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum# + these, +// ^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#these. + should, +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#should. + be, +// ^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#be. + recognized, +// ^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#recognized. + as, +// ^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#as. + terms +// ^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#terms. + } + + public interface Goated { +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Goated# + boolean withTheSauce(); +// ^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Goated#withTheSauce(). + } + + public void myCoolMethod() { +// ^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#myCoolMethod(). + class WhatIsGoingOn {} + boolean iThinkThisIsAllowedButWeDontReallyCare = true; +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition local 1 + } + } + } + + + diff --git a/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_workspace_indexing.snap b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_workspace_indexing.snap new file mode 100644 index 00000000000..e72be905a46 --- /dev/null +++ b/docker-images/syntax-highlighter/crates/scip-syntax/tests/snapshots/integration_test__java_workspace_indexing.snap @@ -0,0 +1,86 @@ +--- +source: crates/scip-syntax/tests/integration_test.rs +expression: index_snapshot +--- +//----FILE=package-info.java + @Deprecated + package foo.bar; +// ^^^^^^^ definition(Package) scip-ctags `foo.bar`/ + + class Baz {} +// ^^^ definition scip-ctags `foo.bar`/Baz# + + +//----FILE=src/main/java/globals.java + package MyPackage; +// ^^^^^^^^^ definition(Package) scip-ctags MyPackage/ + + public class globals { +// ^^^^^^^ definition scip-ctags MyPackage/globals# + private static int field1; +// ^^^^^^ definition scip-ctags MyPackage/globals#field1. + protected static int field2; +// ^^^^^^ definition scip-ctags MyPackage/globals#field2. + public static int field3; +// ^^^^^^ definition scip-ctags MyPackage/globals#field3. + private int field4; +// ^^^^^^ definition scip-ctags MyPackage/globals#field4. + protected int field5; +// ^^^^^^ definition scip-ctags MyPackage/globals#field5. + public int field6; +// ^^^^^^ definition scip-ctags MyPackage/globals#field6. + + private static void method1() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method1(). + protected static void method2() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method2(). + public static void method3() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method3(). + private void method4() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method4(). + protected void method5() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method5(). + public void method6() {} +// ^^^^^^^ definition scip-ctags MyPackage/globals#method6(). + + public static final String COOLEST_STRING = "probably this one"; +// ^^^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#COOLEST_STRING. + + public class ClassInAClass { +// ^^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass# + boolean classy = true; +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#classy. + + public static enum Enum { +// ^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum# + these, +// ^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#these. + should, +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#should. + be, +// ^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#be. + recognized, +// ^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#recognized. + as, +// ^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#as. + terms +// ^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Enum#terms. + } + + public interface Goated { +// ^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Goated# + boolean withTheSauce(); +// ^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#Goated#withTheSauce(). + } + + public void myCoolMethod() { +// ^^^^^^^^^^^^ definition scip-ctags MyPackage/globals#ClassInAClass#myCoolMethod(). + class WhatIsGoingOn {} + boolean iThinkThisIsAllowedButWeDontReallyCare = true; +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition local 1 + } + } + } + + +