diff --git a/BUILD.bazel b/BUILD.bazel index ac37a2a0490..13cc4575137 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -205,22 +205,24 @@ buildifier( # Go gazelle_binary( - name = "gazelle-buf", + name = "gazelle-bin", languages = [ # Loads the native proto extension "@bazel_gazelle//language/proto:go_default_library", # Gazelle-buf does not include the Go plugin by default, so we have to add it # ourselves. "@bazel_gazelle//language/go:go_default_library", + # Bundled with aspect-cli, but we're missing out on buf that way + "@aspect_cli//gazelle/js:js", # Loads the Buf extension - "@rules_buf//gazelle/buf:buf", # NOTE: This needs to be loaded after the proto language + "@rules_buf//gazelle/buf:buf", ], ) gazelle( name = "gazelle", - gazelle = ":gazelle-buf", + gazelle = ":gazelle-bin", ) sh_binary( diff --git a/WORKSPACE b/WORKSPACE index 970a48a8c7b..da640ee6bd8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -141,6 +141,16 @@ http_archive( urls = ["https://github.com/keith/buildifier-prebuilt/archive/6.1.0.tar.gz"], ) +http_archive( + name = "aspect_cli", + repo_mapping = { + "@com_github_smacker_go_tree_sitter": "@aspectcli-com_github_smacker_go_tree_sitter", + }, + sha256 = "045f0186edb25706dfe77d9c4916eec630a2b2736f9abb59e37eaac122d4b771", + strip_prefix = "aspect-cli-5.8.20", + url = "https://github.com/aspect-build/aspect-cli/archive/5.8.20.tar.gz", +) + # hermetic_cc_toolchain setup ================================ HERMETIC_CC_TOOLCHAIN_VERSION = "v2.2.1" @@ -271,9 +281,9 @@ go_repository( name = "org_golang_google_protobuf", build_file_proto_mode = "disable_global", importpath = "google.golang.org/protobuf", - sum = "h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=", - version = "v1.31.1", -) # keep + sum = "h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=", + version = "v1.32.0", +) # Pin protoc-gen-go-grpc to 1.3.0 # See also //:gen-go-grpc @@ -285,6 +295,20 @@ go_repository( version = "v1.3.0", ) # keep +# Pin specific version for aspect-cli's gazelle rules, with versions +# that it requires but that our codebase doesnt support. +go_repository( + name = "aspectcli-com_github_smacker_go_tree_sitter", + build_file_proto_mode = "disable_global", + importpath = "github.com/smacker/go-tree-sitter", + sum = "h1:DxgjlvWYsb80WEN2Zv3WqJFAg2DKjUQJO6URGdf1x6Y=", + version = "v0.0.0-20230720070738-0d0a9f78d8f8", +) # keep + +load("@aspect_cli//:go.bzl", aspect_cli_deps = "deps") + +aspect_cli_deps() + # gazelle:repository_macro deps.bzl%go_dependencies go_dependencies() diff --git a/dev/ci/bazel-configure.sh b/dev/ci/bazel-configure.sh index 9a01a54b921..5d7c36fcf1c 100755 --- a/dev/ci/bazel-configure.sh +++ b/dev/ci/bazel-configure.sh @@ -8,8 +8,6 @@ PATH="$(dirname "${runfiles_dir}/${GO}"):${PATH}" # Remove bazelisk from path PATH=$(echo "${PATH}" | awk -v RS=: -v ORS=: '/bazelisk/ {next} {print}') export PATH -# Allow Aspect to re-enter again -export ASPECT_REENTRANT= cd "${BUILD_WORKSPACE_DIRECTORY}" @@ -17,7 +15,7 @@ bazel \ --bazelrc=.bazelrc \ --bazelrc=.aspect/bazelrc/ci.bazelrc \ --bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc \ - configure + run //:gazelle if [ "${CI:-}" ]; then git ls-files --exclude-standard --others | xargs git add --intent-to-add || true diff --git a/dev/ci/bazel-prechecks.sh b/dev/ci/bazel-prechecks.sh index 013f6f04a7b..3b30f2a07fd 100755 --- a/dev/ci/bazel-prechecks.sh +++ b/dev/ci/bazel-prechecks.sh @@ -30,8 +30,8 @@ function generate_diff_artifact() { trap generate_diff_artifact EXIT -echo "--- :bazel: Running bazel configure" -bazel "${bazelrc[@]}" configure +echo "--- :bazel: Running bazel run //:gazelle" +bazel "${bazelrc[@]}" run //:gazelle echo "--- Checking if BUILD.bazel files were updated" # Account for the possibility of a BUILD.bazel to be totally new, and thus untracked. @@ -39,7 +39,7 @@ git ls-files --exclude-standard --others | grep BUILD.bazel | xargs git add --in git diff --exit-code || EXIT_CODE=$? # do not fail on non-zero exit -# if we get a non-zero exit code, bazel configure updated files +# if we get a non-zero exit code, bazel run //:gazelle updated files if [[ $EXIT_CODE -ne 0 ]]; then mkdir -p ./annotations cat <<-'END' > ./annotations/bazel-prechecks.md @@ -48,7 +48,7 @@ if [[ $EXIT_CODE -ne 0 ]]; then BUILD.bazel files need to be updated to match the repository state. You should run the following command and commit the result ``` - bazel configure + sg bazel configure ``` #### For more information please see the [Bazel FAQ](https://docs.sourcegraph.com/dev/background-information/bazel/faq) @@ -63,7 +63,7 @@ bazel "${bazelrc[@]}" run //:gazelle-update-repos echo "--- Checking if deps.bzl was updated" git diff --exit-code || EXIT_CODE=$? # do not fail on non-zero exit -# if we get a non-zero exit code, bazel configure updated files +# if we get a non-zero exit code, bazel run //:gazelle-update-repos updated files if [[ $EXIT_CODE -ne 0 ]]; then mkdir -p ./annotations cat <<-'END' > ./annotations/bazel-prechecks.md @@ -72,7 +72,7 @@ if [[ $EXIT_CODE -ne 0 ]]; then `deps.bzl` needs to be updated to match the repository state. You should run the following command and commit the result ``` - bazel run //:gazelle-update-repos + sg bazel configure godeps ``` #### For more information please see the [Bazel FAQ](https://docs.sourcegraph.com/dev/background-information/bazel/faq) diff --git a/dev/sg/BUILD.bazel b/dev/sg/BUILD.bazel index a6c3e3729e6..56c619ba1ac 100644 --- a/dev/sg/BUILD.bazel +++ b/dev/sg/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "os.go", "sg_analytics.go", "sg_audit.go", + "sg_bazel.go", "sg_cloud.go", "sg_db.go", "sg_deploy.go", @@ -115,6 +116,8 @@ go_library( "@in_gopkg_yaml_v3//:yaml_v3", "@io_opentelemetry_go_otel//attribute", "@io_opentelemetry_go_otel_trace//:trace", + "@org_golang_x_exp//maps", + "@org_golang_x_exp//slices", "@org_golang_x_mod//semver", "@org_golang_x_oauth2//:oauth2", "@org_golang_x_text//cases", diff --git a/dev/sg/main.go b/dev/sg/main.go index 61145991727..2bcf06f341a 100644 --- a/dev/sg/main.go +++ b/dev/sg/main.go @@ -266,6 +266,7 @@ var sg = &cli.App{ testCommand, lintCommand, generateCommand, + bazelCommand, dbCommand, migrationCommand, insightsCommand, diff --git a/dev/sg/sg_bazel.go b/dev/sg/sg_bazel.go new file mode 100644 index 00000000000..87ad5054f4b --- /dev/null +++ b/dev/sg/sg_bazel.go @@ -0,0 +1,146 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/urfave/cli/v2" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" + + "github.com/sourcegraph/sourcegraph/dev/sg/internal/category" + "github.com/sourcegraph/sourcegraph/dev/sg/internal/std" + "github.com/sourcegraph/sourcegraph/dev/sg/root" + "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/lib/output" +) + +type bzlgenTarget struct { + order int + cmd string + args []string + env []string + protip string +} + +var bzlgenTargets = map[string]bzlgenTarget{ + "builds": { + order: 1, + cmd: "run", + args: []string{"//:gazelle"}, + }, + "godeps": { + cmd: "run", + args: []string{"//:gazelle-update-repos"}, + }, + "rustdeps": { + cmd: "sync", + args: []string{"--only=crate_index"}, + env: []string{"CARGO_BAZEL_REPIN=1"}, + protip: "run with CARGO_BAZEL_ISOLATED=0 for faster (but less sandboxed) repinning.", + }, +} + +var bazelCommand = &cli.Command{ + Name: "bazel", + SkipFlagParsing: true, + Usage: "placeholder, handled in top-level Action below", + Category: category.Dev, + Action: func(ctx *cli.Context) error { + root, err := root.RepositoryRoot() + if err != nil { + return err + } + + if slices.Equal(ctx.Args().Slice(), []string{"help"}) || slices.Equal(ctx.Args().Slice(), []string{"--help"}) || slices.Equal(ctx.Args().Slice(), []string{"-h"}) { + fmt.Println("Additional commands from sg:") + fmt.Println(" configure Wrappers around some commands to generate various files required by Bazel") + } + + cmd := exec.CommandContext(ctx.Context, "bazel", ctx.Args().Slice()...) + cmd.Dir = root + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + return cmd.Run() + }, + Subcommands: []*cli.Command{ + { + Name: "configure", + Usage: "Wrappers around some commands to generate various files required by Bazel", + UsageText: "sg bazel configure [category...]", + Description: `For convenience, a number of Bazel commands are wrapped by this command to update various files required by Bazel. + +Available categories: + - builds: updates BUILD.bazel files for Go & Typescript targets. + - godeps: updates the bazel Go dependency targets based on go.mod changes. + - rustdeps: updates the cargo bazel lockfile. + - all: catch-all for the above + +If no categories are referenced, then 'builds' is assumed as the default.`, + Before: func(ctx *cli.Context) error { + for _, arg := range ctx.Args().Slice() { + if _, ok := bzlgenTargets[arg]; !ok && arg != "all" { + cli.HandleExitCoder(errors.Errorf("category doesn't exist %q, run `sg bazel configure --help` for full info.", arg)) + cli.ShowSubcommandHelpAndExit(ctx, 1) + return nil + } + } + return nil + }, + Action: func(ctx *cli.Context) error { + var categories []bzlgenTarget + var categoryNames []string + if slices.Contains(ctx.Args().Slice(), "all") { + categories = maps.Values(bzlgenTargets) + categoryNames = maps.Keys(bzlgenTargets) + } else if ctx.NArg() == 0 { + categories = []bzlgenTarget{bzlgenTargets["builds"]} + categoryNames = []string{"builds"} + } else { + for i := 0; i < ctx.NArg(); i++ { + categories = append(categories, bzlgenTargets[ctx.Args().Get(i)]) + categoryNames = append(categoryNames, ctx.Args().Get(i)) + } + } + + slices.SortFunc(categories, func(a, b bzlgenTarget) bool { + return a.order < b.order + }) + + std.Out.WriteLine(output.Emojif(output.EmojiAsterisk, "Invoking the following Bazel generating categories: %s", strings.Join(categoryNames, ", "))) + + for _, c := range categories { + root, err := root.RepositoryRoot() + if err != nil { + return err + } + + std.Out.WriteNoticef("running command %q", strings.Join(append([]string{"bazel", c.cmd}, c.args...), " ")) + if c.protip != "" { + std.Out.WriteLine(output.Emojif(output.EmojiLightbulb, "pro-tip: %s", c.protip)) + } + + args := append([]string{c.cmd, "--noshow_progress"}, c.args...) + cmd := exec.CommandContext(ctx.Context, "bazel", args...) + cmd.Dir = root + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = c.env + cmd.Env = append(cmd.Env, os.Environ()...) + + err = cmd.Run() + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 110 { + return nil + } else if err != nil { + return err + } + } + return nil + }, + }, + }, +} diff --git a/internal/grpc/example/weather/v1/BUILD.bazel b/internal/grpc/example/weather/v1/BUILD.bazel index 65859491c7f..214017592bc 100644 --- a/internal/grpc/example/weather/v1/BUILD.bazel +++ b/internal/grpc/example/weather/v1/BUILD.bazel @@ -12,6 +12,7 @@ go_library( proto_library( name = "grpc_example_weather_v1_proto", srcs = ["weather.proto"], + strip_import_prefix = "/internal", visibility = ["//:__subpackages__"], deps = ["@com_google_protobuf//:timestamp_proto"], ) diff --git a/internal/grpc/testprotos/news/v1/BUILD.bazel b/internal/grpc/testprotos/news/v1/BUILD.bazel index 511735c2ee8..b47ce8e1e7b 100644 --- a/internal/grpc/testprotos/news/v1/BUILD.bazel +++ b/internal/grpc/testprotos/news/v1/BUILD.bazel @@ -5,6 +5,7 @@ load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") proto_library( name = "news_proto", srcs = ["news.proto"], + strip_import_prefix = "/internal", visibility = ["//:__subpackages__"], deps = ["@com_google_protobuf//:timestamp_proto"], ) diff --git a/shell.nix b/shell.nix index b3914114f09..9563b1b0a80 100644 --- a/shell.nix +++ b/shell.nix @@ -27,9 +27,6 @@ let exec ${pkgs.bazelisk}/bin/bazelisk "$@" '' else '' unset TMPDIR TMP - if [ "$1" == "configure" ]; then - exec env --unset=USE_BAZEL_VERSION ${pkgs.bazelisk}/bin/bazelisk "$@" - fi exec ${pkgs.bazel_7}/bin/bazel "$@" ''); bazel-watcher = writeShellScriptBin "ibazel" '' @@ -112,13 +109,14 @@ mkShell.override { stdenv = if hostPlatform.isMacOS then pkgs.clang11Stdenv else rustfmt libiconv clippy + + bazel-buildtools ] ++ lib.optional hostPlatform.isLinux (with pkgs; [ # bazel via nix is broken on MacOS for us. Lets just rely on bazelisk from brew. # special sauce bazel stuff. bazelisk # needed to please sg, but not used directly by us bazel-fhs bazel-watcher - bazel-buildtools ]) ++ lib.optional hostPlatform.isMacOS [ bazel-wrapper ]; # Startup postgres, redis & set nixos specific stuff