sg: add commands to wrap common bazel generating commands (#59833)

### The Old

Replaces `aspect configure` (used via `bazel configure`) which had a few issues:
- Uses host Go toolchain instead of bazel-managed one
- Didnt use our //:gazelle binary, which is configured with `rules_buf` gazelle rules
- Not able to configure other gazelle rules outside of the fixed set that comes with aspect cli

### The New

`sg bazel configure` and its targets (currently `builds`, `godeps` and `rustdeps`). Passing none only runs `sg bazel configure builds`, the pseudo-target `all` runs all of them.

`sg bazel [other arg]` passes to `bazel` underneath, as a convenience option for if people get more used to running `sg bazel ...` than `bazel` directly (or from shell history)

```
$ bazel-bin/dev/sg/sg_/sg bazel help configure
NAME:
   sg bazel configure - Wrappers around some commands to generate various files required by Bazel

USAGE:
   sg bazel configure [category...]

DESCRIPTION:
   For convenience, a number of Bazel commands are wrapped by this command to update various files required by Bazel.

   Available categories:
     - builds: updates BUILD.bazel files for Go & Typescript targets.
     - godeps: updates the bazel Go dependency targets based on go.mod changes.
     - rustdeps: updates the cargo bazel lockfile.
     - all: catch-all for the above

   If no categories are referenced, then 'builds' is assumed as the default.

OPTIONS:
   --help, -h  show help
```

```
$ bazel-bin/dev/sg/sg_/sg bazel help
Additional commands from sg:
  configure           Wrappers around some commands to generate various files required by Bazel
                                               [bazel release 7.0.0- (@non-git)]
Usage: bazel <command> <options> ...

Available commands:
  analyze-profile     Analyzes build profile data.
  aquery              Analyzes the given targets and queries the action graph.
  build               Builds the specified targets.
  canonicalize-flags  Canonicalizes a list of bazel options.
  clean               Removes output files and optionally stops the server.
  coverage            Generates code coverage report for specified test targets.
  cquery              Loads, analyzes, and queries the specified targets w/ configurations.
  dump                Dumps the internal state of the bazel server process.
  fetch               Fetches external repositories that are prerequisites to the targets.
  help                Prints help for commands, or the index.
  info                Displays runtime info about the bazel server.
  license             Prints the license of this software.
  mobile-install      Installs targets to mobile devices.
  mod                 Queries the Bzlmod external dependency graph
  print_action        Prints the command line args for compiling a file.
  query               Executes a dependency graph query.
  run                 Runs the specified target.
  shutdown            Stops the bazel server.
  sync                Syncs all repositories specified in the workspace file
  test                Builds and runs the specified test targets.
  version             Prints version information for bazel.

Getting more help:
  bazel help <command>
                   Prints help and options for <command>.
  bazel help startup_options
                   Options for the JVM hosting bazel.
  bazel help target-syntax
                   Explains the syntax for specifying targets.
  bazel help info-keys
                   Displays a list of keys used by the info command.
```

## Test plan

`bazel run //dev/sg:sg -- bazel configure` and others

BUILD.bazel

@@ -205,22 +205,24 @@ buildifier(
 # Go
 
 gazelle_binary(
-    name = "gazelle-buf",
+    name = "gazelle-bin",
     languages = [
         # Loads the native proto extension
         "@bazel_gazelle//language/proto:go_default_library",
         # Gazelle-buf does not include the Go plugin by default, so we have to add it
         # ourselves.
         "@bazel_gazelle//language/go:go_default_library",
+        # Bundled with aspect-cli, but we're missing out on buf that way
+        "@aspect_cli//gazelle/js:js",
         # Loads the Buf extension
-        "@rules_buf//gazelle/buf:buf",
         # NOTE: This needs to be loaded after the proto language
+        "@rules_buf//gazelle/buf:buf",
     ],
 )
 
 gazelle(
     name = "gazelle",
-    gazelle = ":gazelle-buf",
+    gazelle = ":gazelle-bin",
 )
 
 sh_binary(

WORKSPACE

@@ -141,6 +141,16 @@ http_archive(
     urls = ["https://github.com/keith/buildifier-prebuilt/archive/6.1.0.tar.gz"],
 )
 
+http_archive(
+    name = "aspect_cli",
+    repo_mapping = {
+        "@com_github_smacker_go_tree_sitter": "@aspectcli-com_github_smacker_go_tree_sitter",
+    },
+    sha256 = "045f0186edb25706dfe77d9c4916eec630a2b2736f9abb59e37eaac122d4b771",
+    strip_prefix = "aspect-cli-5.8.20",
+    url = "https://github.com/aspect-build/aspect-cli/archive/5.8.20.tar.gz",
+)
+
 # hermetic_cc_toolchain setup ================================
 HERMETIC_CC_TOOLCHAIN_VERSION = "v2.2.1"
 
@@ -271,9 +281,9 @@ go_repository(
     name = "org_golang_google_protobuf",
     build_file_proto_mode = "disable_global",
     importpath = "google.golang.org/protobuf",
-    sum = "h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=",
-    version = "v1.31.1",
-)  # keep
+    sum = "h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=",
+    version = "v1.32.0",
+)
 
 # Pin protoc-gen-go-grpc to 1.3.0
 # See also //:gen-go-grpc
@@ -285,6 +295,20 @@ go_repository(
     version = "v1.3.0",
 )  # keep
 
+# Pin specific version for aspect-cli's gazelle rules, with versions
+# that it requires but that our codebase doesnt support.
+go_repository(
+    name = "aspectcli-com_github_smacker_go_tree_sitter",
+    build_file_proto_mode = "disable_global",
+    importpath = "github.com/smacker/go-tree-sitter",
+    sum = "h1:DxgjlvWYsb80WEN2Zv3WqJFAg2DKjUQJO6URGdf1x6Y=",
+    version = "v0.0.0-20230720070738-0d0a9f78d8f8",
+)  # keep
+
+load("@aspect_cli//:go.bzl", aspect_cli_deps = "deps")
+
+aspect_cli_deps()
+
 # gazelle:repository_macro deps.bzl%go_dependencies
 go_dependencies()
 

dev/ci/bazel-configure.sh

@@ -8,8 +8,6 @@ PATH="$(dirname "${runfiles_dir}/${GO}"):${PATH}"
 # Remove bazelisk from path
 PATH=$(echo "${PATH}" | awk -v RS=: -v ORS=: '/bazelisk/ {next} {print}')
 export PATH
-# Allow Aspect to re-enter again
-export ASPECT_REENTRANT=
 
 cd "${BUILD_WORKSPACE_DIRECTORY}"
 
@@ -17,7 +15,7 @@ bazel \
   --bazelrc=.bazelrc \
   --bazelrc=.aspect/bazelrc/ci.bazelrc \
   --bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc \
-  configure
+  run //:gazelle
 
 if [ "${CI:-}" ]; then
   git ls-files --exclude-standard --others | xargs git add --intent-to-add || true

dev/ci/bazel-prechecks.sh

@@ -30,8 +30,8 @@ function generate_diff_artifact() {
 
 trap generate_diff_artifact EXIT
 
-echo "--- :bazel: Running bazel configure"
-bazel "${bazelrc[@]}" configure
+echo "--- :bazel: Running bazel run //:gazelle"
+bazel "${bazelrc[@]}" run //:gazelle
 
 echo "--- Checking if BUILD.bazel files were updated"
 # Account for the possibility of a BUILD.bazel to be totally new, and thus untracked.
@@ -39,7 +39,7 @@ git ls-files --exclude-standard --others | grep BUILD.bazel | xargs git add --in
 
 git diff --exit-code || EXIT_CODE=$? # do not fail on non-zero exit
 
-# if we get a non-zero exit code, bazel configure updated files
+# if we get a non-zero exit code, bazel run //:gazelle updated files
 if [[ $EXIT_CODE -ne 0 ]]; then
   mkdir -p ./annotations
   cat <<-'END' > ./annotations/bazel-prechecks.md
@@ -48,7 +48,7 @@ if [[ $EXIT_CODE -ne 0 ]]; then
   BUILD.bazel files need to be updated to match the repository state. You should run the following command and commit the result
 
   ```
-  bazel configure
+  sg bazel configure
   ```
 
   #### For more information please see the [Bazel FAQ](https://docs.sourcegraph.com/dev/background-information/bazel/faq)
@@ -63,7 +63,7 @@ bazel "${bazelrc[@]}" run //:gazelle-update-repos
 echo "--- Checking if deps.bzl was updated"
 git diff --exit-code || EXIT_CODE=$? # do not fail on non-zero exit
 
-# if we get a non-zero exit code, bazel configure updated files
+# if we get a non-zero exit code, bazel run //:gazelle-update-repos updated files
 if [[ $EXIT_CODE -ne 0 ]]; then
   mkdir -p ./annotations
   cat <<-'END' > ./annotations/bazel-prechecks.md
@@ -72,7 +72,7 @@ if [[ $EXIT_CODE -ne 0 ]]; then
   `deps.bzl` needs to be updated to match the repository state. You should run the following command and commit the result
 
   ```
-  bazel run //:gazelle-update-repos
+  sg bazel configure godeps
   ```
 
   #### For more information please see the [Bazel FAQ](https://docs.sourcegraph.com/dev/background-information/bazel/faq)

dev/sg/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "os.go",
         "sg_analytics.go",
         "sg_audit.go",
+        "sg_bazel.go",
         "sg_cloud.go",
         "sg_db.go",
         "sg_deploy.go",
@@ -115,6 +116,8 @@ go_library(
         "@in_gopkg_yaml_v3//:yaml_v3",
         "@io_opentelemetry_go_otel//attribute",
         "@io_opentelemetry_go_otel_trace//:trace",
+        "@org_golang_x_exp//maps",
+        "@org_golang_x_exp//slices",
         "@org_golang_x_mod//semver",
         "@org_golang_x_oauth2//:oauth2",
         "@org_golang_x_text//cases",

dev/sg/main.go

@@ -266,6 +266,7 @@ var sg = &cli.App{
 		testCommand,
 		lintCommand,
 		generateCommand,
+		bazelCommand,
 		dbCommand,
 		migrationCommand,
 		insightsCommand,

dev/sg/sg_bazel.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+
+	"github.com/sourcegraph/sourcegraph/dev/sg/internal/category"
+	"github.com/sourcegraph/sourcegraph/dev/sg/internal/std"
+	"github.com/sourcegraph/sourcegraph/dev/sg/root"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+	"github.com/sourcegraph/sourcegraph/lib/output"
+)
+
+type bzlgenTarget struct {
+	order  int
+	cmd    string
+	args   []string
+	env    []string
+	protip string
+}
+
+var bzlgenTargets = map[string]bzlgenTarget{
+	"builds": {
+		order: 1,
+		cmd:   "run",
+		args:  []string{"//:gazelle"},
+	},
+	"godeps": {
+		cmd:  "run",
+		args: []string{"//:gazelle-update-repos"},
+	},
+	"rustdeps": {
+		cmd:    "sync",
+		args:   []string{"--only=crate_index"},
+		env:    []string{"CARGO_BAZEL_REPIN=1"},
+		protip: "run with CARGO_BAZEL_ISOLATED=0 for faster (but less sandboxed) repinning.",
+	},
+}
+
+var bazelCommand = &cli.Command{
+	Name:            "bazel",
+	SkipFlagParsing: true,
+	Usage:           "placeholder, handled in top-level Action below",
+	Category:        category.Dev,
+	Action: func(ctx *cli.Context) error {
+		root, err := root.RepositoryRoot()
+		if err != nil {
+			return err
+		}
+
+		if slices.Equal(ctx.Args().Slice(), []string{"help"}) || slices.Equal(ctx.Args().Slice(), []string{"--help"}) || slices.Equal(ctx.Args().Slice(), []string{"-h"}) {
+			fmt.Println("Additional commands from sg:")
+			fmt.Println("  configure           Wrappers around some commands to generate various files required by Bazel")
+		}
+
+		cmd := exec.CommandContext(ctx.Context, "bazel", ctx.Args().Slice()...)
+		cmd.Dir = root
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = os.Stdin
+		return cmd.Run()
+	},
+	Subcommands: []*cli.Command{
+		{
+			Name:      "configure",
+			Usage:     "Wrappers around some commands to generate various files required by Bazel",
+			UsageText: "sg bazel configure [category...]",
+			Description: `For convenience, a number of Bazel commands are wrapped by this command to update various files required by Bazel.
+
+Available categories:
+	- builds: updates BUILD.bazel files for Go & Typescript targets.
+	- godeps: updates the bazel Go dependency targets based on go.mod changes.
+	- rustdeps: updates the cargo bazel lockfile.
+	- all: catch-all for the above
+
+If no categories are referenced, then 'builds' is assumed as the default.`,
+			Before: func(ctx *cli.Context) error {
+				for _, arg := range ctx.Args().Slice() {
+					if _, ok := bzlgenTargets[arg]; !ok && arg != "all" {
+						cli.HandleExitCoder(errors.Errorf("category doesn't exist %q, run `sg bazel configure --help` for full info.", arg))
+						cli.ShowSubcommandHelpAndExit(ctx, 1)
+						return nil
+					}
+				}
+				return nil
+			},
+			Action: func(ctx *cli.Context) error {
+				var categories []bzlgenTarget
+				var categoryNames []string
+				if slices.Contains(ctx.Args().Slice(), "all") {
+					categories = maps.Values(bzlgenTargets)
+					categoryNames = maps.Keys(bzlgenTargets)
+				} else if ctx.NArg() == 0 {
+					categories = []bzlgenTarget{bzlgenTargets["builds"]}
+					categoryNames = []string{"builds"}
+				} else {
+					for i := 0; i < ctx.NArg(); i++ {
+						categories = append(categories, bzlgenTargets[ctx.Args().Get(i)])
+						categoryNames = append(categoryNames, ctx.Args().Get(i))
+					}
+				}
+
+				slices.SortFunc(categories, func(a, b bzlgenTarget) bool {
+					return a.order < b.order
+				})
+
+				std.Out.WriteLine(output.Emojif(output.EmojiAsterisk, "Invoking the following Bazel generating categories: %s", strings.Join(categoryNames, ", ")))
+
+				for _, c := range categories {
+					root, err := root.RepositoryRoot()
+					if err != nil {
+						return err
+					}
+
+					std.Out.WriteNoticef("running command %q", strings.Join(append([]string{"bazel", c.cmd}, c.args...), " "))
+					if c.protip != "" {
+						std.Out.WriteLine(output.Emojif(output.EmojiLightbulb, "pro-tip: %s", c.protip))
+					}
+
+					args := append([]string{c.cmd, "--noshow_progress"}, c.args...)
+					cmd := exec.CommandContext(ctx.Context, "bazel", args...)
+					cmd.Dir = root
+					cmd.Stdout = os.Stdout
+					cmd.Stderr = os.Stderr
+					cmd.Env = c.env
+					cmd.Env = append(cmd.Env, os.Environ()...)
+
+					err = cmd.Run()
+					var exitErr *exec.ExitError
+					if errors.As(err, &exitErr) && exitErr.ExitCode() == 110 {
+						return nil
+					} else if err != nil {
+						return err
+					}
+				}
+				return nil
+			},
+		},
+	},
+}

internal/grpc/example/weather/v1/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
 proto_library(
     name = "grpc_example_weather_v1_proto",
     srcs = ["weather.proto"],
+    strip_import_prefix = "/internal",
     visibility = ["//:__subpackages__"],
     deps = ["@com_google_protobuf//:timestamp_proto"],
 )

internal/grpc/testprotos/news/v1/BUILD.bazel

@@ -5,6 +5,7 @@ load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 proto_library(
     name = "news_proto",
     srcs = ["news.proto"],
+    strip_import_prefix = "/internal",
     visibility = ["//:__subpackages__"],
     deps = ["@com_google_protobuf//:timestamp_proto"],
 )

shell.nix

@@ -27,9 +27,6 @@ let
     exec ${pkgs.bazelisk}/bin/bazelisk "$@"
   '' else ''
     unset TMPDIR TMP
-    if [ "$1" == "configure" ]; then
-      exec env --unset=USE_BAZEL_VERSION ${pkgs.bazelisk}/bin/bazelisk "$@"
-    fi
     exec ${pkgs.bazel_7}/bin/bazel "$@"
   '');
   bazel-watcher = writeShellScriptBin "ibazel" ''
@@ -112,13 +109,14 @@ mkShell.override { stdenv = if hostPlatform.isMacOS then pkgs.clang11Stdenv else
     rustfmt
     libiconv
     clippy
+
+    bazel-buildtools
   ] ++ lib.optional hostPlatform.isLinux (with pkgs; [
     # bazel via nix is broken on MacOS for us. Lets just rely on bazelisk from brew.
     # special sauce bazel stuff.
     bazelisk # needed to please sg, but not used directly by us
     bazel-fhs
     bazel-watcher
-    bazel-buildtools
   ]) ++ lib.optional hostPlatform.isMacOS [ bazel-wrapper ];
 
   # Startup postgres, redis & set nixos specific stuff