From 7e98cb98eef512e54916cd9a05e8f288b2fef3e1 Mon Sep 17 00:00:00 2001 From: Noah S-C Date: Wed, 31 Jan 2024 16:54:04 +0000 Subject: [PATCH] sg: add commands to wrap common bazel generating commands (#59833) ### The Old Replaces `aspect configure` (used via `bazel configure`) which had a few issues: - Uses host Go toolchain instead of bazel-managed one - Didnt use our //:gazelle binary, which is configured with `rules_buf` gazelle rules - Not able to configure other gazelle rules outside of the fixed set that comes with aspect cli ### The New `sg bazel configure` and its targets (currently `builds`, `godeps` and `rustdeps`). Passing none only runs `sg bazel configure builds`, the pseudo-target `all` runs all of them. `sg bazel [other arg]` passes to `bazel` underneath, as a convenience option for if people get more used to running `sg bazel ...` than `bazel` directly (or from shell history) ``` $ bazel-bin/dev/sg/sg_/sg bazel help configure NAME: sg bazel configure - Wrappers around some commands to generate various files required by Bazel USAGE: sg bazel configure [category...] DESCRIPTION: For convenience, a number of Bazel commands are wrapped by this command to update various files required by Bazel. Available categories: - builds: updates BUILD.bazel files for Go & Typescript targets. - godeps: updates the bazel Go dependency targets based on go.mod changes. - rustdeps: updates the cargo bazel lockfile. - all: catch-all for the above If no categories are referenced, then 'builds' is assumed as the default. OPTIONS: --help, -h show help ``` ``` $ bazel-bin/dev/sg/sg_/sg bazel help Additional commands from sg: configure Wrappers around some commands to generate various files required by Bazel [bazel release 7.0.0- (@non-git)] Usage: bazel ... Available commands: analyze-profile Analyzes build profile data. aquery Analyzes the given targets and queries the action graph. build Builds the specified targets. canonicalize-flags Canonicalizes a list of bazel options. clean Removes output files and optionally stops the server. coverage Generates code coverage report for specified test targets. cquery Loads, analyzes, and queries the specified targets w/ configurations. dump Dumps the internal state of the bazel server process. fetch Fetches external repositories that are prerequisites to the targets. help Prints help for commands, or the index. info Displays runtime info about the bazel server. license Prints the license of this software. mobile-install Installs targets to mobile devices. mod Queries the Bzlmod external dependency graph print_action Prints the command line args for compiling a file. query Executes a dependency graph query. run Runs the specified target. shutdown Stops the bazel server. sync Syncs all repositories specified in the workspace file test Builds and runs the specified test targets. version Prints version information for bazel. Getting more help: bazel help Prints help and options for . bazel help startup_options Options for the JVM hosting bazel. bazel help target-syntax Explains the syntax for specifying targets. bazel help info-keys Displays a list of keys used by the info command. ``` ## Test plan `bazel run //dev/sg:sg -- bazel configure` and others --- BUILD.bazel | 8 +- WORKSPACE | 30 +++- dev/ci/bazel-configure.sh | 4 +- dev/ci/bazel-prechecks.sh | 12 +- dev/sg/BUILD.bazel | 3 + dev/sg/main.go | 1 + dev/sg/sg_bazel.go | 146 +++++++++++++++++++ internal/grpc/example/weather/v1/BUILD.bazel | 1 + internal/grpc/testprotos/news/v1/BUILD.bazel | 1 + shell.nix | 6 +- 10 files changed, 193 insertions(+), 19 deletions(-) create mode 100644 dev/sg/sg_bazel.go diff --git a/BUILD.bazel b/BUILD.bazel index ac37a2a0490..13cc4575137 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -205,22 +205,24 @@ buildifier( # Go gazelle_binary( - name = "gazelle-buf", + name = "gazelle-bin", languages = [ # Loads the native proto extension "@bazel_gazelle//language/proto:go_default_library", # Gazelle-buf does not include the Go plugin by default, so we have to add it # ourselves. "@bazel_gazelle//language/go:go_default_library", + # Bundled with aspect-cli, but we're missing out on buf that way + "@aspect_cli//gazelle/js:js", # Loads the Buf extension - "@rules_buf//gazelle/buf:buf", # NOTE: This needs to be loaded after the proto language + "@rules_buf//gazelle/buf:buf", ], ) gazelle( name = "gazelle", - gazelle = ":gazelle-buf", + gazelle = ":gazelle-bin", ) sh_binary( diff --git a/WORKSPACE b/WORKSPACE index 970a48a8c7b..da640ee6bd8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -141,6 +141,16 @@ http_archive( urls = ["https://github.com/keith/buildifier-prebuilt/archive/6.1.0.tar.gz"], ) +http_archive( + name = "aspect_cli", + repo_mapping = { + "@com_github_smacker_go_tree_sitter": "@aspectcli-com_github_smacker_go_tree_sitter", + }, + sha256 = "045f0186edb25706dfe77d9c4916eec630a2b2736f9abb59e37eaac122d4b771", + strip_prefix = "aspect-cli-5.8.20", + url = "https://github.com/aspect-build/aspect-cli/archive/5.8.20.tar.gz", +) + # hermetic_cc_toolchain setup ================================ HERMETIC_CC_TOOLCHAIN_VERSION = "v2.2.1" @@ -271,9 +281,9 @@ go_repository( name = "org_golang_google_protobuf", build_file_proto_mode = "disable_global", importpath = "google.golang.org/protobuf", - sum = "h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=", - version = "v1.31.1", -) # keep + sum = "h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=", + version = "v1.32.0", +) # Pin protoc-gen-go-grpc to 1.3.0 # See also //:gen-go-grpc @@ -285,6 +295,20 @@ go_repository( version = "v1.3.0", ) # keep +# Pin specific version for aspect-cli's gazelle rules, with versions +# that it requires but that our codebase doesnt support. +go_repository( + name = "aspectcli-com_github_smacker_go_tree_sitter", + build_file_proto_mode = "disable_global", + importpath = "github.com/smacker/go-tree-sitter", + sum = "h1:DxgjlvWYsb80WEN2Zv3WqJFAg2DKjUQJO6URGdf1x6Y=", + version = "v0.0.0-20230720070738-0d0a9f78d8f8", +) # keep + +load("@aspect_cli//:go.bzl", aspect_cli_deps = "deps") + +aspect_cli_deps() + # gazelle:repository_macro deps.bzl%go_dependencies go_dependencies() diff --git a/dev/ci/bazel-configure.sh b/dev/ci/bazel-configure.sh index 9a01a54b921..5d7c36fcf1c 100755 --- a/dev/ci/bazel-configure.sh +++ b/dev/ci/bazel-configure.sh @@ -8,8 +8,6 @@ PATH="$(dirname "${runfiles_dir}/${GO}"):${PATH}" # Remove bazelisk from path PATH=$(echo "${PATH}" | awk -v RS=: -v ORS=: '/bazelisk/ {next} {print}') export PATH -# Allow Aspect to re-enter again -export ASPECT_REENTRANT= cd "${BUILD_WORKSPACE_DIRECTORY}" @@ -17,7 +15,7 @@ bazel \ --bazelrc=.bazelrc \ --bazelrc=.aspect/bazelrc/ci.bazelrc \ --bazelrc=.aspect/bazelrc/ci.sourcegraph.bazelrc \ - configure + run //:gazelle if [ "${CI:-}" ]; then git ls-files --exclude-standard --others | xargs git add --intent-to-add || true diff --git a/dev/ci/bazel-prechecks.sh b/dev/ci/bazel-prechecks.sh index 013f6f04a7b..3b30f2a07fd 100755 --- a/dev/ci/bazel-prechecks.sh +++ b/dev/ci/bazel-prechecks.sh @@ -30,8 +30,8 @@ function generate_diff_artifact() { trap generate_diff_artifact EXIT -echo "--- :bazel: Running bazel configure" -bazel "${bazelrc[@]}" configure +echo "--- :bazel: Running bazel run //:gazelle" +bazel "${bazelrc[@]}" run //:gazelle echo "--- Checking if BUILD.bazel files were updated" # Account for the possibility of a BUILD.bazel to be totally new, and thus untracked. @@ -39,7 +39,7 @@ git ls-files --exclude-standard --others | grep BUILD.bazel | xargs git add --in git diff --exit-code || EXIT_CODE=$? # do not fail on non-zero exit -# if we get a non-zero exit code, bazel configure updated files +# if we get a non-zero exit code, bazel run //:gazelle updated files if [[ $EXIT_CODE -ne 0 ]]; then mkdir -p ./annotations cat <<-'END' > ./annotations/bazel-prechecks.md @@ -48,7 +48,7 @@ if [[ $EXIT_CODE -ne 0 ]]; then BUILD.bazel files need to be updated to match the repository state. You should run the following command and commit the result ``` - bazel configure + sg bazel configure ``` #### For more information please see the [Bazel FAQ](https://docs.sourcegraph.com/dev/background-information/bazel/faq) @@ -63,7 +63,7 @@ bazel "${bazelrc[@]}" run //:gazelle-update-repos echo "--- Checking if deps.bzl was updated" git diff --exit-code || EXIT_CODE=$? # do not fail on non-zero exit -# if we get a non-zero exit code, bazel configure updated files +# if we get a non-zero exit code, bazel run //:gazelle-update-repos updated files if [[ $EXIT_CODE -ne 0 ]]; then mkdir -p ./annotations cat <<-'END' > ./annotations/bazel-prechecks.md @@ -72,7 +72,7 @@ if [[ $EXIT_CODE -ne 0 ]]; then `deps.bzl` needs to be updated to match the repository state. You should run the following command and commit the result ``` - bazel run //:gazelle-update-repos + sg bazel configure godeps ``` #### For more information please see the [Bazel FAQ](https://docs.sourcegraph.com/dev/background-information/bazel/faq) diff --git a/dev/sg/BUILD.bazel b/dev/sg/BUILD.bazel index a6c3e3729e6..56c619ba1ac 100644 --- a/dev/sg/BUILD.bazel +++ b/dev/sg/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "os.go", "sg_analytics.go", "sg_audit.go", + "sg_bazel.go", "sg_cloud.go", "sg_db.go", "sg_deploy.go", @@ -115,6 +116,8 @@ go_library( "@in_gopkg_yaml_v3//:yaml_v3", "@io_opentelemetry_go_otel//attribute", "@io_opentelemetry_go_otel_trace//:trace", + "@org_golang_x_exp//maps", + "@org_golang_x_exp//slices", "@org_golang_x_mod//semver", "@org_golang_x_oauth2//:oauth2", "@org_golang_x_text//cases", diff --git a/dev/sg/main.go b/dev/sg/main.go index 61145991727..2bcf06f341a 100644 --- a/dev/sg/main.go +++ b/dev/sg/main.go @@ -266,6 +266,7 @@ var sg = &cli.App{ testCommand, lintCommand, generateCommand, + bazelCommand, dbCommand, migrationCommand, insightsCommand, diff --git a/dev/sg/sg_bazel.go b/dev/sg/sg_bazel.go new file mode 100644 index 00000000000..87ad5054f4b --- /dev/null +++ b/dev/sg/sg_bazel.go @@ -0,0 +1,146 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/urfave/cli/v2" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" + + "github.com/sourcegraph/sourcegraph/dev/sg/internal/category" + "github.com/sourcegraph/sourcegraph/dev/sg/internal/std" + "github.com/sourcegraph/sourcegraph/dev/sg/root" + "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/lib/output" +) + +type bzlgenTarget struct { + order int + cmd string + args []string + env []string + protip string +} + +var bzlgenTargets = map[string]bzlgenTarget{ + "builds": { + order: 1, + cmd: "run", + args: []string{"//:gazelle"}, + }, + "godeps": { + cmd: "run", + args: []string{"//:gazelle-update-repos"}, + }, + "rustdeps": { + cmd: "sync", + args: []string{"--only=crate_index"}, + env: []string{"CARGO_BAZEL_REPIN=1"}, + protip: "run with CARGO_BAZEL_ISOLATED=0 for faster (but less sandboxed) repinning.", + }, +} + +var bazelCommand = &cli.Command{ + Name: "bazel", + SkipFlagParsing: true, + Usage: "placeholder, handled in top-level Action below", + Category: category.Dev, + Action: func(ctx *cli.Context) error { + root, err := root.RepositoryRoot() + if err != nil { + return err + } + + if slices.Equal(ctx.Args().Slice(), []string{"help"}) || slices.Equal(ctx.Args().Slice(), []string{"--help"}) || slices.Equal(ctx.Args().Slice(), []string{"-h"}) { + fmt.Println("Additional commands from sg:") + fmt.Println(" configure Wrappers around some commands to generate various files required by Bazel") + } + + cmd := exec.CommandContext(ctx.Context, "bazel", ctx.Args().Slice()...) + cmd.Dir = root + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + return cmd.Run() + }, + Subcommands: []*cli.Command{ + { + Name: "configure", + Usage: "Wrappers around some commands to generate various files required by Bazel", + UsageText: "sg bazel configure [category...]", + Description: `For convenience, a number of Bazel commands are wrapped by this command to update various files required by Bazel. + +Available categories: + - builds: updates BUILD.bazel files for Go & Typescript targets. + - godeps: updates the bazel Go dependency targets based on go.mod changes. + - rustdeps: updates the cargo bazel lockfile. + - all: catch-all for the above + +If no categories are referenced, then 'builds' is assumed as the default.`, + Before: func(ctx *cli.Context) error { + for _, arg := range ctx.Args().Slice() { + if _, ok := bzlgenTargets[arg]; !ok && arg != "all" { + cli.HandleExitCoder(errors.Errorf("category doesn't exist %q, run `sg bazel configure --help` for full info.", arg)) + cli.ShowSubcommandHelpAndExit(ctx, 1) + return nil + } + } + return nil + }, + Action: func(ctx *cli.Context) error { + var categories []bzlgenTarget + var categoryNames []string + if slices.Contains(ctx.Args().Slice(), "all") { + categories = maps.Values(bzlgenTargets) + categoryNames = maps.Keys(bzlgenTargets) + } else if ctx.NArg() == 0 { + categories = []bzlgenTarget{bzlgenTargets["builds"]} + categoryNames = []string{"builds"} + } else { + for i := 0; i < ctx.NArg(); i++ { + categories = append(categories, bzlgenTargets[ctx.Args().Get(i)]) + categoryNames = append(categoryNames, ctx.Args().Get(i)) + } + } + + slices.SortFunc(categories, func(a, b bzlgenTarget) bool { + return a.order < b.order + }) + + std.Out.WriteLine(output.Emojif(output.EmojiAsterisk, "Invoking the following Bazel generating categories: %s", strings.Join(categoryNames, ", "))) + + for _, c := range categories { + root, err := root.RepositoryRoot() + if err != nil { + return err + } + + std.Out.WriteNoticef("running command %q", strings.Join(append([]string{"bazel", c.cmd}, c.args...), " ")) + if c.protip != "" { + std.Out.WriteLine(output.Emojif(output.EmojiLightbulb, "pro-tip: %s", c.protip)) + } + + args := append([]string{c.cmd, "--noshow_progress"}, c.args...) + cmd := exec.CommandContext(ctx.Context, "bazel", args...) + cmd.Dir = root + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = c.env + cmd.Env = append(cmd.Env, os.Environ()...) + + err = cmd.Run() + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 110 { + return nil + } else if err != nil { + return err + } + } + return nil + }, + }, + }, +} diff --git a/internal/grpc/example/weather/v1/BUILD.bazel b/internal/grpc/example/weather/v1/BUILD.bazel index 65859491c7f..214017592bc 100644 --- a/internal/grpc/example/weather/v1/BUILD.bazel +++ b/internal/grpc/example/weather/v1/BUILD.bazel @@ -12,6 +12,7 @@ go_library( proto_library( name = "grpc_example_weather_v1_proto", srcs = ["weather.proto"], + strip_import_prefix = "/internal", visibility = ["//:__subpackages__"], deps = ["@com_google_protobuf//:timestamp_proto"], ) diff --git a/internal/grpc/testprotos/news/v1/BUILD.bazel b/internal/grpc/testprotos/news/v1/BUILD.bazel index 511735c2ee8..b47ce8e1e7b 100644 --- a/internal/grpc/testprotos/news/v1/BUILD.bazel +++ b/internal/grpc/testprotos/news/v1/BUILD.bazel @@ -5,6 +5,7 @@ load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") proto_library( name = "news_proto", srcs = ["news.proto"], + strip_import_prefix = "/internal", visibility = ["//:__subpackages__"], deps = ["@com_google_protobuf//:timestamp_proto"], ) diff --git a/shell.nix b/shell.nix index b3914114f09..9563b1b0a80 100644 --- a/shell.nix +++ b/shell.nix @@ -27,9 +27,6 @@ let exec ${pkgs.bazelisk}/bin/bazelisk "$@" '' else '' unset TMPDIR TMP - if [ "$1" == "configure" ]; then - exec env --unset=USE_BAZEL_VERSION ${pkgs.bazelisk}/bin/bazelisk "$@" - fi exec ${pkgs.bazel_7}/bin/bazel "$@" ''); bazel-watcher = writeShellScriptBin "ibazel" '' @@ -112,13 +109,14 @@ mkShell.override { stdenv = if hostPlatform.isMacOS then pkgs.clang11Stdenv else rustfmt libiconv clippy + + bazel-buildtools ] ++ lib.optional hostPlatform.isLinux (with pkgs; [ # bazel via nix is broken on MacOS for us. Lets just rely on bazelisk from brew. # special sauce bazel stuff. bazelisk # needed to please sg, but not used directly by us bazel-fhs bazel-watcher - bazel-buildtools ]) ++ lib.optional hostPlatform.isMacOS [ bazel-wrapper ]; # Startup postgres, redis & set nixos specific stuff