From d6e053ab3d0ef7ba68efcb0189b9adb662079537 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 29 Apr 2024 18:56:18 +0200 Subject: [PATCH] chore(sg): sg bazel uses remote cache by default (unless in CI) (#62245) --- .../bazelrc/remote_cache_for_local.bazelrc | 23 +++++++++++ dev/remote_cache_local_env.sh | 10 +++++ dev/sg/sg_bazel.go | 40 +++++++++++++++++-- 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 .aspect/bazelrc/remote_cache_for_local.bazelrc create mode 100755 dev/remote_cache_local_env.sh diff --git a/.aspect/bazelrc/remote_cache_for_local.bazelrc b/.aspect/bazelrc/remote_cache_for_local.bazelrc new file mode 100644 index 00000000000..afa961b852c --- /dev/null +++ b/.aspect/bazelrc/remote_cache_for_local.bazelrc @@ -0,0 +1,23 @@ +# Only download remote outputs of top level targets to the local machine. +# Docs: https://bazel.build/reference/command-line-reference#flag--remote_download_toplevel +build --remote_download_toplevel + +# Upload locally executed action results to the remote cache. +# Docs: https://bazel.build/reference/command-line-reference#flag--remote_upload_local_results +build --remote_upload_local_results + +# Fall back to standalone local execution strategy if remote execution fails. If the grpc remote +# cache connection fails, it will fail the build, add this so it falls back to the local cache. +# Docs: https://bazel.build/reference/command-line-reference#flag--remote_local_fallback +build --remote_local_fallback + +# These likely perform faster locally than the overhead of pulling/pushing from/to the remote cache, +# as well as being able to reduce how much we push to the cache +common --modify_execution_info=CopyDirectory=+no-remote,CopyToDirectory=+no-remote,CopyFile=+no-remote + +common --credential_helper=storage.googleapis.com=%workspace%/dev/remote_cache_local_env.sh +common --remote_cache=https://storage.googleapis.com/local_bazel_remote_cache + +# If true, remote cache I/O will happen in the background instead of taking place as the part of a spawn. +# Docs: https://bazel.build/reference/command-line-reference#flag--experimental_remote_cache_async +common --experimental_remote_cache_async diff --git a/dev/remote_cache_local_env.sh b/dev/remote_cache_local_env.sh new file mode 100755 index 00000000000..18b9363f3fb --- /dev/null +++ b/dev/remote_cache_local_env.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +GCP_PROJECT="sourcegraph-local-dev" + +function emit_headers() { + echo "{\"headers\":{\"Authorization\":[\"Bearer ${1}\"]}}" +} + +emit_headers "$(gcloud --project ${GCP_PROJECT} auth print-access-token)" +exit 0 diff --git a/dev/sg/sg_bazel.go b/dev/sg/sg_bazel.go index 88d345b8207..66ad9b97625 100644 --- a/dev/sg/sg_bazel.go +++ b/dev/sg/sg_bazel.go @@ -51,13 +51,47 @@ var bazelCommand = &cli.Command{ HideHelpCommand: true, Usage: "Proxies the bazel CLI with custom commands for local dev convenience", Category: category.Dev, - Action: func(ctx *cli.Context) error { - if slices.Equal(ctx.Args().Slice(), []string{"help"}) || slices.Equal(ctx.Args().Slice(), []string{"--help"}) || slices.Equal(ctx.Args().Slice(), []string{"-h"}) { + Action: func(cctx *cli.Context) error { + if slices.Equal(cctx.Args().Slice(), []string{"help"}) || slices.Equal(cctx.Args().Slice(), []string{"--help"}) || slices.Equal(cctx.Args().Slice(), []string{"-h"}) { fmt.Println("Additional commands from sg:") fmt.Println(" configure Wrappers around some commands to generate various files required by Bazel") + fmt.Println("Additional flags from sg:") + fmt.Println(" --disable-remote-cache Disable use of the remote cache for local env.") } - cmd := exec.CommandContext(ctx.Context, "bazel", ctx.Args().Slice()...) + // Walk the args, looking for our custom flag to disable the remote cache. + // If we find it, we take not of it, but do not append it to the final args + // that will be passed to the bazel command. Everything else is passed as-is. + var disableRemoteCache bool + args := make([]string, 0, len(cctx.Args().Slice())) + for _, arg := range cctx.Args().Slice() { + switch arg { + case "--disable-remote-cache": + disableRemoteCache = true + case "--disable-remote-cache=true": + disableRemoteCache = true + case "--disable-remote-cache=false": + disableRemoteCache = false + default: + args = append(args, arg) + } + } + + // If we end up running `sg bazel` in CI, we don't want to use the remote cache for local environment, + // so we force disable the flag explicilty. + if os.Getenv("CI") == "true" || os.Getenv("BUILDKITE") == "true" { + disableRemoteCache = true + } + + if !disableRemoteCache { + newArgs := make([]string, 0, len(args)+1) + // Bazelrc flags must be added before the actual command (build, run, test ...) + newArgs = append(newArgs, "--bazelrc=.aspect/bazelrc/remote_cache_for_local.bazelrc") + newArgs = append(newArgs, args...) + args = newArgs + } + + cmd := exec.CommandContext(cctx.Context, "bazel", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdin