From 1dd80520dbfe34c454edec5e7ccc5bd691b41f7b Mon Sep 17 00:00:00 2001
From: Robert Lin <robert@bobheadxi.dev>
Date: Wed, 13 Apr 2022 09:53:10 -0700
Subject: [PATCH] dev/ci: only asdf install go in pipeline gen (#33853)

Cuts the pipeline generation step from ~60s to ~25s - doing a fresh asdf install golang seems significantly faster than using the asdf tools cache.

Together with #33821 , this significantly reduces the time it takes for builds to get to an "actually running" state (~120s -> ~50s)
---
 .buildkite/hooks/pre-command | 12 +++++++++---
 .buildkite/pipeline.yml      |  1 +
 dev/ci/asdf-install.sh       |  4 ++--
 dev/ci/go-backcompat/test.sh |  2 ++
 4 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 83b71b2866b..aecfd2965fa 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -23,8 +23,14 @@ mkdir -p ./annotations/
 
 # asdf setup
 # ----------
-if [[ ! "$BUILDKITE_STEP_KEY" == "pipeline-upload" ]]; then
-  ./dev/ci/asdf-install.sh
+echo "~~~ Preparing asdf dependencies"
+
+if [[ "$BUILDKITE_STEP_KEY" == "pipeline-gen" ]]; then
+  echo "pipeline-gen step: running go install only"
+  asdf install golang
+elif [[ "$BUILDKITE_STEP_KEY" == "pipeline-upload" ]]; then
+  echo "pipeline-upload step: skipping asdf install"
 else
-  echo "skipping asdf install for pipeline upload setp"
+  echo "running normal install"
+  ./dev/ci/asdf-install.sh
 fi
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index c0a3782c663..d2e9f47be60 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -11,6 +11,7 @@ steps:
   - group: "Pipeline setup"
     steps:
       - label: ':hammer_and_wrench: :pipeline: Generate pipeline'
+        key: 'pipeline-gen'
         agents: { queue: standard }
         # Prioritize generating pipelines so that jobs can get generated and queued up as soon
         # as possible, so as to better assess pipeline load e.g. to scale the Buildkite fleet.
diff --git a/dev/ci/asdf-install.sh b/dev/ci/asdf-install.sh
index c4fe8940148..f362a6c4357 100755
--- a/dev/ci/asdf-install.sh
+++ b/dev/ci/asdf-install.sh
@@ -5,13 +5,13 @@
 # In most cases you should not need to call this script directly.
 if [[ ! "$BUILDKITE" == "true" ]]; then
   # Not-in-buildkite simple install.
-  echo "~~~ asdf install"
+  echo "asdf install"
   asdf install
   echo "done installing"
   # We can't use exit 0 here, it would prevent the variables to be exported (that's a particular buildkite hook peculiarity).
 else
   # We need awscli to use asdf cache
-  echo "~~~ asdf install from cache"
+  echo "asdf install from cache"
   asdf install awscli
   echo "done installing awscli"
 
diff --git a/dev/ci/go-backcompat/test.sh b/dev/ci/go-backcompat/test.sh
index 35dd49ccf98..730784e33a0 100755
--- a/dev/ci/go-backcompat/test.sh
+++ b/dev/ci/go-backcompat/test.sh
@@ -132,8 +132,10 @@ fi
 
 # Re-run asdf to ensure we have the correct set of utilities to
 # run the currently checked out version of the Go unit tests.
+echo "--- asdf install checked out tools"
 ./dev/ci/asdf-install.sh
 
+echo "--- run tests"
 if ! ./dev/ci/go-test.sh "$@"; then
   annotation=$(
     cat <<EOF