diff --git a/dev/ci/internal/ci/images_operations.go b/dev/ci/internal/ci/images_operations.go index 66291cf3863..4c1635f09e0 100644 --- a/dev/ci/internal/ci/images_operations.go +++ b/dev/ci/internal/ci/images_operations.go @@ -87,11 +87,15 @@ func bazelPushImagesCmd(c Config, isCandidate bool, opts ...bk.StepOpt) func(*bk stepKey := "bazel-push-images" candidate := "" cloudEphemeral := "" + // Until we fix rate limiting with DockerHub, we use 4 to mitigate it. + jobConcurrency := 4 if isCandidate { stepName = ":bazel::docker: Push candidate Images" stepKey = stepKey + "-candidate" candidate = "true" + // But when we're pushing candidate images, we're totally fine with 8. + jobConcurrency = 8 } // Default registries. devRegistry := images.SourcegraphDockerDevRegistry @@ -129,6 +133,7 @@ func bazelPushImagesCmd(c Config, isCandidate bool, opts ...bk.StepOpt) func(*bk bk.Env("DEV_REGISTRY", devRegistry), bk.Env("PROD_REGISTRY", prodRegistry), bk.Env("ADDITIONAL_PROD_REGISTRIES", additionalProdRegistry), + bk.Env("PUSH_CONCURRENT_JOBS", fmt.Sprintf("%d", jobConcurrency)), bk.AutomaticRetry(1), bk.ArtifactPaths("build_event_log.bin", "execution_log.zstd", "bazel-profile.gz"), bk.AnnotatedCmd( diff --git a/dev/ci/push_all.sh b/dev/ci/push_all.sh index 5529aa2ebf2..865e1c0b687 100755 --- a/dev/ci/push_all.sh +++ b/dev/ci/push_all.sh @@ -214,7 +214,11 @@ echo "--- :bazel::docker: Pushing images..." log_file=$(mktemp) # shellcheck disable=SC2064 trap "rm -rf $log_file" EXIT -parallel --jobs=8 --line-buffer --joblog "$log_file" -v <"$job_file" + +# See dev/ci/internal/ci/images_operations.go +JOBS="${PUSH_CONCURRENT_JOBS:-4}" + +parallel --jobs="$JOBS" --line-buffer --joblog "$log_file" -v <"$job_file" # Pretty print the output from gnu parallel while read -r line; do