From 9f25c9fe4e75466708af33b21a0d43e83ae9f5f4 Mon Sep 17 00:00:00 2001 From: Varun Gandhi Date: Mon, 15 Jul 2024 09:48:40 +0800 Subject: [PATCH] fix: Increase precise-code-intel-worker's default concurrency --- cmd/precise-code-intel-worker/shared/config.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/precise-code-intel-worker/shared/config.go b/cmd/precise-code-intel-worker/shared/config.go index 22f97d9c334..1aa11c282d6 100644 --- a/cmd/precise-code-intel-worker/shared/config.go +++ b/cmd/precise-code-intel-worker/shared/config.go @@ -1,6 +1,8 @@ package shared import ( + "runtime" + "strconv" "time" "github.com/sourcegraph/sourcegraph/internal/codeintel/shared/lsifuploadstore" @@ -23,7 +25,12 @@ func (c *Config) Load() { c.LSIFUploadStoreConfig.Load() c.WorkerPollInterval = c.GetInterval("PRECISE_CODE_INTEL_WORKER_POLL_INTERVAL", "1s", "Interval between queries to the upload queue.") - c.WorkerConcurrency = c.GetInt("PRECISE_CODE_INTEL_WORKER_CONCURRENCY", "1", "The maximum number of indexes that can be processed concurrently.") + // If the worker has multiple cores available, let's make sure we're making good use of those. + // As of 2024 July 15, I/O takes about 2x the time as CPU processing, + // (see https://github.com/sourcegraph/sourcegraph/pull/61826) + // so try to spin up more goroutines since we're not worried about + // context switching overhead. + c.WorkerConcurrency = c.GetInt("PRECISE_CODE_INTEL_WORKER_CONCURRENCY", strconv.Itoa(2*runtime.GOMAXPROCS(-1)), "The maximum number of indexes that can be processed concurrently.") c.WorkerBudget = int64(c.GetInt("PRECISE_CODE_INTEL_WORKER_BUDGET", "0", "The amount of compressed input data (in bytes) a worker can process concurrently. Zero acts as an infinite budget.")) c.MaximumRuntimePerJob = c.GetInterval("PRECISE_CODE_INTEL_WORKER_MAXIMUM_RUNTIME_PER_JOB", "25m", "The maximum time a single LSIF processing job can take.") }