From 84b2afc235dd9c6d467c4f65cb212468c9d89e19 Mon Sep 17 00:00:00 2001 From: Erik Seliger Date: Thu, 23 May 2024 18:31:20 +0200 Subject: [PATCH] worker: Reduce frequency of very frequently run jobs (#62864) worker: Reduce frequency of very frequent jobs I looked at background routines and noticed that many routines are running _very_ frequently, which can cause higher than necessary load, and also stress on redis to store all these runs. I tried to put in some more sensible numbers (aligning it with what most other routines seem to use). For many instances, this will mean a 30x reduction in job invocations. Test plan: Code review from owners, and will watch S2 and dotcom for potential issues. --- cmd/worker/internal/encryption/config.go | 4 ++-- cmd/worker/internal/permissions/config.go | 2 +- .../dependencies/internal/background/job_packages_filter.go | 2 +- .../ranking/internal/background/coordinator/config.go | 2 +- .../codeintel/ranking/internal/background/exporter/config.go | 2 +- .../codeintel/ranking/internal/background/janitor/config.go | 2 +- .../codeintel/ranking/internal/background/mapper/config.go | 2 +- .../codeintel/ranking/internal/background/reducer/config.go | 2 +- .../uploads/internal/background/backfiller/config.go | 2 +- .../uploads/internal/background/commitgraph/config.go | 2 +- .../codeintel/uploads/internal/background/expirer/config.go | 2 +- internal/executor/types/cache.go | 2 +- internal/goroutine/recorder/recorder.go | 3 +-- 13 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cmd/worker/internal/encryption/config.go b/cmd/worker/internal/encryption/config.go index fc2a4f86594..681bae5166b 100644 --- a/cmd/worker/internal/encryption/config.go +++ b/cmd/worker/internal/encryption/config.go @@ -17,7 +17,7 @@ type config struct { var ConfigInst = &config{} func (c *config) Load() { - c.EncryptionInterval = c.GetInterval("RECORD_ENCRYPTER_INTERVAL", "1s", "How frequently to encrypt/decrypt a batch of records in the database.") - c.MetricsInterval = c.GetInterval("RECORD_ENCRYPTER_METRICS_INTERVAL", "10s", "How frequently to update progress metrics related to encryption/decryption.") + c.EncryptionInterval = c.GetInterval("RECORD_ENCRYPTER_INTERVAL", "10s", "How frequently to encrypt/decrypt a batch of records in the database.") + c.MetricsInterval = c.GetInterval("RECORD_ENCRYPTER_METRICS_INTERVAL", "30s", "How frequently to update progress metrics related to encryption/decryption.") c.Decrypt = c.GetBool("ALLOW_DECRYPTION", "false", "If true, encrypted records will be decrypted and stored in plaintext.") } diff --git a/cmd/worker/internal/permissions/config.go b/cmd/worker/internal/permissions/config.go index 32c2474e36f..2f8e804061e 100644 --- a/cmd/worker/internal/permissions/config.go +++ b/cmd/worker/internal/permissions/config.go @@ -18,7 +18,7 @@ type config struct { var ConfigInst = &config{} func (c *config) Load() { - c.WorkerPollInterval = c.GetInterval("BITBUCKET_PROJECT_PERMISSIONS_WORKER_POLL_INTERVAL", "1s", "How frequently to query the job queue") + c.WorkerPollInterval = c.GetInterval("BITBUCKET_PROJECT_PERMISSIONS_WORKER_POLL_INTERVAL", "10s", "How frequently to query the job queue") c.WorkerConcurrency = c.GetInt("BITBUCKET_PROJECT_PERMISSIONS_WORKER_CONCURRENCY", "1", "The maximum number of projects that can be processed concurrently") c.WorkerRetryInterval = c.GetInterval("BITBUCKET_PROJECT_PERMISSIONS_WORKER_RETRY_INTERVAL", "30s", "The minimum number of time to wait before retrying a failed job") } diff --git a/internal/codeintel/dependencies/internal/background/job_packages_filter.go b/internal/codeintel/dependencies/internal/background/job_packages_filter.go index 9b6e26dd3c7..cb02677c198 100644 --- a/internal/codeintel/dependencies/internal/background/job_packages_filter.go +++ b/internal/codeintel/dependencies/internal/background/job_packages_filter.go @@ -35,7 +35,7 @@ func NewPackagesFilterApplicator( goroutine.HandlerFunc(job.handle), goroutine.WithName("codeintel.package-filter-applicator"), goroutine.WithDescription("applies package repo filters to all package repo references to precompute their blocked status"), - goroutine.WithInterval(time.Second*5), + goroutine.WithInterval(time.Second*30), ) } diff --git a/internal/codeintel/ranking/internal/background/coordinator/config.go b/internal/codeintel/ranking/internal/background/coordinator/config.go index f45fec0329b..8a0044ce272 100644 --- a/internal/codeintel/ranking/internal/background/coordinator/config.go +++ b/internal/codeintel/ranking/internal/background/coordinator/config.go @@ -13,5 +13,5 @@ type Config struct { } func (c *Config) Load() { - c.Interval = c.GetInterval("CODEINTEL_RANKING_COORDINATOR_INTERVAL", "1s", "How frequently to run the ranking coordinator.") + c.Interval = c.GetInterval("CODEINTEL_RANKING_COORDINATOR_INTERVAL", "30s", "How frequently to run the ranking coordinator.") } diff --git a/internal/codeintel/ranking/internal/background/exporter/config.go b/internal/codeintel/ranking/internal/background/exporter/config.go index 19b333e97b7..2ef4934e97c 100644 --- a/internal/codeintel/ranking/internal/background/exporter/config.go +++ b/internal/codeintel/ranking/internal/background/exporter/config.go @@ -15,7 +15,7 @@ type Config struct { } func (c *Config) Load() { - c.Interval = c.GetInterval("CODEINTEL_RANKING_SYMBOL_EXPORTER_INTERVAL", "1s", "How frequently to serialize a batch of the code intel graph for ranking.") + c.Interval = c.GetInterval("CODEINTEL_RANKING_SYMBOL_EXPORTER_INTERVAL", "30s", "How frequently to serialize a batch of the code intel graph for ranking.") c.ReadBatchSize = c.GetInt("CODEINTEL_RANKING_SYMBOL_EXPORTER_READ_BATCH_SIZE", "16", "How many uploads to process at once.") c.WriteBatchSize = c.GetInt("CODEINTEL_RANKING_SYMBOL_EXPORTER_WRITE_BATCH_SIZE", "10000", "The number of definitions and references to populate the ranking graph per batch.") } diff --git a/internal/codeintel/ranking/internal/background/janitor/config.go b/internal/codeintel/ranking/internal/background/janitor/config.go index 65b28b1fd50..3ac3a9ca60b 100644 --- a/internal/codeintel/ranking/internal/background/janitor/config.go +++ b/internal/codeintel/ranking/internal/background/janitor/config.go @@ -13,5 +13,5 @@ type Config struct { } func (c *Config) Load() { - c.Interval = c.GetInterval("CODEINTEL_RANKING_JANITOR_INTERVAL", "1s", "How frequently to run the ranking janitor.") + c.Interval = c.GetInterval("CODEINTEL_RANKING_JANITOR_INTERVAL", "30s", "How frequently to run the ranking janitor.") } diff --git a/internal/codeintel/ranking/internal/background/mapper/config.go b/internal/codeintel/ranking/internal/background/mapper/config.go index df53e2712f5..ed521ec06cc 100644 --- a/internal/codeintel/ranking/internal/background/mapper/config.go +++ b/internal/codeintel/ranking/internal/background/mapper/config.go @@ -14,6 +14,6 @@ type Config struct { } func (c *Config) Load() { - c.Interval = c.GetInterval("CODEINTEL_RANKING_MAPPER_INTERVAL", "1s", "How frequently to run the ranking mapper.") + c.Interval = c.GetInterval("CODEINTEL_RANKING_MAPPER_INTERVAL", "30s", "How frequently to run the ranking mapper.") c.BatchSize = c.GetInt("CODEINTEL_RANKING_MAPPER_BATCH_SIZE", "100", "How many definitions and references to map at once.") } diff --git a/internal/codeintel/ranking/internal/background/reducer/config.go b/internal/codeintel/ranking/internal/background/reducer/config.go index 56cf24cde7b..71952f73de1 100644 --- a/internal/codeintel/ranking/internal/background/reducer/config.go +++ b/internal/codeintel/ranking/internal/background/reducer/config.go @@ -14,6 +14,6 @@ type Config struct { } func (c *Config) Load() { - c.Interval = c.GetInterval("CODEINTEL_RANKING_REDUCER_INTERVAL", "1s", "How frequently to run the ranking reducer.") + c.Interval = c.GetInterval("CODEINTEL_RANKING_REDUCER_INTERVAL", "30s", "How frequently to run the ranking reducer.") c.BatchSize = c.GetInt("CODEINTEL_RANKING_REDUCER_BATCH_SIZE", "1000", "How many path counts to reduce at once.") } diff --git a/internal/codeintel/uploads/internal/background/backfiller/config.go b/internal/codeintel/uploads/internal/background/backfiller/config.go index 82385f8f840..bf3cc9f62f4 100644 --- a/internal/codeintel/uploads/internal/background/backfiller/config.go +++ b/internal/codeintel/uploads/internal/background/backfiller/config.go @@ -14,6 +14,6 @@ type Config struct { } func (c *Config) Load() { - c.Interval = c.GetInterval("CODEINTEL_UPLOAD_BACKFILLER_INTERVAL", "10s", "The frequency with which to run periodic codeintel backfiller tasks.") + c.Interval = c.GetInterval("CODEINTEL_UPLOAD_BACKFILLER_INTERVAL", "30s", "The frequency with which to run periodic codeintel backfiller tasks.") c.BatchSize = c.GetInt("CODEINTEL_UPLOAD_BACKFILLER_BATCH_SIZE", "100", "The number of upload to populate an unset `commited_at` field per batch.") } diff --git a/internal/codeintel/uploads/internal/background/commitgraph/config.go b/internal/codeintel/uploads/internal/background/commitgraph/config.go index 54d363bf6f4..33bb1ac3272 100644 --- a/internal/codeintel/uploads/internal/background/commitgraph/config.go +++ b/internal/codeintel/uploads/internal/background/commitgraph/config.go @@ -20,7 +20,7 @@ func (c *Config) Load() { maxAgeForNonStaleTags := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_COMMITGRAPH_MAX_AGE_FOR_NON_STALE_TAGS", "PRECISE_CODE_INTEL_MAX_AGE_FOR_NON_STALE_TAGS") commitGraphUpdateTaskInterval := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATE_TASK_INTERVAL", "PRECISE_CODE_INTEL_COMMIT_GRAPH_UPDATE_TASK_INTERVAL") - c.Interval = c.GetInterval("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATER_INTERVAL", "1s", "How frequently to run the upload commitgraph updater routine.") + c.Interval = c.GetInterval("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATER_INTERVAL", "10s", "How frequently to run the upload commitgraph updater routine.") c.MaxAgeForNonStaleBranches = c.GetInterval(maxAgeForNonStaleBranches, "2160h", "The age after which a branch should be considered stale. Code intelligence indexes will be evicted from stale branches.") // about 3 months c.MaxAgeForNonStaleTags = c.GetInterval(maxAgeForNonStaleTags, "8760h", "The age after which a tagged commit should be considered stale. Code intelligence indexes will be evicted from stale tagged commits.") // about 1 year c.CommitGraphUpdateTaskInterval = c.GetInterval(commitGraphUpdateTaskInterval, "10s", "The frequency with which to run periodic codeintel commit graph update tasks.") diff --git a/internal/codeintel/uploads/internal/background/expirer/config.go b/internal/codeintel/uploads/internal/background/expirer/config.go index 1e29dc10b5b..890a11cd661 100644 --- a/internal/codeintel/uploads/internal/background/expirer/config.go +++ b/internal/codeintel/uploads/internal/background/expirer/config.go @@ -27,7 +27,7 @@ func (c *Config) Load() { uploadProcessDelay := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_EXPIRER_UPLOAD_PROCESS_DELAY", "PRECISE_CODE_INTEL_RETENTION_UPLOAD_PROCESS_DELAY") c.CommitBatchSize = c.GetInt(commitBatchSize, "100", "The number of commits to process per upload at a time.") - c.ExpirerInterval = c.GetInterval("CODEINTEL_UPLOAD_EXPIRER_INTERVAL", "1s", "How frequently to run the upload expirer routine.") + c.ExpirerInterval = c.GetInterval("CODEINTEL_UPLOAD_EXPIRER_INTERVAL", "30s", "How frequently to run the upload expirer routine.") c.PolicyBatchSize = c.GetInt(policyBatchSize, "100", "The number of policies to consider for expiration at a time.") c.RepositoryBatchSize = c.GetInt(repositoryBatchSize, "100", "The number of repositories to consider for expiration at a time.") c.RepositoryProcessDelay = c.GetInterval(repositoryProcessDelay, "24h", "The minimum frequency that the same repository's uploads can be considered for expiration.") diff --git a/internal/executor/types/cache.go b/internal/executor/types/cache.go index 9cf0b47ff2b..ad448d8ce9c 100644 --- a/internal/executor/types/cache.go +++ b/internal/executor/types/cache.go @@ -9,7 +9,7 @@ import ( const ( DequeueCachePrefix = "executor_multihandler_dequeues" DequeueTtl = 5 * time.Minute - CleanupInterval = 5 * time.Second + CleanupInterval = 30 * time.Second ) var DequeuePropertiesPerQueue = &schema.DequeueCacheConfig{ diff --git a/internal/goroutine/recorder/recorder.go b/internal/goroutine/recorder/recorder.go index fbe5a6ba7c3..6640a0dc6c8 100644 --- a/internal/goroutine/recorder/recorder.go +++ b/internal/goroutine/recorder/recorder.go @@ -5,6 +5,7 @@ import ( "time" "github.com/sourcegraph/log" + "github.com/sourcegraph/sourcegraph/internal/rcache" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -147,8 +148,6 @@ func (m *Recorder) LogRun(r Recordable, duration time.Duration, runErr error) { // Update host's and job's “last seen” dates m.saveKnownHostName() m.saveKnownJobName(r.JobName()) - - m.logger.Debug("Hello from " + r.Name() + "! 😄") } // saveRun saves a run in the Redis list under the "*:recentRuns" key.