worker: Reduce frequency of very frequently run jobs (#62864)

worker: Reduce frequency of very frequent jobs

I looked at background routines and noticed that many routines are running _very_ frequently, which can cause higher than necessary load, and also stress on redis to store all these runs.

I tried to put in some more sensible numbers (aligning it with what most other routines seem to use). For many instances, this will mean a 30x reduction in job invocations.

Test plan:

Code review from owners, and will watch S2 and dotcom for potential issues.

cmd/worker/internal/encryption/config.go

@@ -17,7 +17,7 @@ type config struct {
 var ConfigInst = &config{}
 
 func (c *config) Load() {
-	c.EncryptionInterval = c.GetInterval("RECORD_ENCRYPTER_INTERVAL", "1s", "How frequently to encrypt/decrypt a batch of records in the database.")
-	c.MetricsInterval = c.GetInterval("RECORD_ENCRYPTER_METRICS_INTERVAL", "10s", "How frequently to update progress metrics related to encryption/decryption.")
+	c.EncryptionInterval = c.GetInterval("RECORD_ENCRYPTER_INTERVAL", "10s", "How frequently to encrypt/decrypt a batch of records in the database.")
+	c.MetricsInterval = c.GetInterval("RECORD_ENCRYPTER_METRICS_INTERVAL", "30s", "How frequently to update progress metrics related to encryption/decryption.")
 	c.Decrypt = c.GetBool("ALLOW_DECRYPTION", "false", "If true, encrypted records will be decrypted and stored in plaintext.")
 }

cmd/worker/internal/permissions/config.go

@@ -18,7 +18,7 @@ type config struct {
 var ConfigInst = &config{}
 
 func (c *config) Load() {
-	c.WorkerPollInterval = c.GetInterval("BITBUCKET_PROJECT_PERMISSIONS_WORKER_POLL_INTERVAL", "1s", "How frequently to query the job queue")
+	c.WorkerPollInterval = c.GetInterval("BITBUCKET_PROJECT_PERMISSIONS_WORKER_POLL_INTERVAL", "10s", "How frequently to query the job queue")
 	c.WorkerConcurrency = c.GetInt("BITBUCKET_PROJECT_PERMISSIONS_WORKER_CONCURRENCY", "1", "The maximum number of projects that can be processed concurrently")
 	c.WorkerRetryInterval = c.GetInterval("BITBUCKET_PROJECT_PERMISSIONS_WORKER_RETRY_INTERVAL", "30s", "The minimum number of time to wait before retrying a failed job")
 }

internal/codeintel/dependencies/internal/background/job_packages_filter.go

@@ -35,7 +35,7 @@ func NewPackagesFilterApplicator(
 		goroutine.HandlerFunc(job.handle),
 		goroutine.WithName("codeintel.package-filter-applicator"),
 		goroutine.WithDescription("applies package repo filters to all package repo references to precompute their blocked status"),
-		goroutine.WithInterval(time.Second*5),
+		goroutine.WithInterval(time.Second*30),
 	)
 }
 

internal/codeintel/ranking/internal/background/coordinator/config.go

@@ -13,5 +13,5 @@ type Config struct {
 }
 
 func (c *Config) Load() {
-	c.Interval = c.GetInterval("CODEINTEL_RANKING_COORDINATOR_INTERVAL", "1s", "How frequently to run the ranking coordinator.")
+	c.Interval = c.GetInterval("CODEINTEL_RANKING_COORDINATOR_INTERVAL", "30s", "How frequently to run the ranking coordinator.")
 }

internal/codeintel/ranking/internal/background/exporter/config.go

@@ -15,7 +15,7 @@ type Config struct {
 }
 
 func (c *Config) Load() {
-	c.Interval = c.GetInterval("CODEINTEL_RANKING_SYMBOL_EXPORTER_INTERVAL", "1s", "How frequently to serialize a batch of the code intel graph for ranking.")
+	c.Interval = c.GetInterval("CODEINTEL_RANKING_SYMBOL_EXPORTER_INTERVAL", "30s", "How frequently to serialize a batch of the code intel graph for ranking.")
 	c.ReadBatchSize = c.GetInt("CODEINTEL_RANKING_SYMBOL_EXPORTER_READ_BATCH_SIZE", "16", "How many uploads to process at once.")
 	c.WriteBatchSize = c.GetInt("CODEINTEL_RANKING_SYMBOL_EXPORTER_WRITE_BATCH_SIZE", "10000", "The number of definitions and references to populate the ranking graph per batch.")
 }

internal/codeintel/ranking/internal/background/janitor/config.go

@@ -13,5 +13,5 @@ type Config struct {
 }
 
 func (c *Config) Load() {
-	c.Interval = c.GetInterval("CODEINTEL_RANKING_JANITOR_INTERVAL", "1s", "How frequently to run the ranking janitor.")
+	c.Interval = c.GetInterval("CODEINTEL_RANKING_JANITOR_INTERVAL", "30s", "How frequently to run the ranking janitor.")
 }

internal/codeintel/ranking/internal/background/mapper/config.go

@@ -14,6 +14,6 @@ type Config struct {
 }
 
 func (c *Config) Load() {
-	c.Interval = c.GetInterval("CODEINTEL_RANKING_MAPPER_INTERVAL", "1s", "How frequently to run the ranking mapper.")
+	c.Interval = c.GetInterval("CODEINTEL_RANKING_MAPPER_INTERVAL", "30s", "How frequently to run the ranking mapper.")
 	c.BatchSize = c.GetInt("CODEINTEL_RANKING_MAPPER_BATCH_SIZE", "100", "How many definitions and references to map at once.")
 }

internal/codeintel/ranking/internal/background/reducer/config.go

@@ -14,6 +14,6 @@ type Config struct {
 }
 
 func (c *Config) Load() {
-	c.Interval = c.GetInterval("CODEINTEL_RANKING_REDUCER_INTERVAL", "1s", "How frequently to run the ranking reducer.")
+	c.Interval = c.GetInterval("CODEINTEL_RANKING_REDUCER_INTERVAL", "30s", "How frequently to run the ranking reducer.")
 	c.BatchSize = c.GetInt("CODEINTEL_RANKING_REDUCER_BATCH_SIZE", "1000", "How many path counts to reduce at once.")
 }

internal/codeintel/uploads/internal/background/backfiller/config.go

@@ -14,6 +14,6 @@ type Config struct {
 }
 
 func (c *Config) Load() {
-	c.Interval = c.GetInterval("CODEINTEL_UPLOAD_BACKFILLER_INTERVAL", "10s", "The frequency with which to run periodic codeintel backfiller tasks.")
+	c.Interval = c.GetInterval("CODEINTEL_UPLOAD_BACKFILLER_INTERVAL", "30s", "The frequency with which to run periodic codeintel backfiller tasks.")
 	c.BatchSize = c.GetInt("CODEINTEL_UPLOAD_BACKFILLER_BATCH_SIZE", "100", "The number of upload to populate an unset `commited_at` field per batch.")
 }

internal/codeintel/uploads/internal/background/commitgraph/config.go

@@ -20,7 +20,7 @@ func (c *Config) Load() {
 	maxAgeForNonStaleTags := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_COMMITGRAPH_MAX_AGE_FOR_NON_STALE_TAGS", "PRECISE_CODE_INTEL_MAX_AGE_FOR_NON_STALE_TAGS")
 	commitGraphUpdateTaskInterval := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATE_TASK_INTERVAL", "PRECISE_CODE_INTEL_COMMIT_GRAPH_UPDATE_TASK_INTERVAL")
 
-	c.Interval = c.GetInterval("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATER_INTERVAL", "1s", "How frequently to run the upload commitgraph updater routine.")
+	c.Interval = c.GetInterval("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATER_INTERVAL", "10s", "How frequently to run the upload commitgraph updater routine.")
 	c.MaxAgeForNonStaleBranches = c.GetInterval(maxAgeForNonStaleBranches, "2160h", "The age after which a branch should be considered stale. Code intelligence indexes will be evicted from stale branches.")      // about 3 months
 	c.MaxAgeForNonStaleTags = c.GetInterval(maxAgeForNonStaleTags, "8760h", "The age after which a tagged commit should be considered stale. Code intelligence indexes will be evicted from stale tagged commits.") // about 1 year
 	c.CommitGraphUpdateTaskInterval = c.GetInterval(commitGraphUpdateTaskInterval, "10s", "The frequency with which to run periodic codeintel commit graph update tasks.")

internal/codeintel/uploads/internal/background/expirer/config.go

@@ -27,7 +27,7 @@ func (c *Config) Load() {
 	uploadProcessDelay := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_EXPIRER_UPLOAD_PROCESS_DELAY", "PRECISE_CODE_INTEL_RETENTION_UPLOAD_PROCESS_DELAY")
 
 	c.CommitBatchSize = c.GetInt(commitBatchSize, "100", "The number of commits to process per upload at a time.")
-	c.ExpirerInterval = c.GetInterval("CODEINTEL_UPLOAD_EXPIRER_INTERVAL", "1s", "How frequently to run the upload expirer routine.")
+	c.ExpirerInterval = c.GetInterval("CODEINTEL_UPLOAD_EXPIRER_INTERVAL", "30s", "How frequently to run the upload expirer routine.")
 	c.PolicyBatchSize = c.GetInt(policyBatchSize, "100", "The number of policies to consider for expiration at a time.")
 	c.RepositoryBatchSize = c.GetInt(repositoryBatchSize, "100", "The number of repositories to consider for expiration at a time.")
 	c.RepositoryProcessDelay = c.GetInterval(repositoryProcessDelay, "24h", "The minimum frequency that the same repository's uploads can be considered for expiration.")

internal/executor/types/cache.go

@@ -9,7 +9,7 @@ import (
 const (
 	DequeueCachePrefix = "executor_multihandler_dequeues"
 	DequeueTtl         = 5 * time.Minute
-	CleanupInterval    = 5 * time.Second
+	CleanupInterval    = 30 * time.Second
 )
 
 var DequeuePropertiesPerQueue = &schema.DequeueCacheConfig{

internal/goroutine/recorder/recorder.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/sourcegraph/log"
+
 	"github.com/sourcegraph/sourcegraph/internal/rcache"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
@@ -147,8 +148,6 @@ func (m *Recorder) LogRun(r Recordable, duration time.Duration, runErr error) {
 	// Update host's and job's “last seen” dates
 	m.saveKnownHostName()
 	m.saveKnownJobName(r.JobName())
-
-	m.logger.Debug("Hello from " + r.Name() + "! 😄")
 }
 
 // saveRun saves a run in the Redis list under the "*:recentRuns" key.