codeintel: Co-locate code next to background job (#42641)

cmd/worker/internal/codeintel/crates_syncer.go

@@ -32,5 +32,5 @@ func (j *cratesSyncerJob) Routines(startupCtx context.Context, logger log.Logger
 		return nil, err
 	}
 
-	return cratesyncer.NewSyncers(services.DependenciesService), nil
+	return cratesyncer.NewCrateSyncer(services.DependenciesService), nil
 }

cmd/worker/internal/codeintel/policies_repomatcher_job.go

@@ -34,5 +34,5 @@ func (j *policiesRepositoryMatcherJob) Routines(startupCtx context.Context, logg
 		return nil, err
 	}
 
-	return repomatcher.NewMatchers(services.PoliciesService), nil
+	return repomatcher.NewRepositoryMatcher(services.PoliciesService), nil
 }

internal/codeintel/autoindexing/background/scheduler/iface.go

@@ -7,6 +7,15 @@ import (
 )
 
 type AutoIndexingService interface {
-	NewScheduler(interval time.Duration, repositoryProcessDelay time.Duration, repositoryBatchSize int, policyBatchSize int) goroutine.BackgroundRoutine
-	NewOnDemandScheduler(interval time.Duration, batchSize int) goroutine.BackgroundRoutine
+	NewScheduler(
+		interval time.Duration,
+		repositoryProcessDelay time.Duration,
+		repositoryBatchSize int,
+		policyBatchSize int,
+	) goroutine.BackgroundRoutine
+
+	NewOnDemandScheduler(
+		interval time.Duration,
+		batchSize int,
+	) goroutine.BackgroundRoutine
 }

internal/codeintel/autoindexing/background/scheduler/init.go

@@ -6,7 +6,16 @@ import (
 
 func NewSchedulers(autoIndexingSvc AutoIndexingService) []goroutine.BackgroundRoutine {
 	return []goroutine.BackgroundRoutine{
-		autoIndexingSvc.NewScheduler(ConfigInst.SchedulerInterval, ConfigInst.RepositoryProcessDelay, ConfigInst.RepositoryBatchSize, ConfigInst.PolicyBatchSize),
-		autoIndexingSvc.NewOnDemandScheduler(ConfigInst.OnDemandSchedulerInterval, ConfigInst.OnDemandBatchsize),
+		autoIndexingSvc.NewScheduler(
+			ConfigInst.SchedulerInterval,
+			ConfigInst.RepositoryProcessDelay,
+			ConfigInst.RepositoryBatchSize,
+			ConfigInst.PolicyBatchSize,
+		),
+
+		autoIndexingSvc.NewOnDemandScheduler(
+			ConfigInst.OnDemandSchedulerInterval,
+			ConfigInst.OnDemandBatchsize,
+		),
 	}
 }

internal/codeintel/dependencies/background/cratesyncer/config.go

@@ -0,0 +1,14 @@
+package cratesyncer
+
+import (
+	"github.com/sourcegraph/sourcegraph/internal/env"
+)
+
+type config struct {
+	env.BaseConfig
+}
+
+var ConfigInst = &config{}
+
+func (c *config) Load() {
+}

internal/codeintel/dependencies/background/cratesyncer/init.go

@@ -2,7 +2,7 @@ package cratesyncer
 
 import "github.com/sourcegraph/sourcegraph/internal/goroutine"
 
-func NewSyncers(depsSvc DependenciesService) []goroutine.BackgroundRoutine {
+func NewCrateSyncer(depsSvc DependenciesService) []goroutine.BackgroundRoutine {
 	return []goroutine.BackgroundRoutine{
 		depsSvc.NewCrateSyncer(),
 	}

internal/codeintel/policies/background/repomatcher/iface.go

@@ -7,5 +7,8 @@ import (
 )
 
 type PolicyService interface {
-	NewRepoMatcher(interval time.Duration, configurationPolicyMembershipBatchSize int) goroutine.BackgroundRoutine
+	NewRepositoryMatcher(
+		interval time.Duration,
+		configurationPolicyMembershipBatchSize int,
+	) goroutine.BackgroundRoutine
 }

internal/codeintel/policies/background/repomatcher/init.go

@@ -4,8 +4,11 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/goroutine"
 )
 
-func NewMatchers(policySvc PolicyService) []goroutine.BackgroundRoutine {
+func NewRepositoryMatcher(policySvc PolicyService) []goroutine.BackgroundRoutine {
 	return []goroutine.BackgroundRoutine{
-		policySvc.NewRepoMatcher(ConfigInst.Interval, ConfigInst.ConfigurationPolicyMembershipBatchSize),
+		policySvc.NewRepositoryMatcher(
+			ConfigInst.Interval,
+			ConfigInst.ConfigurationPolicyMembershipBatchSize,
+		),
 	}
 }

internal/codeintel/policies/repomatcher.go

@@ -4,17 +4,19 @@ import (
 	"context"
 	"time"
 
+	"github.com/sourcegraph/sourcegraph/internal/codeintel/types"
 	"github.com/sourcegraph/sourcegraph/internal/conf"
 	"github.com/sourcegraph/sourcegraph/internal/goroutine"
+	"github.com/sourcegraph/sourcegraph/internal/observation"
 )
 
-func (s *Service) NewRepoMatcher(interval time.Duration, configurationPolicyMembershipBatchSize int) goroutine.BackgroundRoutine {
+func (s *Service) NewRepositoryMatcher(interval time.Duration, configurationPolicyMembershipBatchSize int) goroutine.BackgroundRoutine {
 	return goroutine.NewPeriodicGoroutine(context.Background(), interval, goroutine.HandlerFunc(func(ctx context.Context) error {
-		return s.handleRepoMatcherBatch(ctx, configurationPolicyMembershipBatchSize)
+		return s.handleRepositoryMatcherBatch(ctx, configurationPolicyMembershipBatchSize)
 	}))
 }
 
-func (s *Service) handleRepoMatcherBatch(ctx context.Context, batchSize int) error {
+func (s *Service) handleRepositoryMatcherBatch(ctx context.Context, batchSize int) error {
 	policies, err := s.SelectPoliciesForRepositoryMembershipUpdate(ctx, batchSize)
 	if err != nil {
 		return err
@@ -43,3 +45,22 @@ func (s *Service) handleRepoMatcherBatch(ctx context.Context, batchSize int) err
 
 	return nil
 }
+
+func (s *Service) SelectPoliciesForRepositoryMembershipUpdate(ctx context.Context, batchSize int) (configurationPolicies []types.ConfigurationPolicy, err error) {
+	ctx, _, endObservation := s.operations.selectPoliciesForRepositoryMembershipUpdate.With(ctx, &err, observation.Args{})
+	defer endObservation(1, observation.Args{})
+
+	configurationPolicies, err = s.store.SelectPoliciesForRepositoryMembershipUpdate(ctx, batchSize)
+	if err != nil {
+		return nil, err
+	}
+
+	return configurationPolicies, nil
+}
+
+func (s *Service) UpdateReposMatchingPatterns(ctx context.Context, patterns []string, policyID int, repositoryMatchLimit *int) (err error) {
+	ctx, _, endObservation := s.operations.updateReposMatchingPatterns.With(ctx, &err, observation.Args{})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.UpdateReposMatchingPatterns(ctx, patterns, policyID, repositoryMatchLimit)
+}

internal/codeintel/policies/service.go

@@ -242,25 +242,6 @@ func (s *Service) GetUnsafeDB() database.DB {
 	return s.store.GetUnsafeDB()
 }
 
-func (s *Service) SelectPoliciesForRepositoryMembershipUpdate(ctx context.Context, batchSize int) (configurationPolicies []types.ConfigurationPolicy, err error) {
-	ctx, _, endObservation := s.operations.selectPoliciesForRepositoryMembershipUpdate.With(ctx, &err, observation.Args{})
-	defer endObservation(1, observation.Args{})
-
-	configurationPolicies, err = s.store.SelectPoliciesForRepositoryMembershipUpdate(ctx, batchSize)
-	if err != nil {
-		return nil, err
-	}
-
-	return configurationPolicies, nil
-}
-
-func (s *Service) UpdateReposMatchingPatterns(ctx context.Context, patterns []string, policyID int, repositoryMatchLimit *int) (err error) {
-	ctx, _, endObservation := s.operations.updateReposMatchingPatterns.With(ctx, &err, observation.Args{})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.UpdateReposMatchingPatterns(ctx, patterns, policyID, repositoryMatchLimit)
-}
-
 func (s *Service) getCommitsVisibleToUpload(ctx context.Context, upload types.Upload) (commits []string, err error) {
 	var token *string
 	for first := true; first || token != nil; first = false {

internal/codeintel/uploads/backfill.go

@@ -4,11 +4,76 @@ import (
 	"context"
 	"time"
 
+	"github.com/opentracing/opentracing-go/log"
+
 	"github.com/sourcegraph/sourcegraph/internal/goroutine"
+	"github.com/sourcegraph/sourcegraph/internal/observation"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
 func (s *Service) NewCommittedAtBackfiller(interval time.Duration, batchSize int) goroutine.BackgroundRoutine {
 	return goroutine.NewPeriodicGoroutine(context.Background(), interval, goroutine.HandlerFunc(func(ctx context.Context) error {
-		return s.BackfillCommittedAtBatch(ctx, batchSize)
+		return s.backfillCommittedAtBatch(ctx, batchSize)
 	}))
 }
+
+// backfillCommittedAtBatch calculates the committed_at value for a batch of upload records that do not have
+// this value set. This method is used to backfill old upload records prior to this value being reliably set
+// during processing.
+func (s *Service) backfillCommittedAtBatch(ctx context.Context, batchSize int) (err error) {
+	ctx, _, endObservation := s.operations.backfillCommittedAtBatch.With(ctx, &err, observation.Args{LogFields: []log.Field{
+		log.Int("batchSize", batchSize),
+	}})
+	defer endObservation(1, observation.Args{})
+
+	tx, err := s.store.Transact(ctx)
+	defer func() {
+		err = tx.Done(err)
+	}()
+
+	batch, err := tx.SourcedCommitsWithoutCommittedAt(ctx, batchSize)
+	if err != nil {
+		return errors.Wrap(err, "store.SourcedCommitsWithoutCommittedAt")
+	}
+
+	for _, sourcedCommits := range batch {
+		for _, commit := range sourcedCommits.Commits {
+			commitDateString, err := s.getCommitDate(ctx, sourcedCommits.RepositoryID, commit)
+			if err != nil {
+				return err
+			}
+
+			// Update commit date of all uploads attached to this this repository and commit
+			if err := tx.UpdateCommittedAt(ctx, sourcedCommits.RepositoryID, commit, commitDateString); err != nil {
+				return errors.Wrap(err, "store.UpdateCommittedAt")
+			}
+		}
+
+		// Mark repository as dirty so the commit graph is recalculated with fresh data
+		if err := tx.SetRepositoryAsDirty(ctx, sourcedCommits.RepositoryID); err != nil {
+			return errors.Wrap(err, "store.SetRepositoryAsDirty")
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) getCommitDate(ctx context.Context, repositoryID int, commit string) (string, error) {
+	_, commitDate, revisionExists, err := s.gitserverClient.CommitDate(ctx, repositoryID, commit)
+	if err != nil {
+		return "", errors.Wrap(err, "gitserver.CommitDate")
+	}
+
+	var commitDateString string
+	if revisionExists {
+		commitDateString = commitDate.Format(time.RFC3339)
+	} else {
+		// Set a value here that we'll filter out on the query side so that we don't
+		// reprocess the same failing batch infinitely. We could alternatively soft
+		// delete the record, but it would be better to keep record deletion behavior
+		// together in the same place (so we have unified metrics on that event).
+		commitDateString = "-infinity"
+	}
+
+	return commitDateString, nil
+}

internal/codeintel/uploads/backfill_test.go

@@ -54,7 +54,7 @@ func TestBackfillCommittedAtBatch(t *testing.T) {
 	}
 
 	for i := 0; i < n/pageSize; i++ {
-		if err := svc.BackfillCommittedAtBatch(ctx, pageSize); err != nil {
+		if err := svc.backfillCommittedAtBatch(ctx, pageSize); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	}
@@ -131,7 +131,7 @@ func TestBackfillCommittedAtBatchUnknownCommits(t *testing.T) {
 	}
 
 	for i := 0; i < n/pageSize; i++ {
-		if err := svc.BackfillCommittedAtBatch(ctx, pageSize); err != nil {
+		if err := svc.backfillCommittedAtBatch(ctx, pageSize); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
 	}

internal/codeintel/uploads/background/backfill/iface.go

@@ -7,5 +7,8 @@ import (
 )
 
 type UploadService interface {
-	NewCommittedAtBackfiller(interval time.Duration, batchSize int) goroutine.BackgroundRoutine
+	NewCommittedAtBackfiller(
+		interval time.Duration,
+		batchSize int,
+	) goroutine.BackgroundRoutine
 }

internal/codeintel/uploads/background/backfill/init.go

@@ -6,6 +6,9 @@ import (
 
 func NewCommittedAtBackfiller(uploadSvc UploadService) []goroutine.BackgroundRoutine {
 	return []goroutine.BackgroundRoutine{
-		uploadSvc.NewCommittedAtBackfiller(ConfigInst.Interval, ConfigInst.BatchSize),
+		uploadSvc.NewCommittedAtBackfiller(
+			ConfigInst.Interval,
+			ConfigInst.BatchSize,
+		),
 	}
 }

internal/codeintel/uploads/background/cleanup/iface.go

@@ -16,5 +16,6 @@ type UploadService interface {
 		commitResolverBatchSize int,
 		commitResolverMaximumCommitLag time.Duration,
 	) goroutine.BackgroundRoutine
+
 	NewUploadResetter(interval time.Duration) *dbworker.Resetter
 }

internal/codeintel/uploads/background/commitgraph/iface.go

@@ -7,5 +7,9 @@ import (
 )
 
 type UploadService interface {
-	NewUpdater(interval time.Duration, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) goroutine.BackgroundRoutine
+	NewCommitGraphUpdater(
+		interval time.Duration,
+		maxAgeForNonStaleBranches time.Duration,
+		maxAgeForNonStaleTags time.Duration,
+	) goroutine.BackgroundRoutine
 }

internal/codeintel/uploads/background/commitgraph/init.go

@@ -6,6 +6,10 @@ import (
 
 func NewUpdater(uploadSvc UploadService) []goroutine.BackgroundRoutine {
 	return []goroutine.BackgroundRoutine{
-		uploadSvc.NewUpdater(ConfigInst.CommitGraphUpdateTaskInterval, ConfigInst.MaxAgeForNonStaleBranches, ConfigInst.MaxAgeForNonStaleTags),
+		uploadSvc.NewCommitGraphUpdater(
+			ConfigInst.CommitGraphUpdateTaskInterval,
+			ConfigInst.MaxAgeForNonStaleBranches,
+			ConfigInst.MaxAgeForNonStaleTags,
+		),
 	}
 }

internal/codeintel/uploads/background/expiration/iface.go

@@ -7,7 +7,8 @@ import (
 )
 
 type UploadService interface {
-	NewExpirer(interval time.Duration,
+	NewUploadExpirer(
+		interval time.Duration,
 		repositoryProcessDelay time.Duration,
 		repositoryBatchSize int,
 		uploadProcessDelay time.Duration,
@@ -15,5 +16,9 @@ type UploadService interface {
 		commitBatchSize int,
 		policyBatchSize int,
 	) goroutine.BackgroundRoutine
-	NewReferenceCountUpdater(interval time.Duration, batchSize int) goroutine.BackgroundRoutine
+
+	NewReferenceCountUpdater(
+		interval time.Duration,
+		batchSize int,
+	) goroutine.BackgroundRoutine
 }

internal/codeintel/uploads/background/expiration/init.go

@@ -6,7 +6,8 @@ import (
 
 func NewExpirationTasks(uploadSvc UploadService) []goroutine.BackgroundRoutine {
 	return []goroutine.BackgroundRoutine{
-		uploadSvc.NewExpirer(ConfigInst.ExpirerInterval,
+		uploadSvc.NewUploadExpirer(
+			ConfigInst.ExpirerInterval,
 			ConfigInst.RepositoryProcessDelay,
 			ConfigInst.RepositoryBatchSize,
 			ConfigInst.UploadProcessDelay,
@@ -14,6 +15,9 @@ func NewExpirationTasks(uploadSvc UploadService) []goroutine.BackgroundRoutine {
 			ConfigInst.CommitBatchSize,
 			ConfigInst.PolicyBatchSize,
 		),
-		uploadSvc.NewReferenceCountUpdater(ConfigInst.ReferenceCountUpdaterInterval, ConfigInst.ReferenceCountUpdaterBatchSize),
+		uploadSvc.NewReferenceCountUpdater(
+			ConfigInst.ReferenceCountUpdaterInterval,
+			ConfigInst.ReferenceCountUpdaterBatchSize,
+		),
 	}
 }

internal/codeintel/uploads/cleanup.go

@@ -5,12 +5,15 @@ import (
 	"sort"
 	"time"
 
+	otlog "github.com/opentracing/opentracing-go/log"
 	"github.com/sourcegraph/log"
 
 	autoindexingshared "github.com/sourcegraph/sourcegraph/internal/codeintel/autoindexing/shared"
+	"github.com/sourcegraph/sourcegraph/internal/codeintel/types"
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared"
 	"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
 	"github.com/sourcegraph/sourcegraph/internal/goroutine"
+	"github.com/sourcegraph/sourcegraph/internal/observation"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
@@ -67,27 +70,27 @@ func (s *Service) handleCleanup(ctx context.Context, cfg janitorConfig) (errs er
 	return errs
 }
 
-func (j *Service) handleDeletedRepository(ctx context.Context) (err error) {
-	uploadsCounts, err := j.DeleteUploadsWithoutRepository(ctx, time.Now())
+func (s *Service) handleDeletedRepository(ctx context.Context) (err error) {
+	uploadsCounts, err := s.DeleteUploadsWithoutRepository(ctx, time.Now())
 	if err != nil {
 		return errors.Wrap(err, "uploadSvc.DeleteUploadsWithoutRepository")
 	}
 
-	indexesCounts, err := j.autoIndexingSvc.DeleteIndexesWithoutRepository(ctx, time.Now())
+	indexesCounts, err := s.autoIndexingSvc.DeleteIndexesWithoutRepository(ctx, time.Now())
 	if err != nil {
 		return errors.Wrap(err, "indexSvc.DeleteIndexesWithoutRepository")
 	}
 
 	for _, counts := range gatherCounts(uploadsCounts, indexesCounts) {
-		j.logger.Debug(
+		s.logger.Debug(
 			"Deleted codeintel records with a deleted repository",
 			log.Int("repository_id", counts.repoID),
 			log.Int("uploads_count", counts.uploadsCount),
 			log.Int("indexes_count", counts.indexesCount),
 		)
 
-		j.janitorMetrics.numUploadRecordsRemoved.Add(float64(counts.uploadsCount))
-		j.janitorMetrics.numIndexRecordsRemoved.Add(float64(counts.indexesCount))
+		s.janitorMetrics.numUploadRecordsRemoved.Add(float64(counts.uploadsCount))
+		s.janitorMetrics.numIndexRecordsRemoved.Add(float64(counts.indexesCount))
 	}
 
 	return nil
@@ -126,20 +129,20 @@ func gatherCounts(uploadsCounts, indexesCounts map[int]int) []recordCount {
 	return recordCounts
 }
 
-func (j *Service) handleUnknownCommit(ctx context.Context, cfg janitorConfig) (err error) {
-	staleUploads, err := j.GetStaleSourcedCommits(ctx, cfg.minimumTimeSinceLastCheck, cfg.commitResolverBatchSize, j.clock.Now())
+func (s *Service) handleUnknownCommit(ctx context.Context, cfg janitorConfig) (err error) {
+	staleUploads, err := s.GetStaleSourcedCommits(ctx, cfg.minimumTimeSinceLastCheck, cfg.commitResolverBatchSize, s.clock.Now())
 	if err != nil {
 		return errors.Wrap(err, "uploadSvc.StaleSourcedCommits")
 	}
 
-	staleIndexes, err := j.autoIndexingSvc.GetStaleSourcedCommits(ctx, cfg.minimumTimeSinceLastCheck, cfg.commitResolverBatchSize, j.clock.Now())
+	staleIndexes, err := s.autoIndexingSvc.GetStaleSourcedCommits(ctx, cfg.minimumTimeSinceLastCheck, cfg.commitResolverBatchSize, s.clock.Now())
 	if err != nil {
 		return errors.Wrap(err, "indexSvc.StaleSourcedCommits")
 	}
 
 	batch := mergeSourceCommits(staleUploads, staleIndexes)
 	for _, sourcedCommits := range batch {
-		if err := j.handleSourcedCommits(ctx, sourcedCommits, cfg); err != nil {
+		if err := s.handleSourcedCommits(ctx, sourcedCommits, cfg); err != nil {
 			return err
 		}
 	}
@@ -174,9 +177,9 @@ type SourcedCommits struct {
 	Commits        []string
 }
 
-func (j *Service) handleSourcedCommits(ctx context.Context, sc SourcedCommits, cfg janitorConfig) error {
+func (s *Service) handleSourcedCommits(ctx context.Context, sc SourcedCommits, cfg janitorConfig) error {
 	for _, commit := range sc.Commits {
-		if err := j.handleCommit(ctx, sc.RepositoryID, sc.RepositoryName, commit, cfg); err != nil {
+		if err := s.handleCommit(ctx, sc.RepositoryID, sc.RepositoryName, commit, cfg); err != nil {
 			return err
 		}
 	}
@@ -184,9 +187,9 @@ func (j *Service) handleSourcedCommits(ctx context.Context, sc SourcedCommits, c
 	return nil
 }
 
-func (j *Service) handleCommit(ctx context.Context, repositoryID int, repositoryName, commit string, cfg janitorConfig) error {
+func (s *Service) handleCommit(ctx context.Context, repositoryID int, repositoryName, commit string, cfg janitorConfig) error {
 	var shouldDelete bool
-	_, err := j.gitserverClient.ResolveRevision(ctx, repositoryID, commit)
+	_, err := s.gitserverClient.ResolveRevision(ctx, repositoryID, commit)
 	if err == nil {
 		// If we have no error then the commit is resolvable and we shouldn't touch it.
 		shouldDelete = false
@@ -206,81 +209,178 @@ func (j *Service) handleCommit(ctx context.Context, repositoryID int, repository
 	}
 
 	if shouldDelete {
-		_, uploadsDeleted, err := j.DeleteSourcedCommits(ctx, repositoryID, commit, cfg.commitResolverMaximumCommitLag, j.clock.Now())
+		_, uploadsDeleted, err := s.DeleteSourcedCommits(ctx, repositoryID, commit, cfg.commitResolverMaximumCommitLag, s.clock.Now())
 		if err != nil {
 			return errors.Wrap(err, "uploadSvc.DeleteSourcedCommits")
 		}
 		if uploadsDeleted > 0 {
 			// log.Debug("Deleted upload records with unresolvable commits", "count", uploadsDeleted)
-			j.janitorMetrics.numUploadRecordsRemoved.Add(float64(uploadsDeleted))
+			s.janitorMetrics.numUploadRecordsRemoved.Add(float64(uploadsDeleted))
 		}
 
-		indexesDeleted, err := j.autoIndexingSvc.DeleteSourcedCommits(ctx, repositoryID, commit, cfg.commitResolverMaximumCommitLag, j.clock.Now())
+		indexesDeleted, err := s.autoIndexingSvc.DeleteSourcedCommits(ctx, repositoryID, commit, cfg.commitResolverMaximumCommitLag, s.clock.Now())
 		if err != nil {
 			return errors.Wrap(err, "indexSvc.DeleteSourcedCommits")
 		}
 		if indexesDeleted > 0 {
 			// log.Debug("Deleted index records with unresolvable commits", "count", indexesDeleted)
-			j.janitorMetrics.numIndexRecordsRemoved.Add(float64(indexesDeleted))
+			s.janitorMetrics.numIndexRecordsRemoved.Add(float64(indexesDeleted))
 		}
 
 		return nil
 	}
 
-	if _, err := j.UpdateSourcedCommits(ctx, repositoryID, commit, j.clock.Now()); err != nil {
+	if _, err := s.UpdateSourcedCommits(ctx, repositoryID, commit, s.clock.Now()); err != nil {
 		return errors.Wrap(err, "uploadSvc.UpdateSourcedCommits")
 	}
 
-	if _, err := j.autoIndexingSvc.UpdateSourcedCommits(ctx, repositoryID, commit, j.clock.Now()); err != nil {
+	if _, err := s.autoIndexingSvc.UpdateSourcedCommits(ctx, repositoryID, commit, s.clock.Now()); err != nil {
 		return errors.Wrap(err, "indexSvc.UpdateSourcedCommits")
 	}
 
 	return nil
 }
 
+func (s *Service) DeleteSourcedCommits(ctx context.Context, repositoryID int, commit string, maximumCommitLag time.Duration, now time.Time) (uploadsUpdated int, uploadsDeleted int, err error) {
+	ctx, _, endObservation := s.operations.deleteSourcedCommits.With(ctx, &err, observation.Args{
+		LogFields: []otlog.Field{
+			otlog.Int("repositoryID", repositoryID),
+			otlog.String("commit", commit),
+			otlog.Int("maximumCommitLag in ms", int(maximumCommitLag.Milliseconds())),
+			otlog.String("now", now.String()),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.DeleteSourcedCommits(ctx, repositoryID, commit, maximumCommitLag, now)
+}
+
+func (s *Service) UpdateSourcedCommits(ctx context.Context, repositoryID int, commit string, now time.Time) (uploadsUpdated int, err error) {
+	ctx, _, endObservation := s.operations.updateSourcedCommits.With(ctx, &err, observation.Args{
+		LogFields: []otlog.Field{
+			otlog.Int("repositoryID", repositoryID),
+			otlog.String("commit", commit),
+			otlog.String("now", now.String()),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.UpdateSourcedCommits(ctx, repositoryID, commit, now)
+}
+
 // handleAbandonedUpload removes upload records which have not left the uploading state within the given TTL.
-func (j *Service) handleAbandonedUpload(ctx context.Context, cfg janitorConfig) error {
-	count, err := j.DeleteUploadsStuckUploading(ctx, time.Now().UTC().Add(-cfg.uploadTimeout))
+func (s *Service) handleAbandonedUpload(ctx context.Context, cfg janitorConfig) error {
+	count, err := s.DeleteUploadsStuckUploading(ctx, time.Now().UTC().Add(-cfg.uploadTimeout))
 	if err != nil {
 		return errors.Wrap(err, "dbstore.DeleteUploadsStuckUploading")
 	}
 	if count > 0 {
-		j.logger.Debug("Deleted abandoned upload records", log.Int("count", count))
-		j.janitorMetrics.numUploadRecordsRemoved.Add(float64(count))
+		s.logger.Debug("Deleted abandoned upload records", log.Int("count", count))
+		s.janitorMetrics.numUploadRecordsRemoved.Add(float64(count))
 	}
 
 	return nil
 }
 
-func (j *Service) handleExpiredUploadDeleter(ctx context.Context) error {
-	count, err := j.SoftDeleteExpiredUploads(ctx)
+func (s *Service) DeleteUploadsStuckUploading(ctx context.Context, uploadedBefore time.Time) (_ int, err error) {
+	ctx, _, endObservation := s.operations.deleteUploadsStuckUploading.With(ctx, &err, observation.Args{
+		LogFields: []otlog.Field{
+			otlog.String("uploadedBefore", uploadedBefore.String()),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.DeleteUploadsStuckUploading(ctx, uploadedBefore)
+}
+
+func (s *Service) handleExpiredUploadDeleter(ctx context.Context) error {
+	count, err := s.SoftDeleteExpiredUploads(ctx)
 	if err != nil {
 		return errors.Wrap(err, "SoftDeleteExpiredUploads")
 	}
 	if count > 0 {
-		j.logger.Info("Deleted expired codeintel uploads", log.Int("count", count))
-		j.janitorMetrics.numUploadRecordsRemoved.Add(float64(count))
+		s.logger.Info("Deleted expired codeintel uploads", log.Int("count", count))
+		s.janitorMetrics.numUploadRecordsRemoved.Add(float64(count))
 	}
 
 	return nil
 }
 
-func (j *Service) handleHardDeleter(ctx context.Context) error {
-	count, err := j.HardDeleteExpiredUploads(ctx)
+func (s *Service) SoftDeleteExpiredUploads(ctx context.Context) (count int, err error) {
+	ctx, _, endObservation := s.operations.softDeleteExpiredUploads.With(ctx, &err, observation.Args{})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.SoftDeleteExpiredUploads(ctx)
+}
+
+func (s *Service) handleHardDeleter(ctx context.Context) error {
+	count, err := s.HardDeleteExpiredUploads(ctx)
 	if err != nil {
 		return errors.Wrap(err, "uploadSvc.HardDeleteExpiredUploads")
 	}
 
-	j.janitorMetrics.numUploadsPurged.Add(float64(count))
+	s.janitorMetrics.numUploadsPurged.Add(float64(count))
 	return nil
 }
 
-func (j *Service) handleAuditLog(ctx context.Context, cfg janitorConfig) (err error) {
-	count, err := j.DeleteOldAuditLogs(ctx, cfg.auditLogMaxAge, time.Now())
+func (s *Service) HardDeleteExpiredUploads(ctx context.Context) (count int, err error) {
+	ctx, _, endObservation := s.operations.hardDeleteUploads.With(ctx, &err, observation.Args{})
+	defer endObservation(1, observation.Args{})
+
+	const uploadsBatchSize = 100
+	options := types.GetUploadsOptions{
+		State:            "deleted",
+		Limit:            uploadsBatchSize,
+		AllowExpired:     true,
+		AllowDeletedRepo: true,
+	}
+
+	for {
+		// Always request the first page of deleted uploads. If this is not
+		// the first iteration of the loop, then the previous iteration has
+		// deleted the records that composed the previous page, and the
+		// previous "second" page is now the first page.
+		uploads, totalCount, err := s.store.GetUploads(ctx, options)
+		if err != nil {
+			return 0, errors.Wrap(err, "store.GetUploads")
+		}
+
+		ids := uploadIDs(uploads)
+		if err := s.lsifstore.DeleteLsifDataByUploadIds(ctx, ids...); err != nil {
+			return 0, errors.Wrap(err, "lsifstore.Clear")
+		}
+
+		if err := s.store.HardDeleteUploadsByIDs(ctx, ids...); err != nil {
+			return 0, errors.Wrap(err, "store.HardDeleteUploadsByIDs")
+		}
+
+		count += len(uploads)
+		if count >= totalCount {
+			break
+		}
+	}
+
+	return count, nil
+}
+
+func (s *Service) handleAuditLog(ctx context.Context, cfg janitorConfig) (err error) {
+	count, err := s.DeleteOldAuditLogs(ctx, cfg.auditLogMaxAge, time.Now())
 	if err != nil {
 		return errors.Wrap(err, "dbstore.DeleteOldAuditLogs")
 	}
 
-	j.janitorMetrics.numAuditLogRecordsExpired.Add(float64(count))
+	s.janitorMetrics.numAuditLogRecordsExpired.Add(float64(count))
 	return nil
 }
+
+func (s *Service) DeleteOldAuditLogs(ctx context.Context, maxAge time.Duration, now time.Time) (count int, err error) {
+	ctx, _, endObservation := s.operations.deleteOldAuditLogs.With(ctx, &err, observation.Args{
+		LogFields: []otlog.Field{
+			otlog.String("maxAge", maxAge.String()),
+			otlog.String("now", now.String()),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.DeleteOldAuditLogs(ctx, maxAge, now)
+}

internal/codeintel/uploads/commitgraph.go

@@ -2,19 +2,166 @@ package uploads
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	"github.com/opentracing/opentracing-go/log"
+	otlog "github.com/opentracing/opentracing-go/log"
+
+	gitserverOptions "github.com/sourcegraph/sourcegraph/internal/gitserver"
+	"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
 	"github.com/sourcegraph/sourcegraph/internal/goroutine"
+	"github.com/sourcegraph/sourcegraph/internal/observation"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
+func (s *Service) NewCommitGraphUpdater(interval time.Duration, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) goroutine.BackgroundRoutine {
+	return goroutine.NewPeriodicGoroutine(context.Background(), interval, goroutine.HandlerFunc(func(ctx context.Context) error {
+		return s.updateAllDirtyCommitGraphs(ctx, maxAgeForNonStaleBranches, maxAgeForNonStaleTags)
+	}))
+}
+
 // Handle periodically re-calculates the commit and upload visibility graph for repositories
 // that are marked as dirty by the worker process. This is done out-of-band from the rest of
 // the upload processing as it is likely that we are processing multiple uploads concurrently
 // for the same repository and should not repeat the work since the last calculation performed
 // will always be the one we want.
 
-func (s *Service) NewUpdater(interval time.Duration, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) goroutine.BackgroundRoutine {
-	return goroutine.NewPeriodicGoroutine(context.Background(), interval, goroutine.HandlerFunc(func(ctx context.Context) error {
-		return s.UpdateDirtyRepositories(ctx, maxAgeForNonStaleBranches, maxAgeForNonStaleTags)
-	}))
+func (s *Service) updateAllDirtyCommitGraphs(ctx context.Context, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) (err error) {
+	ctx, _, endObservation := s.operations.updateDirtyRepositories.With(ctx, &err, observation.Args{
+		LogFields: []log.Field{
+			log.Int("maxAgeForNonStaleBranches in ms", int(maxAgeForNonStaleBranches.Milliseconds())),
+			log.Int("maxAgeForNonStaleTags in ms", int(maxAgeForNonStaleTags.Milliseconds())),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	repositoryIDs, err := s.GetDirtyRepositories(ctx)
+	if err != nil {
+		return errors.Wrap(err, "uploadSvc.DirtyRepositories")
+	}
+
+	var updateErr error
+	for repositoryID, dirtyFlag := range repositoryIDs {
+		if err := s.lockAndUpdateUploadsVisibleToCommits(ctx, repositoryID, dirtyFlag, maxAgeForNonStaleBranches, maxAgeForNonStaleTags); err != nil {
+			if updateErr == nil {
+				updateErr = err
+			} else {
+				updateErr = errors.Append(updateErr, err)
+			}
+		}
+	}
+
+	return updateErr
+}
+
+func (s *Service) GetDirtyRepositories(ctx context.Context) (_ map[int]int, err error) {
+	ctx, _, endObservation := s.operations.getDirtyRepositories.With(ctx, &err, observation.Args{})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.GetDirtyRepositories(ctx)
+}
+
+// lockAndUpdateUploadsVisibleToCommits will call UpdateUploadsVisibleToCommits while holding an advisory lock to give exclusive access to the
+// update procedure for this repository. If the lock is already held, this method will simply do nothing.
+func (s *Service) lockAndUpdateUploadsVisibleToCommits(ctx context.Context, repositoryID, dirtyToken int, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) (err error) {
+	ctx, trace, endObservation := s.operations.updateUploadsVisibleToCommits.With(ctx, &err, observation.Args{
+		LogFields: []log.Field{
+			log.Int("repositoryID", repositoryID),
+			log.Int("dirtyToken", dirtyToken),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	ok, unlock, err := s.locker.Lock(ctx, int32(repositoryID), false)
+	if err != nil || !ok {
+		return errors.Wrap(err, "locker.Lock")
+	}
+	defer func() {
+		err = unlock(err)
+	}()
+
+	// The following process pulls the commit graph for the given repository from gitserver, pulls the set of LSIF
+	// upload objects for the given repository from Postgres, and correlates them into a visibility
+	// graph. This graph is then upserted back into Postgres for use by find closest dumps queries.
+	//
+	// The user should supply a dirty token that is associated with the given repository so that
+	// the repository can be unmarked as long as the repository is not marked as dirty again before
+	// the update completes.
+
+	// Construct a view of the git graph that we will later decorate with upload information.
+	commitGraph, err := s.getCommitGraph(ctx, repositoryID)
+	if err != nil {
+		return err
+	}
+	trace.Log(log.Int("numCommitGraphKeys", len(commitGraph.Order())))
+
+	refDescriptions, err := s.gitserverClient.RefDescriptions(ctx, repositoryID)
+	if err != nil {
+		return errors.Wrap(err, "gitserver.RefDescriptions")
+	}
+	trace.Log(log.Int("numRefDescriptions", len(refDescriptions)))
+
+	// Decorate the commit graph with the set of processed uploads are visible from each commit,
+	// then bulk update the denormalized view in Postgres. We call this with an empty graph as well
+	// so that we end up clearing the stale data and bulk inserting nothing.
+	if err := s.UpdateUploadsVisibleToCommits(ctx, repositoryID, commitGraph, refDescriptions, maxAgeForNonStaleBranches, maxAgeForNonStaleTags, dirtyToken, time.Time{}); err != nil {
+		return errors.Wrap(err, "uploadSvc.UpdateUploadsVisibleToCommits")
+	}
+
+	return nil
+}
+
+// getCommitGraph builds a partial commit graph that includes the most recent commits on each branch
+// extending back as as the date of the oldest commit for which we have a processed upload for this
+// repository.
+//
+// This optimization is necessary as decorating the commit graph is an operation that scales with
+// the size of both the git graph and the number of uploads (multiplicatively). For repositories with
+// a very large number of commits or distinct roots (most monorepos) this is a necessary optimization.
+//
+// The number of commits pulled back here should not grow over time unless the repo is growing at an
+// accelerating rate, as we routinely expire old information for active repositories in a janitor
+// process.
+func (s *Service) getCommitGraph(ctx context.Context, repositoryID int) (*gitdomain.CommitGraph, error) {
+	commitDate, ok, err := s.GetOldestCommitDate(ctx, repositoryID)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		// No uploads exist for this repository
+		return gitdomain.ParseCommitGraph(nil), nil
+	}
+
+	// The --since flag for git log is exclusive, but we want to include the commit where the
+	// oldest dump is defined. This flag only has second resolution, so we shouldn't be pulling
+	// back any more data than we wanted.
+	commitDate = commitDate.Add(-time.Second)
+
+	commitGraph, err := s.gitserverClient.CommitGraph(ctx, repositoryID, gitserverOptions.CommitGraphOptions{
+		AllRefs: true,
+		Since:   &commitDate,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "gitserver.CommitGraph")
+	}
+
+	return commitGraph, nil
+}
+
+func (s *Service) UpdateUploadsVisibleToCommits(ctx context.Context, repositoryID int, graph *gitdomain.CommitGraph, refDescriptions map[string][]gitdomain.RefDescription, maxAgeForNonStaleBranches, maxAgeForNonStaleTags time.Duration, dirtyToken int, now time.Time) (err error) {
+	ctx, _, endObservation := s.operations.updateUploadsVisibleToCommits.With(ctx, &err, observation.Args{
+		LogFields: []otlog.Field{
+			otlog.Int("repositoryID", repositoryID),
+			otlog.String("graph", fmt.Sprintf("%v", graph)),
+			otlog.String("refDescriptions", fmt.Sprintf("%v", refDescriptions)),
+			otlog.String("maxAgeForNonStaleBranches", maxAgeForNonStaleBranches.String()),
+			otlog.String("maxAgeForNonStaleTags", maxAgeForNonStaleTags.String()),
+			otlog.Int("dirtyToken", dirtyToken),
+			otlog.String("now", now.String()),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.UpdateUploadsVisibleToCommits(ctx, repositoryID, graph, refDescriptions, maxAgeForNonStaleBranches, maxAgeForNonStaleTags, dirtyToken, now)
 }

internal/codeintel/uploads/expirer.go

@@ -4,16 +4,18 @@ import (
 	"context"
 	"time"
 
+	otlog "github.com/opentracing/opentracing-go/log"
 	"github.com/sourcegraph/log"
 
 	policiesEnterprise "github.com/sourcegraph/sourcegraph/internal/codeintel/policies/enterprise"
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/types"
 	"github.com/sourcegraph/sourcegraph/internal/goroutine"
+	"github.com/sourcegraph/sourcegraph/internal/observation"
 	"github.com/sourcegraph/sourcegraph/internal/timeutil"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
-func (s *Service) NewExpirer(
+func (s *Service) NewUploadExpirer(
 	interval time.Duration,
 	repositoryProcessDelay time.Duration,
 	repositoryBatchSize int,
@@ -23,7 +25,7 @@ func (s *Service) NewExpirer(
 	policyBatchSize int,
 ) goroutine.BackgroundRoutine {
 	return goroutine.NewPeriodicGoroutine(context.Background(), interval, goroutine.HandlerFunc(func(ctx context.Context) error {
-		return s.handleUploadExpirer(ctx, expirerConfig{
+		return s.handleExpiredUploadsBatch(ctx, expirerConfig{
 			repositoryProcessDelay: repositoryProcessDelay,
 			repositoryBatchSize:    repositoryBatchSize,
 			uploadProcessDelay:     uploadProcessDelay,
@@ -43,12 +45,12 @@ type expirerConfig struct {
 	policyBatchSize        int
 }
 
-// HandleUploadExpirer compares the age of upload records against the age of uploads
+// handleExpiredUploadsBatch compares the age of upload records against the age of uploads
 // protected by global and repository specific data retention policies.
 //
 // Uploads that are older than the protected retention age are marked as expired. Expired records with
 // no dependents will be removed by the expiredUploadDeleter.
-func (e *Service) handleUploadExpirer(ctx context.Context, cfg expirerConfig) (err error) {
+func (e *Service) handleExpiredUploadsBatch(ctx context.Context, cfg expirerConfig) (err error) {
 	// Get the batch of repositories that we'll handle in this invocation of the periodic goroutine. This
 	// set should contain repositories that have yet to be updated, or that have been updated least recently.
 	// This allows us to update every repository reliably, even if it takes a long time to process through
@@ -79,6 +81,18 @@ func (e *Service) handleUploadExpirer(ctx context.Context, cfg expirerConfig) (e
 	return err
 }
 
+func (s *Service) SetRepositoriesForRetentionScan(ctx context.Context, processDelay time.Duration, limit int) (_ []int, err error) {
+	ctx, _, endObservation := s.operations.setRepositoriesForRetentionScan.With(ctx, &err, observation.Args{
+		LogFields: []otlog.Field{
+			otlog.Int("processDelay in ms", int(processDelay.Milliseconds())),
+			otlog.Int("limit", limit),
+		},
+	})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.SetRepositoriesForRetentionScan(ctx, processDelay, limit)
+}
+
 func (e *Service) handleRepository(ctx context.Context, repositoryID int, cfg expirerConfig, now time.Time) error {
 	e.expirationMetrics.numRepositoriesScanned.Inc()
 

internal/codeintel/uploads/expirer_test.go

@@ -35,7 +35,7 @@ func TestUploadExpirer(t *testing.T) {
 		clock:             clock,
 	}
 
-	if err := uploadExpirer.handleUploadExpirer(context.Background(), expirerConfig{
+	if err := uploadExpirer.handleExpiredUploadsBatch(context.Background(), expirerConfig{
 		repositoryProcessDelay: 24 * time.Hour,
 		repositoryBatchSize:    100,
 		uploadProcessDelay:     24 * time.Hour,

internal/codeintel/uploads/refcount.go

@@ -1,14 +0,0 @@
-package uploads
-
-import (
-	"context"
-	"time"
-
-	"github.com/sourcegraph/sourcegraph/internal/goroutine"
-)
-
-func (s *Service) NewReferenceCountUpdater(interval time.Duration, batchSize int) goroutine.BackgroundRoutine {
-	return goroutine.NewPeriodicGoroutine(context.Background(), interval, goroutine.HandlerFunc(func(ctx context.Context) error {
-		return s.BackfillReferenceCountBatch(ctx, batchSize)
-	}))
-}

internal/codeintel/uploads/reference_counts.go

@@ -0,0 +1,26 @@
+package uploads
+
+import (
+	"context"
+	"time"
+
+	"github.com/opentracing/opentracing-go/log"
+
+	"github.com/sourcegraph/sourcegraph/internal/goroutine"
+	"github.com/sourcegraph/sourcegraph/internal/observation"
+)
+
+func (s *Service) NewReferenceCountUpdater(interval time.Duration, batchSize int) goroutine.BackgroundRoutine {
+	return goroutine.NewPeriodicGoroutine(context.Background(), interval, goroutine.HandlerFunc(func(ctx context.Context) error {
+		return s.backfillReferenceCountBatch(ctx, batchSize)
+	}))
+}
+
+func (s *Service) backfillReferenceCountBatch(ctx context.Context, batchSize int) (err error) {
+	ctx, _, endObservation := s.operations.backfillReferenceCountBatch.With(ctx, &err, observation.Args{
+		LogFields: []log.Field{log.Int("batchSize", batchSize)},
+	})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.BackfillReferenceCountBatch(ctx, batchSize)
+}

internal/codeintel/uploads/service.go

@@ -16,7 +16,6 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/store"
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared"
 	"github.com/sourcegraph/sourcegraph/internal/gitserver"
-	gitserverOptions "github.com/sourcegraph/sourcegraph/internal/gitserver"
 	"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
 	"github.com/sourcegraph/sourcegraph/internal/observation"
 	"github.com/sourcegraph/sourcegraph/internal/workerutil/dbworker"
@@ -42,7 +41,6 @@ type service interface {
 	GetRepositoriesMaxStaleAge(ctx context.Context) (_ time.Duration, err error)
 	GetDirtyRepositories(ctx context.Context) (_ map[int]int, err error)
 	SetRepositoryAsDirty(ctx context.Context, repositoryID int) (err error)
-	UpdateDirtyRepositories(ctx context.Context, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) (err error)
 	SetRepositoriesForRetentionScan(ctx context.Context, processDelay time.Duration, limit int) (_ []int, err error)
 
 	// Uploads
@@ -164,24 +162,6 @@ func (s *Service) GetCommitGraphMetadata(ctx context.Context, repositoryID int)
 	return s.store.GetCommitGraphMetadata(ctx, repositoryID)
 }
 
-func (s *Service) UpdateSourcedCommits(ctx context.Context, repositoryID int, commit string, now time.Time) (uploadsUpdated int, err error) {
-	ctx, _, endObservation := s.operations.updateSourcedCommits.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{log.Int("repositoryID", repositoryID), log.String("commit", commit), log.String("now", now.String())},
-	})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.UpdateSourcedCommits(ctx, repositoryID, commit, now)
-}
-
-func (s *Service) DeleteSourcedCommits(ctx context.Context, repositoryID int, commit string, maximumCommitLag time.Duration, now time.Time) (uploadsUpdated int, uploadsDeleted int, err error) {
-	ctx, _, endObservation := s.operations.deleteSourcedCommits.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{log.Int("repositoryID", repositoryID), log.String("commit", commit), log.Int("maximumCommitLag in ms", int(maximumCommitLag.Milliseconds())), log.String("now", now.String())},
-	})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.DeleteSourcedCommits(ctx, repositoryID, commit, maximumCommitLag, now)
-}
-
 func (s *Service) GetOldestCommitDate(ctx context.Context, repositoryID int) (time.Time, bool, error) {
 	ctx, _, endObservation := s.operations.getOldestCommitDate.With(ctx, nil, observation.Args{
 		LogFields: []log.Field{log.Int("repositoryID", repositoryID)},
@@ -200,130 +180,6 @@ func (s *Service) SetRepositoryAsDirty(ctx context.Context, repositoryID int) (e
 	return s.store.SetRepositoryAsDirty(ctx, repositoryID)
 }
 
-func (s *Service) UpdateDirtyRepositories(ctx context.Context, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) (err error) {
-	ctx, _, endObservation := s.operations.updateDirtyRepositories.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{
-			log.Int("maxAgeForNonStaleBranches in ms", int(maxAgeForNonStaleBranches.Milliseconds())),
-			log.Int("maxAgeForNonStaleTags in ms", int(maxAgeForNonStaleTags.Milliseconds())),
-		},
-	})
-	defer endObservation(1, observation.Args{})
-
-	repositoryIDs, err := s.GetDirtyRepositories(ctx)
-	if err != nil {
-		return errors.Wrap(err, "uploadSvc.DirtyRepositories")
-	}
-
-	var updateErr error
-	for repositoryID, dirtyFlag := range repositoryIDs {
-		if err := s.lockAndUpdateUploadsVisibleToCommits(ctx, repositoryID, dirtyFlag, maxAgeForNonStaleBranches, maxAgeForNonStaleTags); err != nil {
-			if updateErr == nil {
-				updateErr = err
-			} else {
-				updateErr = errors.Append(updateErr, err)
-			}
-		}
-	}
-
-	return updateErr
-}
-
-// lockAndUpdateUploadsVisibleToCommits will call UpdateUploadsVisibleToCommits while holding an advisory lock to give exclusive access to the
-// update procedure for this repository. If the lock is already held, this method will simply do nothing.
-func (s *Service) lockAndUpdateUploadsVisibleToCommits(ctx context.Context, repositoryID, dirtyToken int, maxAgeForNonStaleBranches time.Duration, maxAgeForNonStaleTags time.Duration) (err error) {
-	ctx, trace, endObservation := s.operations.updateUploadsVisibleToCommits.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{
-			log.Int("repositoryID", repositoryID),
-			log.Int("dirtyToken", dirtyToken),
-		},
-	})
-	defer endObservation(1, observation.Args{})
-
-	ok, unlock, err := s.locker.Lock(ctx, int32(repositoryID), false)
-	if err != nil || !ok {
-		return errors.Wrap(err, "locker.Lock")
-	}
-	defer func() {
-		err = unlock(err)
-	}()
-
-	// The following process pulls the commit graph for the given repository from gitserver, pulls the set of LSIF
-	// upload objects for the given repository from Postgres, and correlates them into a visibility
-	// graph. This graph is then upserted back into Postgres for use by find closest dumps queries.
-	//
-	// The user should supply a dirty token that is associated with the given repository so that
-	// the repository can be unmarked as long as the repository is not marked as dirty again before
-	// the update completes.
-
-	// Construct a view of the git graph that we will later decorate with upload information.
-	commitGraph, err := s.getCommitGraph(ctx, repositoryID)
-	if err != nil {
-		return err
-	}
-	trace.Log(log.Int("numCommitGraphKeys", len(commitGraph.Order())))
-
-	refDescriptions, err := s.gitserverClient.RefDescriptions(ctx, repositoryID)
-	if err != nil {
-		return errors.Wrap(err, "gitserver.RefDescriptions")
-	}
-	trace.Log(log.Int("numRefDescriptions", len(refDescriptions)))
-
-	// Decorate the commit graph with the set of processed uploads are visible from each commit,
-	// then bulk update the denormalized view in Postgres. We call this with an empty graph as well
-	// so that we end up clearing the stale data and bulk inserting nothing.
-	if err := s.UpdateUploadsVisibleToCommits(ctx, repositoryID, commitGraph, refDescriptions, maxAgeForNonStaleBranches, maxAgeForNonStaleTags, dirtyToken, time.Time{}); err != nil {
-		return errors.Wrap(err, "uploadSvc.UpdateUploadsVisibleToCommits")
-	}
-
-	return nil
-}
-
-// getCommitGraph builds a partial commit graph that includes the most recent commits on each branch
-// extending back as as the date of the oldest commit for which we have a processed upload for this
-// repository.
-//
-// This optimization is necessary as decorating the commit graph is an operation that scales with
-// the size of both the git graph and the number of uploads (multiplicatively). For repositories with
-// a very large number of commits or distinct roots (most monorepos) this is a necessary optimization.
-//
-// The number of commits pulled back here should not grow over time unless the repo is growing at an
-// accelerating rate, as we routinely expire old information for active repositories in a janitor
-// process.
-func (s *Service) getCommitGraph(ctx context.Context, repositoryID int) (*gitdomain.CommitGraph, error) {
-	commitDate, ok, err := s.GetOldestCommitDate(ctx, repositoryID)
-	if err != nil {
-		return nil, err
-	}
-	if !ok {
-		// No uploads exist for this repository
-		return gitdomain.ParseCommitGraph(nil), nil
-	}
-
-	// The --since flag for git log is exclusive, but we want to include the commit where the
-	// oldest dump is defined. This flag only has second resolution, so we shouldn't be pulling
-	// back any more data than we wanted.
-	commitDate = commitDate.Add(-time.Second)
-
-	commitGraph, err := s.gitserverClient.CommitGraph(ctx, repositoryID, gitserverOptions.CommitGraphOptions{
-		AllRefs: true,
-		Since:   &commitDate,
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "gitserver.CommitGraph")
-	}
-
-	return commitGraph, nil
-}
-
-func (s *Service) SetRepositoriesForRetentionScan(ctx context.Context, processDelay time.Duration, limit int) (_ []int, err error) {
-	ctx, _, endObservation := s.operations.setRepositoriesForRetentionScan.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{log.Int("processDelay in ms", int(processDelay.Milliseconds())), log.Int("limit", limit)},
-	})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.SetRepositoriesForRetentionScan(ctx, processDelay, limit)
-}
-
 func (s *Service) GetRepoName(ctx context.Context, repositoryID int) (_ string, err error) {
 	ctx, _, endObservation := s.operations.getRepoName.With(ctx, &err, observation.Args{LogFields: []log.Field{log.Int("repositoryID", repositoryID)}})
 	defer endObservation(1, observation.Args{})
@@ -354,13 +210,6 @@ func (s *Service) GetRepositoriesMaxStaleAge(ctx context.Context) (_ time.Durati
 	return s.store.GetRepositoriesMaxStaleAge(ctx)
 }
 
-func (s *Service) GetDirtyRepositories(ctx context.Context) (_ map[int]int, err error) {
-	ctx, _, endObservation := s.operations.getDirtyRepositories.With(ctx, &err, observation.Args{})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.GetDirtyRepositories(ctx)
-}
-
 func (s *Service) GetUploads(ctx context.Context, opts types.GetUploadsOptions) (uploads []types.Upload, totalCount int, err error) {
 	ctx, _, endObservation := s.operations.getUploads.With(ctx, &err, observation.Args{
 		LogFields: []log.Field{log.Int("repositoryID", opts.RepositoryID), log.String("state", opts.State), log.String("term", opts.Term)},
@@ -401,23 +250,6 @@ func (s *Service) GetUploadIDsWithReferences(ctx context.Context, orderedMoniker
 	return s.store.GetUploadIDsWithReferences(ctx, orderedMonikers, ignoreIDs, repositoryID, commit, limit, offset, trace)
 }
 
-func (s *Service) UpdateUploadsVisibleToCommits(ctx context.Context, repositoryID int, graph *gitdomain.CommitGraph, refDescriptions map[string][]gitdomain.RefDescription, maxAgeForNonStaleBranches, maxAgeForNonStaleTags time.Duration, dirtyToken int, now time.Time) (err error) {
-	ctx, _, endObservation := s.operations.updateUploadsVisibleToCommits.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{
-			log.Int("repositoryID", repositoryID),
-			log.String("graph", fmt.Sprintf("%v", graph)),
-			log.String("refDescriptions", fmt.Sprintf("%v", refDescriptions)),
-			log.String("maxAgeForNonStaleBranches", maxAgeForNonStaleBranches.String()),
-			log.String("maxAgeForNonStaleTags", maxAgeForNonStaleTags.String()),
-			log.Int("dirtyToken", dirtyToken),
-			log.String("now", now.String()),
-		},
-	})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.UpdateUploadsVisibleToCommits(ctx, repositoryID, graph, refDescriptions, maxAgeForNonStaleBranches, maxAgeForNonStaleTags, dirtyToken, now)
-}
-
 func (s *Service) UpdateUploadRetention(ctx context.Context, protectedIDs, expiredIDs []int) (err error) {
 	ctx, _, endObservation := s.operations.updateUploadRetention.With(ctx, &err, observation.Args{
 		LogFields: []log.Field{log.String("protectedIDs", fmt.Sprintf("%v", protectedIDs)), log.String("expiredIDs", fmt.Sprintf("%v", expiredIDs))},
@@ -426,16 +258,6 @@ func (s *Service) UpdateUploadRetention(ctx context.Context, protectedIDs, expir
 
 	return s.store.UpdateUploadRetention(ctx, protectedIDs, expiredIDs)
 }
-
-func (s *Service) BackfillReferenceCountBatch(ctx context.Context, batchSize int) (err error) {
-	ctx, _, endObservation := s.operations.backfillReferenceCountBatch.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{log.Int("batchSize", batchSize)},
-	})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.BackfillReferenceCountBatch(ctx, batchSize)
-}
-
 func (s *Service) UpdateUploadsReferenceCounts(ctx context.Context, ids []int, dependencyUpdateType shared.DependencyReferenceCountUpdateType) (updated int, err error) {
 	ctx, _, endObservation := s.operations.updateUploadsReferenceCounts.With(ctx, &err, observation.Args{
 		LogFields: []log.Field{log.String("ids", fmt.Sprintf("%v", ids))},
@@ -445,22 +267,6 @@ func (s *Service) UpdateUploadsReferenceCounts(ctx context.Context, ids []int, d
 	return s.store.UpdateUploadsReferenceCounts(ctx, ids, dependencyUpdateType)
 }
 
-func (s *Service) SoftDeleteExpiredUploads(ctx context.Context) (count int, err error) {
-	ctx, _, endObservation := s.operations.softDeleteExpiredUploads.With(ctx, &err, observation.Args{})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.SoftDeleteExpiredUploads(ctx)
-}
-
-func (s *Service) DeleteUploadsStuckUploading(ctx context.Context, uploadedBefore time.Time) (_ int, err error) {
-	ctx, _, endObservation := s.operations.deleteUploadsStuckUploading.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{log.String("uploadedBefore", uploadedBefore.String())},
-	})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.DeleteUploadsStuckUploading(ctx, uploadedBefore)
-}
-
 func (s *Service) DeleteUploadsWithoutRepository(ctx context.Context, now time.Time) (_ map[int]int, err error) {
 	ctx, _, endObservation := s.operations.deleteUploadsWithoutRepository.With(ctx, &err, observation.Args{
 		LogFields: []log.Field{log.String("now", now.String())},
@@ -600,46 +406,6 @@ func (s *Service) GetDumpsByIDs(ctx context.Context, ids []int) (_ []types.Dump,
 	return s.store.GetDumpsByIDs(ctx, ids)
 }
 
-func (s *Service) HardDeleteExpiredUploads(ctx context.Context) (count int, err error) {
-	ctx, _, endObservation := s.operations.hardDeleteUploads.With(ctx, &err, observation.Args{})
-	defer endObservation(1, observation.Args{})
-
-	const uploadsBatchSize = 100
-	options := types.GetUploadsOptions{
-		State:            "deleted",
-		Limit:            uploadsBatchSize,
-		AllowExpired:     true,
-		AllowDeletedRepo: true,
-	}
-
-	for {
-		// Always request the first page of deleted uploads. If this is not
-		// the first iteration of the loop, then the previous iteration has
-		// deleted the records that composed the previous page, and the
-		// previous "second" page is now the first page.
-		uploads, totalCount, err := s.store.GetUploads(ctx, options)
-		if err != nil {
-			return 0, errors.Wrap(err, "store.GetUploads")
-		}
-
-		ids := uploadIDs(uploads)
-		if err := s.lsifstore.DeleteLsifDataByUploadIds(ctx, ids...); err != nil {
-			return 0, errors.Wrap(err, "lsifstore.Clear")
-		}
-
-		if err := s.store.HardDeleteUploadsByIDs(ctx, ids...); err != nil {
-			return 0, errors.Wrap(err, "store.HardDeleteUploadsByIDs")
-		}
-
-		count += len(uploads)
-		if count >= totalCount {
-			break
-		}
-	}
-
-	return count, nil
-}
-
 func (s *Service) UpdatePackages(ctx context.Context, dumpID int, packages []precise.Package) (err error) {
 	ctx, _, endObservation := s.operations.updatePackages.With(ctx, &err, observation.Args{
 		LogFields: []log.Field{log.Int("dumpID", dumpID), log.String("packages", fmt.Sprintf("%v", packages))},
@@ -676,56 +442,6 @@ func (s *Service) GetAuditLogsForUpload(ctx context.Context, uploadID int) (_ []
 	return s.store.GetAuditLogsForUpload(ctx, uploadID)
 }
 
-func (s *Service) DeleteOldAuditLogs(ctx context.Context, maxAge time.Duration, now time.Time) (count int, err error) {
-	ctx, _, endObservation := s.operations.deleteOldAuditLogs.With(ctx, &err, observation.Args{
-		LogFields: []log.Field{log.String("maxAge", maxAge.String()), log.String("now", now.String())},
-	})
-	defer endObservation(1, observation.Args{})
-
-	return s.store.DeleteOldAuditLogs(ctx, maxAge, now)
-}
-
-// BackfillCommittedAtBatch calculates the committed_at value for a batch of upload records that do not have
-// this value set. This method is used to backfill old upload records prior to this value being reliably set
-// during processing.
-func (s *Service) BackfillCommittedAtBatch(ctx context.Context, batchSize int) (err error) {
-	ctx, _, endObservation := s.operations.backfillCommittedAtBatch.With(ctx, &err, observation.Args{LogFields: []log.Field{
-		log.Int("batchSize", batchSize),
-	}})
-	defer endObservation(1, observation.Args{})
-
-	tx, err := s.store.Transact(ctx)
-	defer func() {
-		err = tx.Done(err)
-	}()
-
-	batch, err := tx.SourcedCommitsWithoutCommittedAt(ctx, batchSize)
-	if err != nil {
-		return errors.Wrap(err, "store.SourcedCommitsWithoutCommittedAt")
-	}
-
-	for _, sourcedCommits := range batch {
-		for _, commit := range sourcedCommits.Commits {
-			commitDateString, err := s.getCommitDate(ctx, sourcedCommits.RepositoryID, commit)
-			if err != nil {
-				return err
-			}
-
-			// Update commit date of all uploads attached to this this repository and commit
-			if err := tx.UpdateCommittedAt(ctx, sourcedCommits.RepositoryID, commit, commitDateString); err != nil {
-				return errors.Wrap(err, "store.UpdateCommittedAt")
-			}
-		}
-
-		// Mark repository as dirty so the commit graph is recalculated with fresh data
-		if err := tx.SetRepositoryAsDirty(ctx, sourcedCommits.RepositoryID); err != nil {
-			return errors.Wrap(err, "store.SetRepositoryAsDirty")
-		}
-	}
-
-	return nil
-}
-
 func (s *Service) GetUploadDocumentsForPath(ctx context.Context, bundleID int, pathPattern string) (_ []string, _ int, err error) {
 	ctx, _, endObservation := s.operations.getUploadDocumentsForPath.With(ctx, &err, observation.Args{
 		LogFields: []log.Field{log.Int("bundleID", bundleID), log.String("pathPattern", pathPattern)},
@@ -753,26 +469,6 @@ func (s *Service) GetLastUploadRetentionScanForRepository(ctx context.Context, r
 	return s.store.GetLastUploadRetentionScanForRepository(ctx, repositoryID)
 }
 
-func (s *Service) getCommitDate(ctx context.Context, repositoryID int, commit string) (string, error) {
-	_, commitDate, revisionExists, err := s.gitserverClient.CommitDate(ctx, repositoryID, commit)
-	if err != nil {
-		return "", errors.Wrap(err, "gitserver.CommitDate")
-	}
-
-	var commitDateString string
-	if revisionExists {
-		commitDateString = commitDate.Format(time.RFC3339)
-	} else {
-		// Set a value here that we'll filter out on the query side so that we don't
-		// reprocess the same failing batch infinitely. We could alternatively soft
-		// delete the record, but it would be better to keep record deletion behavior
-		// together in the same place (so we have unified metrics on that event).
-		commitDateString = "-infinity"
-	}
-
-	return commitDateString, nil
-}
-
 func (s *Service) GetListTags(ctx context.Context, repo api.RepoName, commitObjs ...string) (_ []*gitdomain.Tag, err error) {
 	ctx, _, endObservation := s.operations.getListTags.With(ctx, &err, observation.Args{
 		LogFields: []log.Field{log.String("repo", string(repo)), log.String("commitObjs", fmt.Sprintf("%v", commitObjs))},