package uploads import ( "context" "time" "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/commitgraph" "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/lsifstore" "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/store" "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared" uploadsshared "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/lib/codeintel/precise" "github.com/sourcegraph/sourcegraph/lib/errors" ) type Service struct { store store.Store repoStore RepoStore lsifstore lsifstore.Store gitserverClient gitserver.Client operations *operations } func newService( observationCtx *observation.Context, store store.Store, repoStore RepoStore, lsifstore lsifstore.Store, gsc gitserver.Client, ) *Service { return &Service{ store: store, repoStore: repoStore, lsifstore: lsifstore, gitserverClient: gsc, operations: newOperations(observationCtx), } } func (s *Service) GetCommitsVisibleToUpload(ctx context.Context, uploadID, limit int, token *string) ([]string, *string, error) { return s.store.GetCommitsVisibleToUpload(ctx, uploadID, limit, token) } func (s *Service) GetCommitGraphMetadata(ctx context.Context, repositoryID int) (bool, *time.Time, error) { return s.store.GetCommitGraphMetadata(ctx, repositoryID) } func (s *Service) GetDirtyRepositories(ctx context.Context) (_ []shared.DirtyRepository, err error) { return s.store.GetDirtyRepositories(ctx) } func (s *Service) GetIndexers(ctx context.Context, opts shared.GetIndexersOptions) ([]string, error) { return s.store.GetIndexers(ctx, opts) } func (s *Service) GetUploads(ctx context.Context, opts shared.GetUploadsOptions) ([]shared.Upload, int, error) { return s.store.GetUploads(ctx, opts) } func (s *Service) GetUploadByID(ctx context.Context, id int) (shared.Upload, bool, error) { return s.store.GetUploadByID(ctx, id) } func (s *Service) GetUploadsByIDs(ctx context.Context, ids ...int) ([]shared.Upload, error) { return s.store.GetUploadsByIDs(ctx, ids...) } func (s *Service) GetUploadIDsWithReferences(ctx context.Context, orderedMonikers []precise.QualifiedMonikerData, ignoreIDs []int, repositoryID int, commit string, limit int, offset int) ([]int, int, int, error) { return s.store.GetUploadIDsWithReferences(ctx, orderedMonikers, ignoreIDs, repositoryID, commit, limit, offset, nil) } func (s *Service) DeleteUploadByID(ctx context.Context, id int) (bool, error) { return s.store.DeleteUploadByID(ctx, id) } func (s *Service) DeleteUploads(ctx context.Context, opts shared.DeleteUploadsOptions) error { return s.store.DeleteUploads(ctx, opts) } func (s *Service) GetRepositoriesMaxStaleAge(ctx context.Context) (_ time.Duration, err error) { return s.store.GetRepositoriesMaxStaleAge(ctx) } // numAncestors is the number of ancestors to query from gitserver when trying to find the closest // ancestor we have data for. Setting this value too low (relative to a repository's commit rate) // will cause requests for an unknown commit return too few results; setting this value too high // will raise the latency of requests for an unknown commit. // // TODO(efritz) - make adjustable via site configuration const numAncestors = 100 // InferClosestUploads will return the set of visible uploads for the given commit. If this commit is // newer than our last refresh of the lsif_nearest_uploads table for this repository, then we will mark // the repository as dirty and quickly approximate the correct set of visible uploads. // // Because updating the entire commit graph is a blocking, expensive, and lock-guarded process, we want // to only do that in the background and do something chearp in latency-sensitive paths. To construct an // approximate result, we query gitserver for a (relatively small) set of ancestors for the given commit, // correlate that with the upload data we have for those commits, and re-run the visibility algorithm over // the graph. This will not always produce the full set of visible commits - some responses may not contain // all results while a subsequent request made after the lsif_nearest_uploads has been updated to include // this commit will. func (s *Service) InferClosestUploads(ctx context.Context, opts shared.UploadMatchingOptions) (_ []shared.CompletedUpload, err error) { ctx, _, endObservation := s.operations.inferClosestUploads.With(ctx, &err, observation.Args{Attrs: opts.Attrs()}) defer endObservation(1, observation.Args{}) repo, err := s.repoStore.Get(ctx, api.RepoID(opts.RepositoryID)) if err != nil { return nil, err } // The parameters exactPath and rootMustEnclosePath align here: if we're looking for completed uploads // that can answer queries for a directory (e.g. diagnostics), we want any completed upload that happens // to intersect the target directory. If we're looking for completed uploads that can answer queries for // a single file, then we need a completed upload with a root that properly encloses that file. if uploads, err := s.store.FindClosestCompletedUploads(ctx, opts); err != nil { return nil, errors.Wrap(err, "store.FindClosestCompletedUploads") } else if len(uploads) != 0 { return uploads, nil } // Repository has no LSIF data at all if repositoryExists, err := s.store.HasRepository(ctx, opts.RepositoryID); err != nil { return nil, errors.Wrap(err, "dbstore.HasRepository") } else if !repositoryExists { return nil, nil } // Commit is known and the empty completed uploads list explicitly means nothing is visible if commitExists, err := s.store.HasCommit(ctx, opts.RepositoryID, opts.Commit); err != nil { return nil, errors.Wrap(err, "dbstore.HasCommit") } else if commitExists { return nil, nil } // Otherwise, the repository has LSIF data but we don't know about the commit. This commit // is probably newer than our last upload. Pull back a portion of the updated commit graph // and try to link it with what we have in the database. Then mark the repository's commit // graph as dirty so it's updated for subsequent requests. commits, err := s.gitserverClient.Commits(ctx, repo.Name, gitserver.CommitsOptions{ Ranges: []string{string(opts.Commit)}, N: numAncestors, Order: gitserver.CommitsOrderTopoDate, }) if err != nil { return nil, errors.Wrap(err, "gitserverClient.Commits") } graph := commitgraph.ParseCommitGraph(commits) uploads, err := s.store.FindClosestCompletedUploadsFromGraphFragment(ctx, opts, graph) if err != nil { return nil, errors.Wrap(err, "dbstore.FindClosestCompletedUploadsFromGraphFragment") } if err := s.store.SetRepositoryAsDirty(ctx, int(opts.RepositoryID)); err != nil { return nil, errors.Wrap(err, "dbstore.MarkRepositoryAsDirty") } return uploads, nil } func (s *Service) GetCompletedUploadsWithDefinitionsForMonikers(ctx context.Context, monikers []precise.QualifiedMonikerData) ([]shared.CompletedUpload, error) { return s.store.GetCompletedUploadsWithDefinitionsForMonikers(ctx, monikers) } func (s *Service) GetCompletedUploadsByIDs(ctx context.Context, ids []int) ([]shared.CompletedUpload, error) { return s.store.GetCompletedUploadsByIDs(ctx, ids) } func (s *Service) ReferencesForUpload(ctx context.Context, uploadID int) (shared.PackageReferenceScanner, error) { return s.store.ReferencesForUpload(ctx, uploadID) } func (s *Service) GetAuditLogsForUpload(ctx context.Context, uploadID int) ([]shared.UploadLog, error) { return s.store.GetAuditLogsForUpload(ctx, uploadID) } // func (s *Service) GetUploadDocumentsForPath(ctx context.Context, bundleID int, pathPattern string) ([]string, int, error) { // return s.lsifstore.GetUploadDocumentsForPath(ctx, bundleID, pathPattern) // } func (s *Service) GetRecentUploadsSummary(ctx context.Context, repositoryID int) ([]shared.UploadsWithRepositoryNamespace, error) { return s.store.GetRecentUploadsSummary(ctx, repositoryID) } func (s *Service) GetLastUploadRetentionScanForRepository(ctx context.Context, repositoryID int) (*time.Time, error) { return s.store.GetLastUploadRetentionScanForRepository(ctx, repositoryID) } func (s *Service) ReindexUploads(ctx context.Context, opts shared.ReindexUploadsOptions) error { return s.store.ReindexUploads(ctx, opts) } func (s *Service) ReindexUploadByID(ctx context.Context, id int) error { return s.store.ReindexUploadByID(ctx, id) } func (s *Service) GetIndexes(ctx context.Context, opts shared.GetIndexesOptions) ([]uploadsshared.AutoIndexJob, int, error) { return s.store.GetAutoIndexJobs(ctx, opts) } func (s *Service) GetAutoIndexJobByID(ctx context.Context, id int) (uploadsshared.AutoIndexJob, bool, error) { return s.store.GetAutoIndexJobByID(ctx, id) } func (s *Service) GetAutoIndexJobsByIDs(ctx context.Context, ids ...int) ([]uploadsshared.AutoIndexJob, error) { return s.store.GetAutoIndexJobsByIDs(ctx, ids...) } func (s *Service) DeleteAutoIndexJobByID(ctx context.Context, id int) (bool, error) { return s.store.DeleteAutoIndexJobByID(ctx, id) } func (s *Service) DeleteAutoIndexJobs(ctx context.Context, opts shared.DeleteAutoIndexJobsOptions) error { return s.store.DeleteAutoIndexJobs(ctx, opts) } func (s *Service) SetRerunAutoIndexJobByID(ctx context.Context, id int) error { return s.store.SetRerunAutoIndexJobByID(ctx, id) } func (s *Service) SetRerunAutoIndexJobs(ctx context.Context, opts shared.SetRerunAutoIndexJobsOptions) error { return s.store.SetRerunAutoIndexJobs(ctx, opts) } func (s *Service) GetRecentIndexesSummary(ctx context.Context, repositoryID int) ([]uploadsshared.IndexesWithRepositoryNamespace, error) { return s.store.GetRecentIndexesSummary(ctx, repositoryID) } func (s *Service) NumRepositoriesWithCodeIntelligence(ctx context.Context) (int, error) { return s.store.NumRepositoriesWithCodeIntelligence(ctx) } func (s *Service) RepositoryIDsWithErrors(ctx context.Context, offset, limit int) ([]uploadsshared.RepositoryWithCount, int, error) { return s.store.RepositoryIDsWithErrors(ctx, offset, limit) }