Packages: expose listing of package repo information in graphql (#47105)

cmd/frontend/graphqlbackend/package_repos.go

@@ -0,0 +1,198 @@
+package graphqlbackend
+
+import (
+	"context"
+
+	"github.com/graph-gophers/graphql-go"
+	"github.com/graph-gophers/graphql-go/relay"
+
+	"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/graphqlutil"
+	"github.com/sourcegraph/sourcegraph/internal/api"
+	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies"
+	"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
+	"github.com/sourcegraph/sourcegraph/internal/database"
+	"github.com/sourcegraph/sourcegraph/internal/gitserver"
+	"github.com/sourcegraph/sourcegraph/internal/observation"
+	"github.com/sourcegraph/sourcegraph/internal/syncx"
+	"github.com/sourcegraph/sourcegraph/internal/types"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+type PackageRepoReferenceConnectionArgs struct {
+	graphqlutil.ConnectionArgs
+	After  *int
+	Scheme *string
+	Name   *string
+}
+
+func (r *schemaResolver) PackageRepoReferences(ctx context.Context, args *PackageRepoReferenceConnectionArgs) (*packageRepoReferenceConnectionResolver, error) {
+	depsService := dependencies.NewService(observation.NewContext(r.logger), r.db)
+
+	var opts dependencies.ListDependencyReposOpts
+
+	if args.Scheme != nil {
+		opts.Scheme = *args.Scheme
+	}
+
+	if args.Name != nil {
+		opts.Name = reposource.PackageName(*args.Name)
+	}
+
+	if args.First != nil {
+		opts.Limit = int(*args.First)
+	}
+
+	if args.After != nil {
+		opts.After = *args.After
+	}
+
+	deps, total, err := depsService.ListPackageRepoRefs(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return &packageRepoReferenceConnectionResolver{r.db, deps, total}, err
+}
+
+type packageRepoReferenceConnectionResolver struct {
+	db    database.DB
+	deps  []dependencies.PackageRepoReference
+	total int
+}
+
+func (r *packageRepoReferenceConnectionResolver) Nodes(ctx context.Context) ([]*packageRepoReferenceResolver, error) {
+	once := syncx.OnceValues(func() (map[api.RepoName]*types.Repo, error) {
+		allNames := make([]string, 0, len(r.deps))
+		for _, dep := range r.deps {
+			name, err := dependencyRepoToRepoName(dep)
+			if err != nil || string(name) == "" {
+				continue
+			}
+			allNames = append(allNames, string(name))
+		}
+
+		repos, err := r.db.Repos().List(ctx, database.ReposListOptions{
+			Names: allNames,
+		})
+		if err != nil {
+			return nil, errors.Wrap(err, "error listing repos")
+		}
+
+		repoMappings := make(map[api.RepoName]*types.Repo, len(repos))
+		for _, repo := range repos {
+			repoMappings[repo.Name] = repo
+		}
+		return repoMappings, nil
+	})
+
+	resolvers := make([]*packageRepoReferenceResolver, 0, len(r.deps))
+	for _, dep := range r.deps {
+		resolvers = append(resolvers, &packageRepoReferenceResolver{r.db, dep, once})
+	}
+
+	return resolvers, nil
+}
+
+func (r *packageRepoReferenceConnectionResolver) TotalCount(ctx context.Context) (int32, error) {
+	return int32(r.total), nil
+}
+
+func (r *packageRepoReferenceConnectionResolver) PageInfo(ctx context.Context) (*graphqlutil.PageInfo, error) {
+	if len(r.deps) == 0 {
+		return graphqlutil.HasNextPage(false), nil
+	}
+
+	next := int32(r.deps[len(r.deps)-1].ID)
+	return graphqlutil.EncodeIntCursor(&next), nil
+}
+
+type packageRepoReferenceResolver struct {
+	db       database.DB
+	dep      dependencies.PackageRepoReference
+	allRepos func() (map[api.RepoName]*types.Repo, error)
+}
+
+func (r *packageRepoReferenceResolver) ID() graphql.ID {
+	return relay.MarshalID("PackageRepoReference", r.dep.ID)
+}
+
+func (r *packageRepoReferenceResolver) Scheme() string {
+	return r.dep.Scheme
+}
+
+func (r *packageRepoReferenceResolver) Name() string {
+	return string(r.dep.Name)
+}
+
+func (r *packageRepoReferenceResolver) Versions() []*packageRepoReferenceVersionResolver {
+	versions := make([]*packageRepoReferenceVersionResolver, 0, len(r.dep.Versions))
+	for _, version := range r.dep.Versions {
+		versions = append(versions, &packageRepoReferenceVersionResolver{version})
+	}
+	return versions
+}
+
+func (r *packageRepoReferenceResolver) Repository(ctx context.Context) (*RepositoryResolver, error) {
+	repoName, err := dependencyRepoToRepoName(r.dep)
+	if err != nil {
+		return nil, err
+	}
+
+	repos, err := r.allRepos()
+	if err != nil {
+		return nil, err
+	}
+
+	if repo, ok := repos[repoName]; ok {
+		return NewRepositoryResolver(r.db, gitserver.NewClient(), repo), nil
+	}
+
+	return nil, nil
+}
+
+type packageRepoReferenceVersionResolver struct {
+	version dependencies.PackageRepoRefVersion
+}
+
+func (r *packageRepoReferenceVersionResolver) ID() graphql.ID {
+	return relay.MarshalID("PackageRepoRefVersion", r.version.ID)
+}
+
+func (r *packageRepoReferenceVersionResolver) PackageRepoReferenceID() graphql.ID {
+	return relay.MarshalID("PackageRepoReference", r.version.PackageRefID)
+}
+
+func (r *packageRepoReferenceVersionResolver) Version() string {
+	return r.version.Version
+}
+
+func dependencyRepoToRepoName(dep dependencies.PackageRepoReference) (repoName api.RepoName, _ error) {
+	switch dep.Scheme {
+	case "python":
+		repoName = reposource.ParsePythonPackageFromName(dep.Name).RepoName()
+	case "scip-ruby":
+		repoName = reposource.ParseRubyPackageFromName(dep.Name).RepoName()
+	case "semanticdb":
+		pkg, err := reposource.ParseMavenPackageFromName(dep.Name)
+		if err != nil {
+			return "", err
+		}
+		repoName = pkg.RepoName()
+	case "npm":
+		pkg, err := reposource.ParseNpmPackageFromPackageSyntax(dep.Name)
+		if err != nil {
+			return "", err
+		}
+		repoName = pkg.RepoName()
+	case "rust-analyzer":
+		repoName = reposource.ParseRustPackageFromName(dep.Name).RepoName()
+	case "go":
+		pkg, err := reposource.ParseGoDependencyFromName(dep.Name)
+		if err != nil {
+			return "", err
+		}
+		repoName = pkg.RepoName()
+	}
+
+	return repoName, nil
+}

cmd/frontend/graphqlbackend/schema.graphql

@@ -1366,6 +1366,31 @@ type Query {
         descending: Boolean = false
     ): NewRepositoryConnection!
 
+    """
+    Query package repo references.
+    """
+    packageRepoReferences(
+        """
+        The exact scheme value to filter by.
+        TODO: should this be an enum?
+        """
+        scheme: String
+        """
+        If supplied, only package repo references that match the given
+        terms by their name will be returned.
+        TODO: fuzzy vs exact?
+        """
+        name: String
+        """
+        Returns the first n external services from the list.
+        """
+        first: Int
+        """
+        Opaque pagination cursor.
+        """
+        after: String
+    ): PackageRepoReferenceConnection!
+
     """
     Looks up a Phabricator repository by name.
     """
@@ -3421,6 +3446,80 @@ type KeyValuePair {
     value: String
 }
 
+"""
+List of package repo references.
+"""
+type PackageRepoReferenceConnection {
+    """
+    A list of package repo references.
+    """
+    nodes: [PackageRepoReference!]!
+
+    """
+    The total number of package repo references in the connection.
+    """
+    totalCount: Int!
+
+    """
+    Pagination information.
+    """
+    pageInfo: PageInfo!
+}
+
+"""
+A reference to a package repo, such as a maven artifact, rust crate etc.
+"""
+type PackageRepoReference {
+    """
+    A unique ID for the package repo reference.
+    """
+    id: ID!
+
+    """
+    The ecosystem/package-manager/indexer scheme under which this package repo
+    reference is uniquely identified e.g. npm/python/rust-analyzer
+    """
+    scheme: String!
+
+    """
+    The name of the package, in a format relevant to the specific ecosystem e.g.
+    maven artifact coordinates (com.sample:text), npm scoped packages (@monkeys/banana).
+    """
+    name: String!
+
+    """
+    The versions of this package known to the sourcegraph instance.
+    """
+    versions: [PackageRepoReferenceVersion!]!
+
+    """
+    The synthetic repository (aka the package repo) created to store the contents of the
+    synced versions of the package repo reference. This type is subject to change once
+    package repos and other non-git code hosts become first-class.
+    """
+    repository: Repository
+}
+
+"""
+A version of a package repo reference.
+"""
+type PackageRepoReferenceVersion {
+    """
+    A unique ID for the package repo reference version.
+    """
+    id: ID!
+
+    """
+    The package repo reference that this ID is for.
+    """
+    packageRepoReferenceID: ID!
+
+    """
+    The version string. Not guaranteed to be semver or any other format.
+    """
+    version: String!
+}
+
 """
 Information and status related to the commit graph of this repository calculated
 for use by code intelligence features.

cmd/gitserver/server/vcs_packages_syncer.go

@@ -60,8 +60,8 @@ type packagesDownloadSource interface {
 // dependenciesService captures the methods we use of the codeintel/dependencies.Service,
 // used to make testing easier.
 type dependenciesService interface {
-	ListDependencyRepos(context.Context, dependencies.ListDependencyReposOpts) ([]dependencies.Repo, error)
-	UpsertDependencyRepos(ctx context.Context, deps []dependencies.Repo) ([]dependencies.Repo, error)
+	ListPackageRepoRefs(context.Context, dependencies.ListDependencyReposOpts) ([]dependencies.PackageRepoReference, int, error)
+	InsertPackageRepoRefs(ctx context.Context, deps []dependencies.MinimalPackageRepoRef) ([]dependencies.PackageRepoReference, []dependencies.PackageRepoRefVersion, error)
 }
 
 func (s *vcsPackagesSyncer) IsCloneable(ctx context.Context, repoUrl *vcs.URL) error {
@@ -152,11 +152,11 @@ func (s *vcsPackagesSyncer) fetchRevspec(ctx context.Context, name reposource.Pa
 		return nil
 	}
 
-	_, err = s.svc.UpsertDependencyRepos(ctx, []dependencies.Repo{
+	_, _, err = s.svc.InsertPackageRepoRefs(ctx, []dependencies.MinimalPackageRepoRef{
 		{
-			Scheme:  dep.Scheme(),
-			Name:    dep.PackageSyntax(),
-			Version: dep.PackageVersion(),
+			Scheme:   dep.Scheme(),
+			Name:     dep.PackageSyntax(),
+			Versions: []string{dep.PackageVersion()},
 		},
 	})
 	if err != nil {
@@ -324,7 +324,7 @@ func (s *vcsPackagesSyncer) gitPushDependencyTag(ctx context.Context, bareGitDir
 }
 
 func (s *vcsPackagesSyncer) versions(ctx context.Context, packageName reposource.PackageName) ([]string, error) {
-	var versions []string
+	var combinedVersions []string
 	for _, d := range s.configDeps {
 		dep, err := s.source.ParseVersionedPackageFromConfiguration(d)
 		if err != nil {
@@ -333,24 +333,33 @@ func (s *vcsPackagesSyncer) versions(ctx context.Context, packageName reposource
 		}
 
 		if dep.PackageSyntax() == packageName {
-			versions = append(versions, dep.PackageVersion())
+			combinedVersions = append(combinedVersions, dep.PackageVersion())
 		}
 	}
 
-	depRepos, err := s.svc.ListDependencyRepos(ctx, dependencies.ListDependencyReposOpts{
-		Scheme:      s.scheme,
-		Name:        packageName,
-		NewestFirst: true,
+	listedPackages, _, err := s.svc.ListPackageRepoRefs(ctx, dependencies.ListDependencyReposOpts{
+		Scheme:              s.scheme,
+		Name:                packageName,
+		ExactNameOnly:       true,
+		MostRecentlyUpdated: true,
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to list dependencies from db")
 	}
 
-	for _, depRepo := range depRepos {
-		versions = append(versions, depRepo.Version)
+	if len(listedPackages) > 1 {
+		return nil, errors.Newf("unexpectedly got more than 1 dependency repo for (scheme=%q,name=%q)", s.scheme, packageName)
 	}
 
-	return versions, nil
+	if len(listedPackages) == 0 {
+		return combinedVersions, nil
+	}
+
+	for _, versions := range listedPackages[0].Versions {
+		combinedVersions = append(combinedVersions, versions.Version)
+	}
+
+	return combinedVersions, nil
 }
 
 func runCommandInDirectory(ctx context.Context, cmd *exec.Cmd, workingDirectory string, dependency reposource.VersionedPackage) (string, error) {

cmd/gitserver/server/vcs_packages_syncer_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/sourcegraph/log/logtest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/slices"
 
 	"github.com/sourcegraph/sourcegraph/internal/api"
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies"
@@ -31,7 +32,7 @@ func TestVcsDependenciesSyncer_Fetch(t *testing.T) {
 		download:      map[string]error{},
 		downloadCount: map[string]int{},
 	}
-	depsService := &fakeDepsService{deps: map[reposource.PackageName][]dependencies.Repo{}}
+	depsService := &fakeDepsService{deps: map[reposource.PackageName]dependencies.PackageRepoReference{}}
 
 	s := vcsPackagesSyncer{
 		logger:      logtest.Scoped(t),
@@ -159,11 +160,10 @@ func TestVcsDependenciesSyncer_Fetch(t *testing.T) {
 		// For context, see https://github.com/sourcegraph/sourcegraph/pull/38811
 		err := s.Fetch(ctx, remoteURL, dir, "v0.0.3^0")
 		require.ErrorContains(t, err, "401 unauthorized") // v0.0.1 is still erroring
-		require.Equal(t, s.svc.(*fakeDepsService).upsertedDeps, []dependencies.Repo{{
-			ID:      0,
-			Scheme:  fakeVersionedPackage{}.Scheme(),
-			Name:    "foo",
-			Version: "0.0.3",
+		require.Equal(t, s.svc.(*fakeDepsService).upsertedDeps, []dependencies.MinimalPackageRepoRef{{
+			Scheme:   fakeVersionedPackage{}.Scheme(),
+			Name:     "foo",
+			Versions: []string{"0.0.3"},
 		}})
 		s.assertRefs(t, dir, bothV2andV3Refs)
 		// We triggered a single download for v0.0.3 since it was lazily requested.
@@ -172,7 +172,7 @@ func TestVcsDependenciesSyncer_Fetch(t *testing.T) {
 	})
 
 	depsSource.download["foo@0.0.4"] = errors.New("0.0.4 not found")
-	s.svc.(*fakeDepsService).upsertedDeps = []dependencies.Repo{}
+	s.svc.(*fakeDepsService).upsertedDeps = []dependencies.MinimalPackageRepoRef{}
 
 	t.Run("lazy-sync error version via revspec", func(t *testing.T) {
 		// the v0.0.4 tag cannot be created on-demand because it returns a "0.0.4 not found" error
@@ -181,7 +181,7 @@ func TestVcsDependenciesSyncer_Fetch(t *testing.T) {
 		// // the 0.0.4 error is silently ignored, we only return the error for v0.0.1.
 		// require.Equal(t, fmt.Sprint(err.Error()), "error pushing dependency {\"foo\" \"0.0.1\"}: 401 unauthorized")
 		// the 0.0.4 dependency was not stored in the database because the download failed.
-		require.Equal(t, s.svc.(*fakeDepsService).upsertedDeps, []dependencies.Repo{})
+		require.Equal(t, s.svc.(*fakeDepsService).upsertedDeps, []dependencies.MinimalPackageRepoRef{})
 		// git tags are unchanged, v0.0.2 and v0.0.3 are cached.
 		s.assertRefs(t, dir, bothV2andV3Refs)
 		// We triggered downloads only for v0.0.4.
@@ -202,55 +202,85 @@ func TestVcsDependenciesSyncer_Fetch(t *testing.T) {
 }
 
 type fakeDepsService struct {
-	deps         map[reposource.PackageName][]dependencies.Repo
-	upsertedDeps []dependencies.Repo
+	deps         map[reposource.PackageName]dependencies.PackageRepoReference
+	upsertedDeps []dependencies.MinimalPackageRepoRef
 }
 
-func (s *fakeDepsService) UpsertDependencyRepos(ctx context.Context, deps []dependencies.Repo) ([]dependencies.Repo, error) {
-	s.upsertedDeps = append(s.upsertedDeps, deps...)
-	for _, dep := range deps {
-		alreadyExists := false
-		for _, existingDep := range s.deps[dep.Name] {
-			if existingDep.Version == dep.Version {
-				alreadyExists = true
-				break
+func (s *fakeDepsService) InsertPackageRepoRefs(ctx context.Context, depsToAdd []dependencies.MinimalPackageRepoRef) (newRepos []dependencies.PackageRepoReference, newVersions []dependencies.PackageRepoRefVersion, _ error) {
+	s.upsertedDeps = append(s.upsertedDeps, depsToAdd...)
+	for _, depToAdd := range depsToAdd {
+		if existingDep, exists := s.deps[depToAdd.Name]; exists {
+			for _, version := range depToAdd.Versions {
+				if !slices.ContainsFunc(existingDep.Versions, func(v dependencies.PackageRepoRefVersion) bool {
+					return v.Version == version
+				}) {
+					existingDep.Versions = append(existingDep.Versions, dependencies.PackageRepoRefVersion{
+						PackageRefID: existingDep.ID,
+						Version:      version,
+					})
+					s.deps[depToAdd.Name] = existingDep
+					newVersions = append(newVersions, dependencies.PackageRepoRefVersion{
+						Version: version,
+					})
+				}
 			}
-		}
-		if !alreadyExists {
-			s.deps[dep.Name] = append(s.deps[dep.Name], dep)
+		} else {
+			versionsForDep := make([]dependencies.PackageRepoRefVersion, 0, len(depToAdd.Versions))
+			for _, version := range depToAdd.Versions {
+				versionsForDep = append(versionsForDep, dependencies.PackageRepoRefVersion{
+					Version: version,
+				})
+			}
+			s.deps[depToAdd.Name] = dependencies.PackageRepoReference{
+				Scheme:   depToAdd.Scheme,
+				Name:     depToAdd.Name,
+				Versions: versionsForDep,
+			}
+			newRepos = append(newRepos, dependencies.PackageRepoReference{
+				Scheme:   depToAdd.Scheme,
+				Name:     depToAdd.Name,
+				Versions: versionsForDep,
+			})
 		}
 	}
-	return deps, nil
+	return
 }
 
-func (s *fakeDepsService) ListDependencyRepos(ctx context.Context, opts dependencies.ListDependencyReposOpts) ([]dependencies.Repo, error) {
-	return s.deps[opts.Name], nil
+func (s *fakeDepsService) ListPackageRepoRefs(ctx context.Context, opts dependencies.ListDependencyReposOpts) ([]dependencies.PackageRepoReference, int, error) {
+	return []dependencies.PackageRepoReference{s.deps[opts.Name]}, 1, nil
 }
 
 func (s *fakeDepsService) Add(deps ...string) {
 	for _, d := range deps {
 		dep, _ := parseFakeDependency(d)
 		name := dep.PackageSyntax()
-		s.deps[name] = append(s.deps[name], dependencies.Repo{
-			Scheme:  dep.Scheme(),
-			Name:    name,
-			Version: dep.PackageVersion(),
-		})
+		if d, ok := s.deps[name]; !ok {
+			s.deps[name] = dependencies.PackageRepoReference{
+				Scheme: dep.Scheme(),
+				Name:   name,
+				Versions: []dependencies.PackageRepoRefVersion{
+					{Version: dep.PackageVersion()},
+				},
+			}
+		} else {
+			d.Versions = append(d.Versions, dependencies.PackageRepoRefVersion{Version: dep.PackageVersion()})
+			s.deps[name] = d
+		}
 	}
 }
 
 func (s *fakeDepsService) Delete(deps ...string) {
 	for _, d := range deps {
-		dep, _ := parseFakeDependency(d)
-		name := dep.PackageSyntax()
-		version := dep.PackageVersion()
-		filtered := s.deps[name][:0]
-		for _, r := range s.deps[name] {
-			if r.Version != version {
-				filtered = append(filtered, r)
-			}
+		depToDelete, _ := parseFakeDependency(d)
+		name := depToDelete.PackageSyntax()
+		version := depToDelete.PackageVersion()
+		dep := s.deps[name]
+		if idx := slices.IndexFunc(dep.Versions, func(v dependencies.PackageRepoRefVersion) bool {
+			return v.Version == version
+		}); idx > -1 {
+			dep.Versions = slices.Delete(dep.Versions, idx, idx+1)
+			s.deps[name] = dep
 		}
-		s.deps[name] = filtered
 	}
 }
 
@@ -280,12 +310,13 @@ func (s *fakeDepsSource) Download(ctx context.Context, dir string, dep reposourc
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(filepath.Join(dir, "README.md"), []byte("README for "+dep.VersionedPackageSyntax()), 0666)
+	return os.WriteFile(filepath.Join(dir, "README.md"), []byte("README for "+dep.VersionedPackageSyntax()), 0o666)
 }
 
 func (fakeDepsSource) ParseVersionedPackageFromNameAndVersion(name reposource.PackageName, version string) (reposource.VersionedPackage, error) {
 	return parseFakeDependency(string(name) + "@" + version)
 }
+
 func (fakeDepsSource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
 	return parseFakeDependency(dep)
 }

cmd/gitserver/server/vcs_syncer_npm_packages_test.go

@@ -148,27 +148,23 @@ func TestNpmCloneCommand(t *testing.T) {
 	checkTagAdded()
 
 	s.runCloneCommand(t, exampleNpmPackageURL, bareGitDirectory, []string{exampleNpmVersionedPackage})
-	checkTagRemoved := func() {
-		assertCommandOutput(t,
-			exec.Command("git", "show", fmt.Sprintf("v%s:%s", exampleNpmVersion, exampleJSFilepath)),
-			bareGitDirectory,
-			exampleJSFileContents,
-		)
-		assertCommandOutput(t,
-			exec.Command("git", "tag", "--list"),
-			bareGitDirectory,
-			fmt.Sprintf("v%s\n", exampleNpmVersion), // verify that second tag has been removed.
-		)
-	}
-	checkTagRemoved()
+	assertCommandOutput(t,
+		exec.Command("git", "show", fmt.Sprintf("v%s:%s", exampleNpmVersion, exampleJSFilepath)),
+		bareGitDirectory,
+		exampleJSFileContents,
+	)
+	assertCommandOutput(t,
+		exec.Command("git", "tag", "--list"),
+		bareGitDirectory,
+		fmt.Sprintf("v%s\n", exampleNpmVersion), // verify that second tag has been removed.
+	)
 
 	// Now run the same tests with the database output instead.
-	if _, err := depsSvc.UpsertDependencyRepos(context.Background(), []dependencies.Repo{
+	if _, _, err := depsSvc.InsertPackageRepoRefs(context.Background(), []dependencies.MinimalPackageRepoRef{
 		{
-			ID:      1,
-			Scheme:  dependencies.NpmPackagesScheme,
-			Name:    "example",
-			Version: exampleNpmVersion,
+			Scheme:   dependencies.NpmPackagesScheme,
+			Name:     "example",
+			Versions: []string{exampleNpmVersion},
 		},
 	}); err != nil {
 		t.Fatalf(err.Error())
@@ -176,12 +172,11 @@ func TestNpmCloneCommand(t *testing.T) {
 	s.runCloneCommand(t, exampleNpmPackageURL, bareGitDirectory, []string{})
 	checkSingleTag()
 
-	if _, err := depsSvc.UpsertDependencyRepos(context.Background(), []dependencies.Repo{
+	if _, _, err := depsSvc.InsertPackageRepoRefs(context.Background(), []dependencies.MinimalPackageRepoRef{
 		{
-			ID:      2,
-			Scheme:  dependencies.NpmPackagesScheme,
-			Name:    "example",
-			Version: exampleNpmVersion2,
+			Scheme:   dependencies.NpmPackagesScheme,
+			Name:     "example",
+			Versions: []string{exampleNpmVersion2},
 		},
 	}); err != nil {
 		t.Fatalf(err.Error())
@@ -189,11 +184,20 @@ func TestNpmCloneCommand(t *testing.T) {
 	s.runCloneCommand(t, exampleNpmPackageURL, bareGitDirectory, []string{})
 	checkTagAdded()
 
-	if err := depsSvc.DeleteDependencyReposByID(context.Background(), 2); err != nil {
+	if err := depsSvc.DeletePackageRepoRefVersionsByID(context.Background(), 2); err != nil {
 		t.Fatalf(err.Error())
 	}
 	s.runCloneCommand(t, exampleNpmPackageURL, bareGitDirectory, []string{})
-	checkTagRemoved()
+	assertCommandOutput(t,
+		exec.Command("git", "show", fmt.Sprintf("v%s:%s", exampleNpmVersion, exampleJSFilepath)),
+		bareGitDirectory,
+		exampleJSFileContents,
+	)
+	assertCommandOutput(t,
+		exec.Command("git", "tag", "--list"),
+		bareGitDirectory,
+		fmt.Sprintf("v%s\n", exampleNpmVersion), // verify that second tag has been removed.
+	)
 }
 
 func createTgz(t *testing.T, fileInfos []fileInfo) []byte {

cmd/gitserver/server/vcs_syncer_python_packages.go

@@ -3,7 +3,6 @@ package server
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"io"
 	"io/fs"
 	"net/url"
@@ -21,27 +20,16 @@ import (
 	"github.com/sourcegraph/sourcegraph/schema"
 )
 
-func assertPythonParsesPlaceholder() *reposource.PythonVersionedPackage {
-	placeholder, err := reposource.ParseVersionedPackage("sourcegraph.com/placeholder@v0.0.0")
-	if err != nil {
-		panic(fmt.Sprintf("expected placeholder dependency to parse but got %v", err))
-	}
-
-	return placeholder
-}
-
 func NewPythonPackagesSyncer(
 	connection *schema.PythonPackagesConnection,
 	svc *dependencies.Service,
 	client *pypi.Client,
 ) VCSSyncer {
-	placeholder := assertPythonParsesPlaceholder()
-
 	return &vcsPackagesSyncer{
 		logger:      log.Scoped("PythonPackagesSyncer", "sync Python packages"),
 		typ:         "python_packages",
 		scheme:      dependencies.PythonPackagesScheme,
-		placeholder: placeholder,
+		placeholder: reposource.ParseVersionedPackage("sourcegraph.com/placeholder@v0.0.0"),
 		svc:         svc,
 		configDeps:  connection.Dependencies,
 		source:      &pythonPackagesSyncer{client: client},
@@ -54,15 +42,15 @@ type pythonPackagesSyncer struct {
 }
 
 func (pythonPackagesSyncer) ParseVersionedPackageFromNameAndVersion(name reposource.PackageName, version string) (reposource.VersionedPackage, error) {
-	return reposource.ParseVersionedPackage(string(name) + "==" + version)
+	return reposource.ParseVersionedPackage(string(name) + "==" + version), nil
 }
 
 func (pythonPackagesSyncer) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
-	return reposource.ParseVersionedPackage(dep)
+	return reposource.ParseVersionedPackage(dep), nil
 }
 
 func (pythonPackagesSyncer) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
-	return reposource.ParsePythonPackageFromName(name)
+	return reposource.ParsePythonPackageFromName(name), nil
 }
 
 func (pythonPackagesSyncer) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {

cmd/gitserver/server/vcs_syncer_ruby_packages.go

@@ -25,7 +25,6 @@ func NewRubyPackagesSyncer(
 	svc *dependencies.Service,
 	client *rubygems.Client,
 ) VCSSyncer {
-
 	return &vcsPackagesSyncer{
 		logger:      log.Scoped("RubyPackagesSyncer", "sync Ruby packages"),
 		typ:         "ruby_packages",
@@ -42,17 +41,17 @@ type rubyDependencySource struct {
 }
 
 func (rubyDependencySource) ParseVersionedPackageFromNameAndVersion(name reposource.PackageName, version string) (reposource.VersionedPackage, error) {
-	return reposource.ParseRubyVersionedPackage(string(name) + "@" + version)
+	return reposource.ParseRubyVersionedPackage(string(name) + "@" + version), nil
 }
 
 func (rubyDependencySource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
-	return reposource.ParseRubyVersionedPackage(dep)
+	return reposource.ParseRubyVersionedPackage(dep), nil
 }
 
 func (rubyDependencySource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
-	return reposource.ParseRubyPackageFromName(name)
-
+	return reposource.ParseRubyPackageFromName(name), nil
 }
+
 func (rubyDependencySource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {
 	return reposource.ParseRubyPackageFromRepoName(repoName)
 }
@@ -106,7 +105,7 @@ func unpackRubyPackage(packageURL string, pkg io.Reader, workDir string) error {
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(filepath.Join(workDir, "rubygems-metadata.yml"), metadataBytes, 0644)
+	return os.WriteFile(filepath.Join(workDir, "rubygems-metadata.yml"), metadataBytes, 0o644)
 }
 
 // unpackRubyDataTarGz unpacks the given `data.tar.gz` from a downloaded RubyGem.

cmd/gitserver/server/vcs_syncer_rust_packages.go

@@ -17,27 +17,16 @@ import (
 	"github.com/sourcegraph/sourcegraph/schema"
 )
 
-func assertRustParsesPlaceholder() *reposource.RustVersionedPackage {
-	placeholder, err := reposource.ParseRustVersionedPackage("sourcegraph.com/placeholder@0.0.0")
-	if err != nil {
-		panic(fmt.Sprintf("expected placeholder dependency to parse but got %v", err))
-	}
-
-	return placeholder
-}
-
 func NewRustPackagesSyncer(
 	connection *schema.RustPackagesConnection,
 	svc *dependencies.Service,
 	client *crates.Client,
 ) VCSSyncer {
-	placeholder := assertRustParsesPlaceholder()
-
 	return &vcsPackagesSyncer{
 		logger:      log.Scoped("RustPackagesSyncer", "sync Rust packages"),
 		typ:         "rust_packages",
 		scheme:      dependencies.RustPackagesScheme,
-		placeholder: placeholder,
+		placeholder: reposource.ParseRustVersionedPackage("sourcegraph.com/placeholder@0.0.0"),
 		svc:         svc,
 		configDeps:  connection.Dependencies,
 		source:      &rustDependencySource{client: client},
@@ -49,15 +38,15 @@ type rustDependencySource struct {
 }
 
 func (rustDependencySource) ParseVersionedPackageFromNameAndVersion(name reposource.PackageName, version string) (reposource.VersionedPackage, error) {
-	return reposource.ParseRustVersionedPackage(string(name) + "@" + version)
+	return reposource.ParseRustVersionedPackage(string(name) + "@" + version), nil
 }
 
 func (rustDependencySource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
-	return reposource.ParseRustVersionedPackage(dep)
+	return reposource.ParseRustVersionedPackage(dep), nil
 }
 
 func (rustDependencySource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
-	return reposource.ParseRustPackageFromName(name)
+	return reposource.ParseRustPackageFromName(name), nil
 }
 
 func (rustDependencySource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {

enterprise/internal/codeintel/autoindexing/internal/background/iface.go

@@ -20,7 +20,7 @@ import (
 )
 
 type DependenciesService interface {
-	UpsertDependencyRepos(ctx context.Context, deps []dependencies.Repo) ([]dependencies.Repo, error)
+	InsertPackageRepoRefs(ctx context.Context, deps []dependencies.MinimalPackageRepoRef) ([]dependencies.PackageRepoReference, []dependencies.PackageRepoRefVersion, error)
 }
 
 type GitserverRepoStore interface {

enterprise/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go

@@ -91,6 +91,8 @@ func (h *dependencySyncSchedulerHandler) Handle(ctx context.Context, logger log.
 		kinds                      = map[string]struct{}{}
 		oldDependencyReposInserted int
 		newDependencyReposInserted int
+		newVersionsInserted        int
+		oldVersionsInserted        int
 		errs                       []error
 	)
 
@@ -125,13 +127,26 @@ func (h *dependencySyncSchedulerHandler) Handle(ctx context.Context, logger log.
 			continue
 		}
 
-		new, err := h.insertDependencyRepo(ctx, pkg)
+		newRepo, newVersion, err := h.insertPackageRepoRef(ctx, pkg)
 		if err != nil {
 			errs = append(errs, err)
-		} else if new {
+			continue
+		}
+
+		if newRepo {
 			newDependencyReposInserted++
+			if newVersion {
+				newVersionsInserted++
+			} else {
+				oldVersionsInserted++
+			}
 		} else {
 			oldDependencyReposInserted++
+			if newVersion {
+				newVersionsInserted++
+			} else {
+				oldVersionsInserted++
+			}
 		}
 	}
 
@@ -156,7 +171,10 @@ func (h *dependencySyncSchedulerHandler) Handle(ctx context.Context, logger log.
 			log.Int("numExtSvc", len(externalServices)),
 			log.Strings("schemaKinds", kindsArray),
 			log.Int("newRepos", newDependencyReposInserted),
-			log.Int("existingInserts", oldDependencyReposInserted))
+			log.Int("existingRepos", oldDependencyReposInserted),
+			log.Int("newVersions", newVersionsInserted),
+			log.Int("existingVersions", oldVersionsInserted),
+		)
 
 		for _, externalService := range externalServices {
 			externalService.NextSyncAt = nextSync
@@ -222,18 +240,18 @@ func newPackage(pkg uploadsshared.Package) (*precise.Package, error) {
 	return &p, nil
 }
 
-func (h *dependencySyncSchedulerHandler) insertDependencyRepo(ctx context.Context, pkg precise.Package) (new bool, err error) {
-	inserted, err := h.depsSvc.UpsertDependencyRepos(ctx, []dependencies.Repo{
+func (h *dependencySyncSchedulerHandler) insertPackageRepoRef(ctx context.Context, pkg precise.Package) (newRepos, newVersions bool, err error) {
+	insertedRepos, insertedVersions, err := h.depsSvc.InsertPackageRepoRefs(ctx, []dependencies.MinimalPackageRepoRef{
 		{
-			Name:    reposource.PackageName(pkg.Name),
-			Scheme:  pkg.Scheme,
-			Version: pkg.Version,
+			Name:     reposource.PackageName(pkg.Name),
+			Scheme:   pkg.Scheme,
+			Versions: []string{pkg.Version},
 		},
 	})
 	if err != nil {
-		return false, errors.Wrap(err, "dbstore.InsertCloneableDependencyRepos")
+		return false, false, errors.Wrap(err, "dbstore.InsertCloneableDependencyRepos")
 	}
-	return len(inserted) != 0, nil
+	return len(insertedRepos) != 0, len(insertedVersions) != 0, nil
 }
 
 // shouldIndexDependencies returns true if the given upload should undergo dependency

enterprise/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler_test.go

@@ -65,8 +65,8 @@ func TestDependencySyncSchedulerJVM(t *testing.T) {
 		t.Errorf("unexpected number of calls to extsvc.List. want=%d have=%d", 1, len(mockExtsvcStore.ListFunc.History()))
 	}
 
-	if len(mockDepedenciesSvc.UpsertDependencyReposFunc.History()) != 1 {
-		t.Errorf("unexpected number of calls to InsertCloneableDependencyRepo. want=%d have=%d", 1, len(mockDepedenciesSvc.UpsertDependencyReposFunc.History()))
+	if len(mockDepedenciesSvc.InsertPackageRepoRefsFunc.History()) != 1 {
+		t.Errorf("unexpected number of calls to InsertCloneableDependencyRepo. want=%d have=%d", 1, len(mockDepedenciesSvc.InsertPackageRepoRefsFunc.History()))
 	}
 }
 
@@ -117,8 +117,8 @@ func TestDependencySyncSchedulerGomod(t *testing.T) {
 		t.Errorf("unexpected number of calls to extsvc.List. want=%d have=%d", 0, len(mockExtsvcStore.ListFunc.History()))
 	}
 
-	if len(mockDepedenciesSvc.UpsertDependencyReposFunc.History()) != 0 {
-		t.Errorf("unexpected number of calls to InsertCloneableDependencyRepo. want=%d have=%d", 0, len(mockDepedenciesSvc.UpsertDependencyReposFunc.History()))
+	if len(mockDepedenciesSvc.InsertPackageRepoRefsFunc.History()) != 0 {
+		t.Errorf("unexpected number of calls to InsertCloneableDependencyRepo. want=%d have=%d", 0, len(mockDepedenciesSvc.InsertPackageRepoRefsFunc.History()))
 	}
 }
 

enterprise/internal/codeintel/autoindexing/internal/background/mocks_test.go

@@ -36,9 +36,9 @@ import (
 // github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/autoindexing/internal/background)
 // used for unit testing.
 type MockDependenciesService struct {
-	// UpsertDependencyReposFunc is an instance of a mock function object
-	// controlling the behavior of the method UpsertDependencyRepos.
-	UpsertDependencyReposFunc *DependenciesServiceUpsertDependencyReposFunc
+	// InsertPackageRepoRefsFunc is an instance of a mock function object
+	// controlling the behavior of the method InsertPackageRepoRefs.
+	InsertPackageRepoRefsFunc *DependenciesServiceInsertPackageRepoRefsFunc
 }
 
 // NewMockDependenciesService creates a new mock of the DependenciesService
@@ -46,8 +46,8 @@ type MockDependenciesService struct {
 // overwritten.
 func NewMockDependenciesService() *MockDependenciesService {
 	return &MockDependenciesService{
-		UpsertDependencyReposFunc: &DependenciesServiceUpsertDependencyReposFunc{
-			defaultHook: func(context.Context, []shared.Repo) (r0 []shared.Repo, r1 error) {
+		InsertPackageRepoRefsFunc: &DependenciesServiceInsertPackageRepoRefsFunc{
+			defaultHook: func(context.Context, []shared.MinimalPackageRepoRef) (r0 []shared.PackageRepoReference, r1 []shared.PackageRepoRefVersion, r2 error) {
 				return
 			},
 		},
@@ -59,9 +59,9 @@ func NewMockDependenciesService() *MockDependenciesService {
 // overwritten.
 func NewStrictMockDependenciesService() *MockDependenciesService {
 	return &MockDependenciesService{
-		UpsertDependencyReposFunc: &DependenciesServiceUpsertDependencyReposFunc{
-			defaultHook: func(context.Context, []shared.Repo) ([]shared.Repo, error) {
-				panic("unexpected invocation of MockDependenciesService.UpsertDependencyRepos")
+		InsertPackageRepoRefsFunc: &DependenciesServiceInsertPackageRepoRefsFunc{
+			defaultHook: func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error) {
+				panic("unexpected invocation of MockDependenciesService.InsertPackageRepoRefs")
 			},
 		},
 	}
@@ -72,43 +72,43 @@ func NewStrictMockDependenciesService() *MockDependenciesService {
 // implementation, unless overwritten.
 func NewMockDependenciesServiceFrom(i DependenciesService) *MockDependenciesService {
 	return &MockDependenciesService{
-		UpsertDependencyReposFunc: &DependenciesServiceUpsertDependencyReposFunc{
-			defaultHook: i.UpsertDependencyRepos,
+		InsertPackageRepoRefsFunc: &DependenciesServiceInsertPackageRepoRefsFunc{
+			defaultHook: i.InsertPackageRepoRefs,
 		},
 	}
 }
 
-// DependenciesServiceUpsertDependencyReposFunc describes the behavior when
-// the UpsertDependencyRepos method of the parent MockDependenciesService
+// DependenciesServiceInsertPackageRepoRefsFunc describes the behavior when
+// the InsertPackageRepoRefs method of the parent MockDependenciesService
 // instance is invoked.
-type DependenciesServiceUpsertDependencyReposFunc struct {
-	defaultHook func(context.Context, []shared.Repo) ([]shared.Repo, error)
-	hooks       []func(context.Context, []shared.Repo) ([]shared.Repo, error)
-	history     []DependenciesServiceUpsertDependencyReposFuncCall
+type DependenciesServiceInsertPackageRepoRefsFunc struct {
+	defaultHook func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error)
+	hooks       []func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error)
+	history     []DependenciesServiceInsertPackageRepoRefsFuncCall
 	mutex       sync.Mutex
 }
 
-// UpsertDependencyRepos delegates to the next hook function in the queue
+// InsertPackageRepoRefs delegates to the next hook function in the queue
 // and stores the parameter and result values of this invocation.
-func (m *MockDependenciesService) UpsertDependencyRepos(v0 context.Context, v1 []shared.Repo) ([]shared.Repo, error) {
-	r0, r1 := m.UpsertDependencyReposFunc.nextHook()(v0, v1)
-	m.UpsertDependencyReposFunc.appendCall(DependenciesServiceUpsertDependencyReposFuncCall{v0, v1, r0, r1})
-	return r0, r1
+func (m *MockDependenciesService) InsertPackageRepoRefs(v0 context.Context, v1 []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error) {
+	r0, r1, r2 := m.InsertPackageRepoRefsFunc.nextHook()(v0, v1)
+	m.InsertPackageRepoRefsFunc.appendCall(DependenciesServiceInsertPackageRepoRefsFuncCall{v0, v1, r0, r1, r2})
+	return r0, r1, r2
 }
 
 // SetDefaultHook sets function that is called when the
-// UpsertDependencyRepos method of the parent MockDependenciesService
+// InsertPackageRepoRefs method of the parent MockDependenciesService
 // instance is invoked and the hook queue is empty.
-func (f *DependenciesServiceUpsertDependencyReposFunc) SetDefaultHook(hook func(context.Context, []shared.Repo) ([]shared.Repo, error)) {
+func (f *DependenciesServiceInsertPackageRepoRefsFunc) SetDefaultHook(hook func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error)) {
 	f.defaultHook = hook
 }
 
 // PushHook adds a function to the end of hook queue. Each invocation of the
-// UpsertDependencyRepos method of the parent MockDependenciesService
+// InsertPackageRepoRefs method of the parent MockDependenciesService
 // instance invokes the hook at the front of the queue and discards it.
 // After the queue is empty, the default hook function is invoked for any
 // future action.
-func (f *DependenciesServiceUpsertDependencyReposFunc) PushHook(hook func(context.Context, []shared.Repo) ([]shared.Repo, error)) {
+func (f *DependenciesServiceInsertPackageRepoRefsFunc) PushHook(hook func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error)) {
 	f.mutex.Lock()
 	f.hooks = append(f.hooks, hook)
 	f.mutex.Unlock()
@@ -116,20 +116,20 @@ func (f *DependenciesServiceUpsertDependencyReposFunc) PushHook(hook func(contex
 
 // SetDefaultReturn calls SetDefaultHook with a function that returns the
 // given values.
-func (f *DependenciesServiceUpsertDependencyReposFunc) SetDefaultReturn(r0 []shared.Repo, r1 error) {
-	f.SetDefaultHook(func(context.Context, []shared.Repo) ([]shared.Repo, error) {
-		return r0, r1
+func (f *DependenciesServiceInsertPackageRepoRefsFunc) SetDefaultReturn(r0 []shared.PackageRepoReference, r1 []shared.PackageRepoRefVersion, r2 error) {
+	f.SetDefaultHook(func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error) {
+		return r0, r1, r2
 	})
 }
 
 // PushReturn calls PushHook with a function that returns the given values.
-func (f *DependenciesServiceUpsertDependencyReposFunc) PushReturn(r0 []shared.Repo, r1 error) {
-	f.PushHook(func(context.Context, []shared.Repo) ([]shared.Repo, error) {
-		return r0, r1
+func (f *DependenciesServiceInsertPackageRepoRefsFunc) PushReturn(r0 []shared.PackageRepoReference, r1 []shared.PackageRepoRefVersion, r2 error) {
+	f.PushHook(func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error) {
+		return r0, r1, r2
 	})
 }
 
-func (f *DependenciesServiceUpsertDependencyReposFunc) nextHook() func(context.Context, []shared.Repo) ([]shared.Repo, error) {
+func (f *DependenciesServiceInsertPackageRepoRefsFunc) nextHook() func(context.Context, []shared.MinimalPackageRepoRef) ([]shared.PackageRepoReference, []shared.PackageRepoRefVersion, error) {
 	f.mutex.Lock()
 	defer f.mutex.Unlock()
 
@@ -142,52 +142,55 @@ func (f *DependenciesServiceUpsertDependencyReposFunc) nextHook() func(context.C
 	return hook
 }
 
-func (f *DependenciesServiceUpsertDependencyReposFunc) appendCall(r0 DependenciesServiceUpsertDependencyReposFuncCall) {
+func (f *DependenciesServiceInsertPackageRepoRefsFunc) appendCall(r0 DependenciesServiceInsertPackageRepoRefsFuncCall) {
 	f.mutex.Lock()
 	f.history = append(f.history, r0)
 	f.mutex.Unlock()
 }
 
 // History returns a sequence of
-// DependenciesServiceUpsertDependencyReposFuncCall objects describing the
+// DependenciesServiceInsertPackageRepoRefsFuncCall objects describing the
 // invocations of this function.
-func (f *DependenciesServiceUpsertDependencyReposFunc) History() []DependenciesServiceUpsertDependencyReposFuncCall {
+func (f *DependenciesServiceInsertPackageRepoRefsFunc) History() []DependenciesServiceInsertPackageRepoRefsFuncCall {
 	f.mutex.Lock()
-	history := make([]DependenciesServiceUpsertDependencyReposFuncCall, len(f.history))
+	history := make([]DependenciesServiceInsertPackageRepoRefsFuncCall, len(f.history))
 	copy(history, f.history)
 	f.mutex.Unlock()
 
 	return history
 }
 
-// DependenciesServiceUpsertDependencyReposFuncCall is an object that
-// describes an invocation of method UpsertDependencyRepos on an instance of
+// DependenciesServiceInsertPackageRepoRefsFuncCall is an object that
+// describes an invocation of method InsertPackageRepoRefs on an instance of
 // MockDependenciesService.
-type DependenciesServiceUpsertDependencyReposFuncCall struct {
+type DependenciesServiceInsertPackageRepoRefsFuncCall struct {
 	// Arg0 is the value of the 1st argument passed to this method
 	// invocation.
 	Arg0 context.Context
 	// Arg1 is the value of the 2nd argument passed to this method
 	// invocation.
-	Arg1 []shared.Repo
+	Arg1 []shared.MinimalPackageRepoRef
 	// Result0 is the value of the 1st result returned from this method
 	// invocation.
-	Result0 []shared.Repo
+	Result0 []shared.PackageRepoReference
 	// Result1 is the value of the 2nd result returned from this method
 	// invocation.
-	Result1 error
+	Result1 []shared.PackageRepoRefVersion
+	// Result2 is the value of the 3rd result returned from this method
+	// invocation.
+	Result2 error
 }
 
 // Args returns an interface slice containing the arguments of this
 // invocation.
-func (c DependenciesServiceUpsertDependencyReposFuncCall) Args() []interface{} {
+func (c DependenciesServiceInsertPackageRepoRefsFuncCall) Args() []interface{} {
 	return []interface{}{c.Arg0, c.Arg1}
 }
 
 // Results returns an interface slice containing the results of this
 // invocation.
-func (c DependenciesServiceUpsertDependencyReposFuncCall) Results() []interface{} {
-	return []interface{}{c.Result0, c.Result1}
+func (c DependenciesServiceInsertPackageRepoRefsFuncCall) Results() []interface{} {
+	return []interface{}{c.Result0, c.Result1, c.Result2}
 }
 
 // MockExternalServiceStore is a mock implementation of the

enterprise/internal/codeintel/autoindexing/internal/inference/infer.go

@@ -77,13 +77,7 @@ func inferRustRepositoryAndRevision(pkg precise.Package) (api.RepoName, string,
 		return "", "", false
 	}
 
-	logger := log.Scoped("inferRustRepositoryAndRevision", "")
-	rustPkg, err := reposource.ParseRustVersionedPackage(pkg.Name)
-	if err != nil {
-		logger.Error("invalid rust package name in database", log.Error(err), log.String("pkg", pkg.Name))
-		return "", "", false
-	}
-
+	rustPkg := reposource.ParseRustVersionedPackage(pkg.Name)
 	return rustPkg.RepoName(), "v" + pkg.Version, true
 }
 
@@ -92,12 +86,7 @@ func inferPythonRepositoryAndRevision(pkg precise.Package) (api.RepoName, string
 		return "", "", false
 	}
 
-	logger := log.Scoped("inferPythonRepositoryAndRevision", "")
-	pythonPkg, err := reposource.ParsePythonPackageFromName(reposource.PackageName(pkg.Name))
-	if err != nil {
-		logger.Error("invalid python package name in database", log.Error(err), log.String("pkg", pkg.Name))
-		return "", "", false
-	}
+	pythonPkg := reposource.ParsePythonPackageFromName(reposource.PackageName(pkg.Name))
 
 	return pythonPkg.RepoName(), pkg.Version, true
 }
@@ -107,12 +96,7 @@ func inferRubyRepositoryAndRevision(pkg precise.Package) (api.RepoName, string,
 		return "", "", false
 	}
 
-	logger := log.Scoped("inferRubyRepositoryAndRevision", "")
-	rubyPkg, err := reposource.ParseRubyPackageFromName(reposource.PackageName(pkg.Name))
-	if err != nil {
-		logger.Error("invalid ruby package name in database", log.Error(err), log.String("pkg", pkg.Name))
-		return "", "", false
-	}
+	rubyPkg := reposource.ParseRubyPackageFromName(reposource.PackageName(pkg.Name))
 
 	return rubyPkg.RepoName(), pkg.Version, true
 }

internal/codeintel/dependencies/internal/background/iface.go

@@ -25,5 +25,5 @@ type ExternalServiceStore interface {
 }
 
 type DependenciesService interface {
-	UpsertDependencyRepos(ctx context.Context, deps []shared.Repo) (_ []shared.Repo, err error)
+	InsertPackageRepoRefs(ctx context.Context, deps []shared.MinimalPackageRepoRef) (_ []shared.PackageRepoReference, _ []shared.PackageRepoRefVersion, err error)
 }

internal/codeintel/dependencies/internal/background/job_cratesyncer.go

@@ -160,11 +160,11 @@ func (b *crateSyncerJob) handleCrateSyncer(ctx context.Context, interval time.Du
 			return err
 		}
 
-		new, err := b.dependenciesSvc.UpsertDependencyRepos(ctx, pkgs)
+		newCrates, newVersions, err := b.dependenciesSvc.InsertPackageRepoRefs(ctx, pkgs)
 		if err != nil {
 			return errors.Wrapf(err, "failed to insert Rust crate")
 		}
-		didInsertNewCrates = didInsertNewCrates || len(new) != 0
+		didInsertNewCrates = didInsertNewCrates || len(newCrates) != 0 || len(newVersions) != 0
 	}
 
 	if didInsertNewCrates {
@@ -226,8 +226,9 @@ func singleRustExternalService(ctx context.Context, store ExternalServiceStore)
 
 // parseCrateInformation parses the newline-delimited JSON file for a crate,
 // assuming the pattern that's used in the github.com/rust-lang/crates.io-index
-func parseCrateInformation(contents string) ([]shared.Repo, error) {
-	var result []shared.Repo
+func parseCrateInformation(contents string) ([]shared.MinimalPackageRepoRef, error) {
+	crates := make(map[reposource.PackageName]*shared.MinimalPackageRepoRef, strings.Count(contents, "\n"))
+
 	for _, line := range strings.Split(contents, "\n") {
 		if line == "" {
 			continue
@@ -243,13 +244,23 @@ func parseCrateInformation(contents string) ([]shared.Repo, error) {
 			return nil, err
 		}
 
-		pkg := shared.Repo{
-			Scheme:  shared.RustPackagesScheme,
-			Name:    reposource.PackageName(info.Name),
-			Version: info.Version,
+		name := reposource.PackageName(info.Name)
+		if crate, ok := crates[name]; ok {
+			crate.Versions = append(crate.Versions, info.Version)
+		} else {
+			crates[name] = &shared.MinimalPackageRepoRef{
+				Scheme:   shared.RustPackagesScheme,
+				Name:     name,
+				Versions: []string{info.Version},
+			}
 		}
-		result = append(result, pkg)
-
 	}
+
+	result := make([]shared.MinimalPackageRepoRef, 0, len(crates))
+
+	for _, crate := range crates {
+		result = append(result, *crate)
+	}
+
 	return result, nil
 }

internal/codeintel/dependencies/internal/store/observability.go

@@ -51,7 +51,7 @@ func newOperations(observationCtx *observation.Context) *operations {
 		preciseDependents:            op("PreciseDependents"),
 		selectRepoRevisionsToResolve: op("SelectRepoRevisionsToResolve"),
 		updateResolvedRevisions:      op("UpdateResolvedRevisions"),
-		upsertDependencyRepos:        op("UpsertDependencyRepos"),
+		upsertDependencyRepos:        op("InsertDependencyRepos"),
 		upsertLockfileGraph:          op("UpsertLockfileGraph"),
 		listLockfileIndexes:          op("ListLockfileIndexes"),
 		getLockfileIndex:             op("GetLockfileIndex"),

internal/codeintel/dependencies/internal/store/scan.go

@@ -6,16 +6,12 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/database/dbutil"
 )
 
-//
-// Scans `shared.Repo`
-
-func scanDependencyRepo(s dbutil.Scanner) (shared.Repo, error) {
-	var v shared.Repo
-	err := s.Scan(&v.ID, &v.Scheme, &v.Name, &v.Version)
-	return v, err
+func scanDependencyRepo(s dbutil.Scanner) (shared.PackageRepoReference, error) {
+	var ref shared.PackageRepoReference
+	var version shared.PackageRepoRefVersion
+	err := s.Scan(&ref.ID, &ref.Scheme, &ref.Name, &version.ID, &version.PackageRefID, &version.Version)
+	ref.Versions = []shared.PackageRepoRefVersion{version}
+	return ref, err
 }
 
-//
-// Scans `[]shared.Repo`
-
 var scanDependencyRepos = basestore.NewSliceScanner(scanDependencyRepo)

internal/codeintel/dependencies/internal/store/store.go

@@ -6,6 +6,7 @@ import (
 	"github.com/keegancsmith/sqlf"
 	"github.com/lib/pq"
 	"github.com/opentracing/opentracing-go/log"
+	"golang.org/x/exp/slices"
 
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies/shared"
 	"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
@@ -14,13 +15,15 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/database/batch"
 	"github.com/sourcegraph/sourcegraph/internal/database/dbutil"
 	"github.com/sourcegraph/sourcegraph/internal/observation"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
 )
 
 // Store provides the interface for package dependencies storage.
 type Store interface {
-	ListDependencyRepos(ctx context.Context, opts ListDependencyReposOpts) (dependencyRepos []shared.Repo, err error)
-	UpsertDependencyRepos(ctx context.Context, deps []shared.Repo) (newDeps []shared.Repo, err error)
-	DeleteDependencyReposByID(ctx context.Context, ids ...int) (err error)
+	ListPackageRepoRefs(ctx context.Context, opts ListDependencyReposOpts) (dependencyRepos []shared.PackageRepoReference, total int, err error)
+	InsertPackageRepoRefs(ctx context.Context, deps []shared.MinimalPackageRepoRef) (newDeps []shared.PackageRepoReference, newVersions []shared.PackageRepoRefVersion, err error)
+	DeletePackageRepoRefsByID(ctx context.Context, ids ...int) (err error)
+	DeletePackageRepoRefVersionsByID(ctx context.Context, ids ...int) (err error)
 }
 
 // store manages the database tables for package dependencies.
@@ -39,16 +42,16 @@ func New(op *observation.Context, db database.DB) *store {
 
 // ListDependencyReposOpts are options for listing dependency repositories.
 type ListDependencyReposOpts struct {
-	Scheme          string
-	Name            reposource.PackageName
-	After           any
-	Limit           int
-	NewestFirst     bool
-	ExcludeVersions bool
+	Scheme              string
+	Name                reposource.PackageName
+	ExactNameOnly       bool
+	After               int
+	Limit               int
+	MostRecentlyUpdated bool
 }
 
 // ListDependencyRepos returns dependency repositories to be synced by gitserver.
-func (s *store) ListDependencyRepos(ctx context.Context, opts ListDependencyReposOpts) (dependencyRepos []shared.Repo, err error) {
+func (s *store) ListPackageRepoRefs(ctx context.Context, opts ListDependencyReposOpts) (dependencyRepos []shared.PackageRepoReference, total int, err error) {
 	ctx, _, endObservation := s.operations.listDependencyRepos.With(ctx, &err, observation.Args{LogFields: []log.Field{
 		log.String("scheme", opts.Scheme),
 	}})
@@ -58,34 +61,68 @@ func (s *store) ListDependencyRepos(ctx context.Context, opts ListDependencyRepo
 		}})
 	}()
 
-	sortExpr := "id ASC"
-	switch {
-	case opts.NewestFirst && !opts.ExcludeVersions:
-		sortExpr = "id DESC"
-	case opts.ExcludeVersions:
-		sortExpr = "name ASC"
+	sortExpr := "ORDER BY lr.id ASC"
+	if opts.MostRecentlyUpdated {
+		sortExpr = "ORDER BY prv.id DESC"
 	}
 
-	selectCols := sqlf.Sprintf("id, scheme, name, version")
-	if opts.ExcludeVersions {
-		// id is likely not stable here, so no one should actually use it. Should we set it to 0?
-		selectCols = sqlf.Sprintf("DISTINCT ON(name) id, scheme, name, '' AS version")
-	}
+	selectColumns := sqlf.Sprintf("lr.id, lr.scheme, lr.name, prv.id, prv.package_id, prv.version")
 
-	return scanDependencyRepos(s.db.Query(ctx, sqlf.Sprintf(
+	depReposMap := basestore.NewOrderedMap[int, shared.PackageRepoReference]()
+	scanner := basestore.NewKeyedCollectionScanner[*basestore.OrderedMap[int, shared.PackageRepoReference], int, shared.PackageRepoReference, shared.PackageRepoReference](depReposMap, func(s dbutil.Scanner) (int, shared.PackageRepoReference, error) {
+		dep, err := scanDependencyRepo(s)
+		return dep.ID, dep, err
+	}, dependencyVersionsReducer{})
+
+	query := sqlf.Sprintf(
 		listDependencyReposQuery,
-		selectCols,
+		selectColumns,
 		sqlf.Join(makeListDependencyReposConds(opts), "AND"),
-		sqlf.Sprintf(sortExpr),
 		makeLimit(opts.Limit),
-	)))
+		sqlf.Sprintf(sortExpr),
+	)
+	err = scanner(s.db.Query(ctx, query))
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "error listing dependency repos")
+	}
+
+	query = sqlf.Sprintf(
+		listDependencyReposQuery,
+		sqlf.Sprintf("COUNT(lr.id)"),
+		sqlf.Join(makeListDependencyReposConds(opts), "AND"),
+		sqlf.Sprintf(""), sqlf.Sprintf(""),
+	)
+	totalCount, _, err := basestore.ScanFirstInt(s.db.Query(ctx, query))
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "error counting dependency repos")
+	}
+
+	dependencyRepos = depReposMap.Values()
+	return dependencyRepos, totalCount, err
+}
+
+type dependencyVersionsReducer struct{}
+
+func (dependencyVersionsReducer) Create() shared.PackageRepoReference {
+	return shared.PackageRepoReference{}
+}
+
+func (dependencyVersionsReducer) Reduce(collection shared.PackageRepoReference, value shared.PackageRepoReference) shared.PackageRepoReference {
+	value.Versions = append(collection.Versions, value.Versions...)
+	collection, value = value, collection
+	return collection
 }
 
 const listDependencyReposQuery = `
 SELECT %s
-FROM lsif_dependency_repos
-WHERE %s
-ORDER BY %s
+FROM (
+	SELECT id, scheme, name
+	FROM lsif_dependency_repos
+	WHERE %s
+	%s
+) lr
+JOIN package_repo_versions prv
+ON lr.id = prv.package_id
 %s
 `
 
@@ -93,29 +130,17 @@ func makeListDependencyReposConds(opts ListDependencyReposOpts) []*sqlf.Query {
 	conds := make([]*sqlf.Query, 0, 3)
 	conds = append(conds, sqlf.Sprintf("scheme = %s", opts.Scheme))
 
-	if opts.Name != "" {
+	if opts.Name != "" && opts.ExactNameOnly {
 		conds = append(conds, sqlf.Sprintf("name = %s", opts.Name))
+	} else if opts.Name != "" {
+		conds = append(conds, sqlf.Sprintf("name LIKE ('%%%%' || %s || '%%%%')", opts.Name))
 	}
 
-	switch after := opts.After.(type) {
-	case nil:
-		break
-	case int:
-		switch {
-		case opts.ExcludeVersions:
-			panic("cannot set ExcludeVersions and pass ID-based offset")
-		case opts.NewestFirst && after > 0:
-			conds = append(conds, sqlf.Sprintf("id < %s", opts.After))
-		case !opts.NewestFirst && after > 0:
-			conds = append(conds, sqlf.Sprintf("id > %s", opts.After))
-		}
-	case string, reposource.PackageName:
-		switch {
-		case opts.NewestFirst:
-			panic("cannot set NewestFirst and pass name-based offset")
-		case opts.ExcludeVersions && after != "":
-			conds = append(conds, sqlf.Sprintf("name > %s", opts.After))
-		}
+	switch {
+	case opts.MostRecentlyUpdated && opts.After > 0:
+		conds = append(conds, sqlf.Sprintf("id < %s", opts.After))
+	case !opts.MostRecentlyUpdated && opts.After > 0:
+		conds = append(conds, sqlf.Sprintf("id > %s", opts.After))
 	}
 
 	return conds
@@ -129,54 +154,230 @@ func makeLimit(limit int) *sqlf.Query {
 	return sqlf.Sprintf("LIMIT %s", limit)
 }
 
-// UpsertDependencyRepos creates the given dependency repos if they don't yet exist. The values
-// that did not exist previously are returned.
-func (s *store) UpsertDependencyRepos(ctx context.Context, deps []shared.Repo) (newDeps []shared.Repo, err error) {
+// InsertDependencyRepos creates the given dependency repos if they don't yet exist. The values that did not exist previously are returned.
+// [{npm, @types/nodejs, [v0.0.1]}, {npm, @types/nodejs, [v0.0.2]}] will be collapsed into [{npm, @types/nodejs, [v0.0.1, v0.0.2]}]
+func (s *store) InsertPackageRepoRefs(ctx context.Context, deps []shared.MinimalPackageRepoRef) (newDeps []shared.PackageRepoReference, newVersions []shared.PackageRepoRefVersion, err error) {
 	ctx, _, endObservation := s.operations.upsertDependencyRepos.With(ctx, &err, observation.Args{LogFields: []log.Field{
-		log.Int("numDeps", len(deps)),
+		log.Int("numInputDeps", len(deps)),
 	}})
 	defer func() {
 		endObservation(1, observation.Args{LogFields: []log.Field{
-			log.Int("numNewDeps", len(newDeps)),
+			log.Int("newDependencies", len(newDeps)),
+			log.Int("newVersion", len(newVersions)),
+			log.Int("numDedupedDeps", len(deps)),
 		}})
 	}()
 
-	callback := func(inserter *batch.Inserter) error {
-		for _, dep := range deps {
-			if err := inserter.Insert(ctx, dep.Scheme, dep.Name, dep.Version); err != nil {
-				return err
-			}
-		}
-
-		return nil
+	if len(deps) == 0 {
+		return
 	}
 
-	returningScanner := func(rows dbutil.Scanner) error {
-		dependencyRepo, err := scanDependencyRepo(rows)
-		if err != nil {
-			return err
+	slices.SortStableFunc(deps, func(a, b shared.MinimalPackageRepoRef) bool {
+		if a.Scheme != b.Scheme {
+			return a.Scheme < b.Scheme
 		}
 
-		newDeps = append(newDeps, dependencyRepo)
-		return nil
+		return a.Name < b.Name
+	})
+
+	// first reduce
+	var lastCommon int
+	for i, dep := range deps[1:] {
+		if dep.Name == deps[lastCommon].Name && dep.Scheme == deps[lastCommon].Scheme {
+			deps[lastCommon].Versions = append(deps[lastCommon].Versions, dep.Versions...)
+			deps[i+1] = shared.MinimalPackageRepoRef{}
+		} else {
+			lastCommon = i + 1
+		}
 	}
 
-	err = batch.WithInserterWithReturn(
+	// then collapse
+	nonDupes := deps[:0]
+	for _, dep := range deps {
+		if dep.Name != "" && dep.Scheme != "" {
+			nonDupes = append(nonDupes, dep)
+		}
+	}
+	// replace the originals :wave
+	deps = nonDupes
+
+	tx, err := s.db.Transact(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer func() {
+		err = tx.Done(err)
+	}()
+
+	for _, tempTableQuery := range []string{temporaryPackageRepoRefsTableQuery, temporaryPackageRepoRefVersionsTableQuery} {
+		if err := tx.Exec(ctx, sqlf.Sprintf(tempTableQuery)); err != nil {
+			return nil, nil, errors.Wrap(err, "failed to create temporary tables")
+		}
+	}
+
+	err = batch.WithInserter(
 		ctx,
-		s.db.Handle(),
-		"lsif_dependency_repos",
+		tx.Handle(),
+		"t_package_repo_refs",
 		batch.MaxNumPostgresParameters,
-		[]string{"scheme", "name", "version"},
-		"ON CONFLICT DO NOTHING",
-		[]string{"id", "scheme", "name", "version"},
-		returningScanner,
-		callback,
+		[]string{"scheme", "name"},
+		func(inserter *batch.Inserter) error {
+			for _, pkg := range deps {
+				if err := inserter.Insert(ctx, pkg.Scheme, pkg.Name); err != nil {
+					return err
+				}
+			}
+			return nil
+		},
 	)
-	return newDeps, err
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to insert package repos in temporary table")
+	}
+
+	newDeps, err = basestore.NewSliceScanner(func(rows dbutil.Scanner) (dep shared.PackageRepoReference, err error) {
+		err = rows.Scan(&dep.ID, &dep.Scheme, &dep.Name)
+		return
+	})(tx.Query(ctx, sqlf.Sprintf(transferPackageRepoRefsQuery)))
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to transfer package repos from temporary table")
+	}
+
+	// we need the IDs of all newly inserted and already existing package repo references
+	// for all of the references in `deps`, so that we have the package repo reference ID that
+	// we need for the package repo reference versions table.
+	// We already have the IDs of newly inserted ones (in `newDeps`), but for simplicity we'll
+	// just search based on (scheme, name) tuple in `deps`.
+
+	// we slice into `deps`, which will continuously shrink as we batch based on the amount of
+	// postgres parameters we can fit. Divide by 2 because for each entry in the batch, we need 2 free params
+	const maxBatchSize = batch.MaxNumPostgresParameters / 2
+	remainingDeps := deps
+
+	allIDs := make([]int, 0, len(deps))
+
+	for len(remainingDeps) > 0 {
+		// avoid slice out of bounds nonsense
+		var batch []shared.MinimalPackageRepoRef
+		if len(remainingDeps) <= maxBatchSize {
+			batch, remainingDeps = remainingDeps, nil
+		} else {
+			batch, remainingDeps = remainingDeps[:maxBatchSize], remainingDeps[maxBatchSize:]
+		}
+
+		// dont over-allocate
+		max := maxBatchSize
+		if len(remainingDeps) < maxBatchSize {
+			max = len(remainingDeps)
+		}
+		params := make([]*sqlf.Query, 0, max)
+		for _, dep := range batch {
+			params = append(params, sqlf.Sprintf("(%s, %s)", dep.Scheme, dep.Name))
+		}
+
+		query := sqlf.Sprintf(
+			getAttemptedInsertDependencyReposQuery,
+			sqlf.Join(params, ", "),
+		)
+
+		allIDsWindow, err := basestore.ScanInts(tx.Query(ctx, query))
+		if err != nil {
+			return nil, nil, err
+		}
+		allIDs = append(allIDs, allIDsWindow...)
+	}
+
+	err = batch.WithInserter(
+		ctx,
+		tx.Handle(),
+		"t_package_repo_versions",
+		batch.MaxNumPostgresParameters,
+		[]string{"package_id", "version"},
+		func(inserter *batch.Inserter) error {
+			for i, dep := range deps {
+				for _, version := range dep.Versions {
+					if err := inserter.Insert(ctx, allIDs[i], version); err != nil {
+						return err
+					}
+				}
+			}
+			return nil
+		})
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to insert package repo versions in temporary table")
+	}
+
+	newVersions, err = basestore.NewSliceScanner(func(rows dbutil.Scanner) (version shared.PackageRepoRefVersion, err error) {
+		err = rows.Scan(&version.ID, &version.PackageRefID, &version.Version)
+		return
+	})(tx.Query(ctx, sqlf.Sprintf(transferPackageRepoRefVersionsQuery)))
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "failed to transfer package repos from temporary table")
+	}
+
+	return newDeps, newVersions, err
 }
 
+const temporaryPackageRepoRefsTableQuery = `
+CREATE TEMPORARY TABLE t_package_repo_refs (
+	scheme TEXT NOT NULL,
+	name TEXT NOT NULL
+) ON COMMIT DROP
+`
+
+const temporaryPackageRepoRefVersionsTableQuery = `
+CREATE TEMPORARY TABLE t_package_repo_versions (
+	package_id BIGINT NOT NULL,
+	version TEXT NOT NULL
+) ON COMMIT DROP
+`
+
+const transferPackageRepoRefsQuery = `
+INSERT INTO lsif_dependency_repos (scheme, name, version)
+SELECT scheme, name, '👁️temporary_sentinel_value👁️'
+FROM (
+	SELECT scheme, name
+	FROM t_package_repo_refs t
+	-- we reduce all package repo refs before insert, so nothing in
+	-- t_package_repo_refs to dedupe
+	EXCEPT ALL
+	(
+		SELECT scheme, name
+		FROM lsif_dependency_repos
+	)
+	-- we order by ID in list as we use ID-based pagination,
+	-- but unit tests rely on name ordering when paginating
+	ORDER BY name
+) diff
+RETURNING id, scheme, name
+`
+
+const transferPackageRepoRefVersionsQuery = `
+INSERT INTO package_repo_versions (package_id, version)
+SELECT package_id, version
+FROM t_package_repo_versions
+-- we dont reduce package repo versions,
+-- so omit 'ALL' here to avoid conflict
+EXCEPT
+(
+	SELECT package_id, version
+	FROM package_repo_versions
+)
+-- unit tests rely on a certain order
+ORDER BY package_id, version
+RETURNING id, package_id, version
+`
+
+// Always use the lowest ID for a given (scheme,name), like in the migration
+// migrations/frontend/1674669326_package_repos_separate_versions_table/up.sql#L41
+const getAttemptedInsertDependencyReposQuery = `
+SELECT MIN(id) FROM lsif_dependency_repos
+WHERE (scheme, name) IN (VALUES %s)
+GROUP BY (scheme, name)
+ORDER BY (scheme, name)
+`
+
 // DeleteDependencyReposByID removes the dependency repos with the given ids, if they exist.
-func (s *store) DeleteDependencyReposByID(ctx context.Context, ids ...int) (err error) {
+func (s *store) DeletePackageRepoRefsByID(ctx context.Context, ids ...int) (err error) {
 	ctx, _, endObservation := s.operations.deleteDependencyReposByID.With(ctx, &err, observation.Args{LogFields: []log.Field{
 		log.Int("numIDs", len(ids)),
 	}})
@@ -194,15 +395,15 @@ DELETE FROM lsif_dependency_repos
 WHERE id = ANY(%s)
 `
 
-// Transact returns a store in a transaction.
-func (s *store) Transact(ctx context.Context) (*store, error) {
-	txBase, err := s.db.Transact(ctx)
-	if err != nil {
-		return nil, err
+func (s *store) DeletePackageRepoRefVersionsByID(ctx context.Context, ids ...int) (err error) {
+	if len(ids) == 0 {
+		return nil
 	}
 
-	return &store{
-		db:         txBase,
-		operations: s.operations,
-	}, nil
+	return s.db.Exec(ctx, sqlf.Sprintf(deleteDependencyRepoVersionsByID, pq.Array(ids)))
 }
+
+const deleteDependencyRepoVersionsByID = `
+DELETE FROM package_repo_versions
+WHERE id = ANY(%s)
+`

internal/codeintel/dependencies/internal/store/store_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/sourcegraph/log/logtest"
 
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies/shared"
-	"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
 	"github.com/sourcegraph/sourcegraph/internal/database"
 	"github.com/sourcegraph/sourcegraph/internal/database/dbtest"
 	"github.com/sourcegraph/sourcegraph/internal/observation"
@@ -24,55 +23,67 @@ func TestUpsertDependencyRepo(t *testing.T) {
 	db := database.NewDB(logger, dbtest.NewDB(logger, t))
 	store := New(&observation.TestContext, db)
 
-	batches := [][]shared.Repo{
+	batches := [][]shared.MinimalPackageRepoRef{
 		{
 			// Test same-set flushes
-			shared.Repo{Scheme: "npm", Name: "bar", Version: "2.0.0"}, // id=1
-			shared.Repo{Scheme: "npm", Name: "bar", Version: "2.0.0"}, // id=2, duplicate
+			shared.MinimalPackageRepoRef{Scheme: "npm", Name: "bar", Versions: []string{"2.0.0"}},
+			shared.MinimalPackageRepoRef{Scheme: "npm", Name: "bar", Versions: []string{"2.0.0"}},
 		},
 		{
-			shared.Repo{Scheme: "npm", Name: "bar", Version: "3.0.0"}, // id=3
-			shared.Repo{Scheme: "npm", Name: "foo", Version: "1.0.0"}, // id=4
+			shared.MinimalPackageRepoRef{Scheme: "npm", Name: "bar", Versions: []string{"3.0.0"}}, // id=3
+			shared.MinimalPackageRepoRef{Scheme: "npm", Name: "foo", Versions: []string{"1.0.0"}}, // id=4
 		},
 		{
 			// Test different-set flushes
-			shared.Repo{Scheme: "npm", Name: "foo", Version: "1.0.0"}, // id=5, duplicate
-			shared.Repo{Scheme: "npm", Name: "foo", Version: "2.0.0"}, // id=6
+			shared.MinimalPackageRepoRef{Scheme: "npm", Name: "foo", Versions: []string{"1.0.0", "2.0.0"}},
 		},
 	}
 
-	var allNewDeps []shared.Repo
+	var allNewDeps []shared.PackageRepoReference
+	var allNewVersions []shared.PackageRepoRefVersion
 	for _, batch := range batches {
-		newDeps, err := store.UpsertDependencyRepos(ctx, batch)
+		newDeps, newVersions, err := store.InsertPackageRepoRefs(ctx, batch)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		allNewDeps = append(allNewDeps, newDeps...)
+		allNewVersions = append(allNewVersions, newVersions...)
 	}
 
-	want := []shared.Repo{
-		{ID: 1, Scheme: "npm", Name: "bar", Version: "2.0.0"},
-		{ID: 3, Scheme: "npm", Name: "bar", Version: "3.0.0"},
-		{ID: 4, Scheme: "npm", Name: "foo", Version: "1.0.0"},
-		{ID: 6, Scheme: "npm", Name: "foo", Version: "2.0.0"},
+	want := []shared.PackageRepoReference{
+		{ID: 1, Scheme: "npm", Name: "bar"},
+		{ID: 2, Scheme: "npm", Name: "foo"},
 	}
-	if diff := cmp.Diff(allNewDeps, want); diff != "" {
-		t.Fatalf("mismatch (-have, +want): %s", diff)
+	if diff := cmp.Diff(want, allNewDeps); diff != "" {
+		t.Fatalf("mismatch (-want, +got): %s", diff)
 	}
 
-	have, err := store.ListDependencyRepos(ctx, ListDependencyReposOpts{
+	wantV := []shared.PackageRepoRefVersion{
+		{ID: 1, PackageRefID: 1, Version: "2.0.0"},
+		{ID: 2, PackageRefID: 1, Version: "3.0.0"},
+		{ID: 3, PackageRefID: 2, Version: "1.0.0"},
+		{ID: 4, PackageRefID: 2, Version: "2.0.0"},
+	}
+	if diff := cmp.Diff(wantV, allNewVersions); diff != "" {
+		t.Fatalf("mismatch (-want, +got): %s", diff)
+	}
+
+	have, _, err := store.ListPackageRepoRefs(ctx, ListDependencyReposOpts{
 		Scheme: shared.NpmPackagesScheme,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if diff := cmp.Diff(have, want); diff != "" {
-		t.Fatalf("mismatch (-have, +want): %s", diff)
+
+	want[0].Versions = []shared.PackageRepoRefVersion{{ID: 1, PackageRefID: 1, Version: "2.0.0"}, {ID: 2, PackageRefID: 1, Version: "3.0.0"}}
+	want[1].Versions = []shared.PackageRepoRefVersion{{ID: 3, PackageRefID: 2, Version: "1.0.0"}, {ID: 4, PackageRefID: 2, Version: "2.0.0"}}
+	if diff := cmp.Diff(want, have); diff != "" {
+		t.Fatalf("mismatch (-want, +got): %s", diff)
 	}
 }
 
-func TestListDependencyRepos(t *testing.T) {
+func TestListPackageRepoRefs(t *testing.T) {
 	if testing.Short() {
 		t.Skip()
 	}
@@ -82,49 +93,48 @@ func TestListDependencyRepos(t *testing.T) {
 	db := database.NewDB(logger, dbtest.NewDB(logger, t))
 	store := New(&observation.TestContext, db)
 
-	batches := []shared.Repo{
-		{Scheme: "npm", Name: "bar", Version: "2.0.0"},    // id=1
-		{Scheme: "npm", Name: "foo", Version: "1.0.0"},    // id=2
-		{Scheme: "npm", Name: "bar", Version: "2.0.1"},    // id=3
-		{Scheme: "npm", Name: "foo", Version: "1.0.0"},    // id=4
-		{Scheme: "npm", Name: "bar", Version: "3.0.0"},    // id=5
-		{Scheme: "npm", Name: "banana", Version: "2.0.0"}, // id=6
-		{Scheme: "npm", Name: "turtle", Version: "4.2.0"}, // id=7
+	batches := []shared.MinimalPackageRepoRef{
+		{Scheme: "npm", Name: "bar", Versions: []string{"2.0.0"}},    // id=1
+		{Scheme: "npm", Name: "foo", Versions: []string{"1.0.0"}},    // id=2
+		{Scheme: "npm", Name: "bar", Versions: []string{"2.0.1"}},    // id=3
+		{Scheme: "npm", Name: "foo", Versions: []string{"1.0.0"}},    // id=4
+		{Scheme: "npm", Name: "bar", Versions: []string{"3.0.0"}},    // id=5
+		{Scheme: "npm", Name: "banana", Versions: []string{"2.0.0"}}, // id=6
+		{Scheme: "npm", Name: "turtle", Versions: []string{"4.2.0"}}, // id=7
 	}
 
-	if _, err := store.UpsertDependencyRepos(ctx, batches); err != nil {
+	if _, _, err := store.InsertPackageRepoRefs(ctx, batches); err != nil {
 		t.Fatal(err)
 	}
 
-	var lastName reposource.PackageName
-	lastName = ""
-	for _, test := range [][]shared.Repo{
+	var lastID int
+	for _, test := range [][]shared.PackageRepoReference{
 		{{Scheme: "npm", Name: "banana"}, {Scheme: "npm", Name: "bar"}, {Scheme: "npm", Name: "foo"}},
 		{{Scheme: "npm", Name: "turtle"}},
 	} {
-		depRepos, err := store.ListDependencyRepos(ctx, ListDependencyReposOpts{
-			Scheme:          "npm",
-			After:           lastName,
-			Limit:           3,
-			ExcludeVersions: true,
+		depRepos, _, err := store.ListPackageRepoRefs(ctx, ListDependencyReposOpts{
+			Scheme: "npm",
+			After:  lastID,
+			Limit:  3,
 		})
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
 
+		lastID = depRepos[len(depRepos)-1].ID
+
 		for i := range depRepos {
 			depRepos[i].ID = 0
+			depRepos[i].Versions = nil
 		}
 
-		lastName = depRepos[len(depRepos)-1].Name
-
-		if diff := cmp.Diff(depRepos, test); diff != "" {
-			t.Fatalf("mismatch (-have, +want): %s", diff)
+		if diff := cmp.Diff(test, depRepos); diff != "" {
+			t.Errorf("mismatch (-want, +got): %s", diff)
 		}
 	}
 }
 
-func TestDeleteDependencyReposByID(t *testing.T) {
+func TestDeletePackageRepoRefsByID(t *testing.T) {
 	if testing.Short() {
 		t.Skip()
 	}
@@ -134,33 +144,38 @@ func TestDeleteDependencyReposByID(t *testing.T) {
 	db := database.NewDB(logger, dbtest.NewDB(logger, t))
 	store := New(&observation.TestContext, db)
 
-	repos := []shared.Repo{
+	repos := []shared.MinimalPackageRepoRef{
 		// Test same-set flushes
-		{ID: 1, Scheme: "npm", Name: "bar", Version: "2.0.0"},
-		{ID: 2, Scheme: "npm", Name: "bar", Version: "3.0.0"}, // deleted
-		{ID: 3, Scheme: "npm", Name: "foo", Version: "1.0.0"}, // deleted
-		{ID: 4, Scheme: "npm", Name: "foo", Version: "2.0.0"},
+		{Scheme: "npm", Name: "bar", Versions: []string{"2.0.0"}},
+		{Scheme: "npm", Name: "bar", Versions: []string{"3.0.0"}}, // deleted
+		{Scheme: "npm", Name: "foo", Versions: []string{"1.0.0"}}, // deleted
+		{Scheme: "npm", Name: "foo", Versions: []string{"2.0.0"}},
+		{Scheme: "npm", Name: "banan", Versions: []string{"4.2.0"}}, // deleted
 	}
 
-	if _, err := store.UpsertDependencyRepos(ctx, repos); err != nil {
+	if _, _, err := store.InsertPackageRepoRefs(ctx, repos); err != nil {
 		t.Fatal(err)
 	}
-	if err := store.DeleteDependencyReposByID(ctx, 2, 3); err != nil {
-		t.Fatalf(err.Error())
+	if err := store.DeletePackageRepoRefsByID(ctx, 1); err != nil {
+		t.Fatal(err)
 	}
 
-	have, err := store.ListDependencyRepos(ctx, ListDependencyReposOpts{
+	if err := store.DeletePackageRepoRefVersionsByID(ctx, 3, 4); err != nil {
+		t.Fatal(err)
+	}
+
+	have, _, err := store.ListPackageRepoRefs(ctx, ListDependencyReposOpts{
 		Scheme: shared.NpmPackagesScheme,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	want := []shared.Repo{
-		{ID: 1, Scheme: "npm", Name: "bar", Version: "2.0.0"},
-		{ID: 4, Scheme: "npm", Name: "foo", Version: "2.0.0"},
+	want := []shared.PackageRepoReference{
+		{ID: 2, Scheme: "npm", Name: "bar", Versions: []shared.PackageRepoRefVersion{{ID: 2, PackageRefID: 2, Version: "2.0.0"}}},
+		{ID: 3, Scheme: "npm", Name: "foo", Versions: []shared.PackageRepoRefVersion{{ID: 5, PackageRefID: 3, Version: "2.0.0"}}},
 	}
-	if diff := cmp.Diff(have, want); diff != "" {
-		t.Fatalf("mismatch (-have, +want): %s", diff)
+	if diff := cmp.Diff(want, have); diff != "" {
+		t.Fatalf("mismatch (-want, +got): %s", diff)
 	}
 }

internal/codeintel/dependencies/observability.go

@@ -8,9 +8,10 @@ import (
 )
 
 type operations struct {
-	listDependencyRepos       *observation.Operation
-	upsertDependencyRepos     *observation.Operation
-	deleteDependencyReposByID *observation.Operation
+	listPackageRepos                 *observation.Operation
+	deletePackageRepoRefVersionsByID *observation.Operation
+	upsertPackageRepoRefs            *observation.Operation
+	deletePackageRepoRefsByID        *observation.Operation
 }
 
 var m = new(metrics.SingletonREDMetrics)
@@ -34,8 +35,9 @@ func newOperations(observationCtx *observation.Context) *operations {
 	}
 
 	return &operations{
-		listDependencyRepos:       op("ListDependencyRepos"),
-		upsertDependencyRepos:     op("UpsertDependencyRepos"),
-		deleteDependencyReposByID: op("DeleteDependencyReposByID"),
+		listPackageRepos:                 op("ListPackageRepoRefs"),
+		deletePackageRepoRefVersionsByID: op("DeletePackageRepoRefVersionsByID"),
+		upsertPackageRepoRefs:            op("InsertPackageRepoRefs"),
+		deletePackageRepoRefsByID:        op("DeletePackageRepoRefsByID"),
 	}
 }

internal/codeintel/dependencies/service.go

@@ -3,6 +3,8 @@ package dependencies
 import (
 	"context"
 
+	"github.com/opentracing/opentracing-go/log"
+
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies/internal/store"
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies/shared"
 	"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
@@ -22,46 +24,69 @@ func newService(observationCtx *observation.Context, store store.Store) *Service
 	}
 }
 
-type Repo = shared.Repo
+type (
+	PackageRepoReference  = shared.PackageRepoReference
+	PackageRepoRefVersion = shared.PackageRepoRefVersion
+	MinimalPackageRepoRef = shared.MinimalPackageRepoRef
+)
 
 type ListDependencyReposOpts struct {
 	// Scheme is the moniker scheme to filter for e.g. 'gomod', 'npm' etc.
 	Scheme string
 	// Name is the package name to filter for e.g. '@types/node' etc.
 	Name reposource.PackageName
+
+	// ExactNameOnly enables exact name matching instead of substring.
+	ExactNameOnly bool
 	// After is the value predominantly used for pagination. When sorting by
 	// newest first, this should be the ID of the last element in the previous
 	// page, when excluding versions it should be the last package name in the
 	// previous page.
-	After any
+	After int
 	// Limit limits the size of the results set to be returned.
 	Limit int
-	// NewestFirst sorts by when a (package, version) was added to the list.
-	// Incompatible with ExcludeVersions below.
-	NewestFirst bool
-	// ExcludeVersions returns one row for every package, instead of one for
-	// every (package, version) tuple. Results will be sorted by name to make
-	// pagination possible. Takes precedence over NewestFirst.
-	ExcludeVersions bool
+	// MostRecentlyUpdated sorts by when a package was updated (either created or
+	// a new version added).
+	MostRecentlyUpdated bool
 }
 
-func (s *Service) ListDependencyRepos(ctx context.Context, opts ListDependencyReposOpts) (_ []Repo, err error) {
-	ctx, _, endObservation := s.operations.listDependencyRepos.With(ctx, &err, observation.Args{})
+func (s *Service) ListPackageRepoRefs(ctx context.Context, opts ListDependencyReposOpts) (_ []PackageRepoReference, total int, err error) {
+	ctx, _, endObservation := s.operations.listPackageRepos.With(ctx, &err, observation.Args{LogFields: []log.Field{
+		log.String("scheme", opts.Scheme),
+		log.String("name", string(opts.Name)),
+		log.Bool("exactOnly", opts.ExactNameOnly),
+		log.Int("after", opts.After),
+		log.Int("limit", opts.Limit),
+		log.Bool("mostRecentlyUpdated", opts.MostRecentlyUpdated),
+	}})
 	defer endObservation(1, observation.Args{})
 
-	return s.store.ListDependencyRepos(ctx, store.ListDependencyReposOpts(opts))
+	return s.store.ListPackageRepoRefs(ctx, store.ListDependencyReposOpts(opts))
 }
 
-func (s *Service) UpsertDependencyRepos(ctx context.Context, deps []Repo) (_ []Repo, err error) {
-	ctx, _, endObservation := s.operations.upsertDependencyRepos.With(ctx, &err, observation.Args{})
+func (s *Service) InsertPackageRepoRefs(ctx context.Context, deps []MinimalPackageRepoRef) (_ []shared.PackageRepoReference, _ []shared.PackageRepoRefVersion, err error) {
+	ctx, _, endObservation := s.operations.upsertPackageRepoRefs.With(ctx, &err, observation.Args{LogFields: []log.Field{
+		log.Int("packageRepoRefs", len(deps)),
+	}})
 	defer endObservation(1, observation.Args{})
 
-	return s.store.UpsertDependencyRepos(ctx, deps)
+	return s.store.InsertPackageRepoRefs(ctx, deps)
 }
 
-func (s *Service) DeleteDependencyReposByID(ctx context.Context, ids ...int) (err error) {
-	ctx, _, endObservation := s.operations.deleteDependencyReposByID.With(ctx, &err, observation.Args{})
+func (s *Service) DeletePackageRepoRefsByID(ctx context.Context, ids ...int) (err error) {
+	ctx, _, endObservation := s.operations.deletePackageRepoRefsByID.With(ctx, &err, observation.Args{LogFields: []log.Field{
+		log.Int("packageRepoRefs", len(ids)),
+	}})
 	defer endObservation(1, observation.Args{})
 
-	return s.store.DeleteDependencyReposByID(ctx, ids...)
+	return s.store.DeletePackageRepoRefsByID(ctx, ids...)
+}
+
+func (s *Service) DeletePackageRepoRefVersionsByID(ctx context.Context, ids ...int) (err error) {
+	ctx, _, endObservation := s.operations.deletePackageRepoRefVersionsByID.With(ctx, &err, observation.Args{LogFields: []log.Field{
+		log.Int("packageRepoRefVersions", len(ids)),
+	}})
+	defer endObservation(1, observation.Args{})
+
+	return s.store.DeletePackageRepoRefVersionsByID(ctx, ids...)
 }

internal/codeintel/dependencies/shared/types.go

@@ -5,19 +5,23 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
 )
 
-type Repo struct {
-	ID      int
-	Scheme  string
-	Name    reposource.PackageName
-	Version string
+type PackageRepoReference struct {
+	ID       int
+	Scheme   string
+	Name     reposource.PackageName
+	Versions []PackageRepoRefVersion
 }
 
-type PackageDependency interface {
-	RepoName() api.RepoName
-	GitTagFromVersion() string
-	Scheme() string
-	PackageSyntax() reposource.PackageName
-	PackageVersion() string
+type PackageRepoRefVersion struct {
+	ID           int
+	PackageRefID int
+	Version      string
+}
+
+type MinimalPackageRepoRef struct {
+	Scheme   string
+	Name     reposource.PackageName
+	Versions []string
 }
 
 type PackageDependencyLiteral struct {
@@ -27,25 +31,3 @@ type PackageDependencyLiteral struct {
 	PackageSyntaxValue     reposource.PackageName
 	PackageVersionValue    string
 }
-
-func TestPackageDependencyLiteral(
-	repoNameValue api.RepoName,
-	gitTagFromVersionValue string,
-	schemeValue string,
-	packageSyntaxValue reposource.PackageName,
-	packageVersionValue string,
-) PackageDependency {
-	return PackageDependencyLiteral{
-		RepoNameValue:          repoNameValue,
-		GitTagFromVersionValue: gitTagFromVersionValue,
-		SchemeValue:            schemeValue,
-		PackageSyntaxValue:     packageSyntaxValue,
-		PackageVersionValue:    packageVersionValue,
-	}
-}
-
-func (d PackageDependencyLiteral) RepoName() api.RepoName                { return d.RepoNameValue }
-func (d PackageDependencyLiteral) GitTagFromVersion() string             { return d.GitTagFromVersionValue }
-func (d PackageDependencyLiteral) Scheme() string                        { return d.SchemeValue }
-func (d PackageDependencyLiteral) PackageSyntax() reposource.PackageName { return d.PackageSyntaxValue }
-func (d PackageDependencyLiteral) PackageVersion() string                { return d.PackageVersionValue }

internal/conf/reposource/python_packages.go

@@ -22,7 +22,7 @@ func NewPythonVersionedPackage(name PackageName, version string) *PythonVersione
 
 // ParseVersionedPackage parses a string in a '<name>(==<version>)?' format into an
 // PythonVersionedPackage.
-func ParseVersionedPackage(dependency string) (*PythonVersionedPackage, error) {
+func ParseVersionedPackage(dependency string) *PythonVersionedPackage {
 	var dep PythonVersionedPackage
 	if i := strings.LastIndex(dependency, "=="); i == -1 {
 		dep.Name = PackageName(dependency)
@@ -30,10 +30,10 @@ func ParseVersionedPackage(dependency string) (*PythonVersionedPackage, error) {
 		dep.Name = PackageName(strings.TrimSpace(dependency[:i]))
 		dep.Version = strings.TrimSpace(dependency[i+2:])
 	}
-	return &dep, nil
+	return &dep
 }
 
-func ParsePythonPackageFromName(name PackageName) (*PythonVersionedPackage, error) {
+func ParsePythonPackageFromName(name PackageName) *PythonVersionedPackage {
 	return ParseVersionedPackage(string(name))
 }
 
@@ -44,7 +44,7 @@ func ParsePythonPackageFromRepoName(name api.RepoName) (*PythonVersionedPackage,
 	if len(dependency) == len(name) {
 		return nil, errors.New("invalid python dependency repo name, missing python/ prefix")
 	}
-	return ParseVersionedPackage(dependency)
+	return ParseVersionedPackage(dependency), nil
 }
 
 func (p *PythonVersionedPackage) Scheme() string {

internal/conf/reposource/ruby_packages.go

@@ -23,7 +23,7 @@ func NewRubyVersionedPackage(name PackageName, version string) *RubyVersionedPac
 
 // ParseRubyVersionedPackage parses a string in a '<name>(@version>)?' format into an
 // RubyVersionedPackage.
-func ParseRubyVersionedPackage(dependency string) (*RubyVersionedPackage, error) {
+func ParseRubyVersionedPackage(dependency string) *RubyVersionedPackage {
 	var dep RubyVersionedPackage
 	if i := strings.LastIndex(dependency, "@"); i == -1 {
 		dep.Name = PackageName(dependency)
@@ -31,10 +31,10 @@ func ParseRubyVersionedPackage(dependency string) (*RubyVersionedPackage, error)
 		dep.Name = PackageName(strings.TrimSpace(dependency[:i]))
 		dep.Version = strings.TrimSpace(dependency[i+1:])
 	}
-	return &dep, nil
+	return &dep
 }
 
-func ParseRubyPackageFromName(name PackageName) (*RubyVersionedPackage, error) {
+func ParseRubyPackageFromName(name PackageName) *RubyVersionedPackage {
 	return ParseRubyVersionedPackage(string(name))
 }
 
@@ -45,7 +45,7 @@ func ParseRubyPackageFromRepoName(name api.RepoName) (*RubyVersionedPackage, err
 	if len(dependency) == len(name) {
 		return nil, errors.Newf("invalid Ruby dependency repo name, missing %s prefix '%s'", rubyPackagesPrefix, name)
 	}
-	return ParseRubyVersionedPackage(dependency)
+	return ParseRubyVersionedPackage(dependency), nil
 }
 
 func (p *RubyVersionedPackage) Scheme() string {

internal/conf/reposource/rust_packages.go

@@ -21,7 +21,7 @@ func NewRustVersionedPackage(name PackageName, version string) *RustVersionedPac
 
 // ParseRustVersionedPackage parses a string in a '<name>(@version>)?' format into an
 // RustVersionedPackage.
-func ParseRustVersionedPackage(dependency string) (*RustVersionedPackage, error) {
+func ParseRustVersionedPackage(dependency string) *RustVersionedPackage {
 	var dep RustVersionedPackage
 	if i := strings.LastIndex(dependency, "@"); i == -1 {
 		dep.Name = PackageName(dependency)
@@ -29,10 +29,10 @@ func ParseRustVersionedPackage(dependency string) (*RustVersionedPackage, error)
 		dep.Name = PackageName(strings.TrimSpace(dependency[:i]))
 		dep.Version = strings.TrimSpace(dependency[i+1:])
 	}
-	return &dep, nil
+	return &dep
 }
 
-func ParseRustPackageFromName(name PackageName) (*RustVersionedPackage, error) {
+func ParseRustPackageFromName(name PackageName) *RustVersionedPackage {
 	return ParseRustVersionedPackage(string(name))
 }
 
@@ -43,7 +43,7 @@ func ParseRustPackageFromRepoName(name api.RepoName) (*RustVersionedPackage, err
 	if len(dependency) == len(name) {
 		return nil, errors.Newf("invalid Rust dependency repo name, missing crates/ prefix '%s'", name)
 	}
-	return ParseRustVersionedPackage(dependency)
+	return ParseRustVersionedPackage(dependency), nil
 }
 
 func (p *RustVersionedPackage) Scheme() string {

internal/database/basestore/scan_collections.go

@@ -126,7 +126,7 @@ func NewKeyedCollectionScanner[Map keyedMap[K, Vs], K comparable, V, Vs any](
 // a SQL rows object to scan a single map value.
 func NewMapScanner[K comparable, V any](f func(dbutil.Scanner) (K, V, error)) func(rows Rows, queryErr error) (map[K]V, error) {
 	return func(rows Rows, queryErr error) (map[K]V, error) {
-		m := &UnorderedMap[K, V]{m: make(map[K]V)}
+		m := NewUnorderedmap[K, V]()
 		err := NewKeyedCollectionScanner[*UnorderedMap[K, V], K, V, V](m, f, SingleValueReducer[V]{})(rows, queryErr)
 		return m.ToMap(), err
 	}
@@ -137,7 +137,7 @@ func NewMapScanner[K comparable, V any](f func(dbutil.Scanner) (K, V, error)) fu
 // multiple times with a SQL rows object to scan a single map key value.
 func NewMapSliceScanner[K comparable, V any](f func(dbutil.Scanner) (K, V, error)) func(rows Rows, queryErr error) (map[K][]V, error) {
 	return func(rows Rows, queryErr error) (map[K][]V, error) {
-		m := &UnorderedMap[K, []V]{m: make(map[K][]V)}
+		m := NewUnorderedmap[K, []V]()
 		err := NewKeyedCollectionScanner[*UnorderedMap[K, []V], K, V, []V](m, f, SliceReducer[V]{})(rows, queryErr)
 		return m.ToMap(), err
 	}
@@ -177,6 +177,10 @@ type UnorderedMap[K comparable, V any] struct {
 	m map[K]V
 }
 
+func NewUnorderedmap[K comparable, V any]() *UnorderedMap[K, V] {
+	return &UnorderedMap[K, V]{m: make(map[K]V)}
+}
+
 func (m UnorderedMap[K, V]) Get(key K) (V, bool) {
 	v, ok := m.m[key]
 	return v, ok
@@ -202,6 +206,10 @@ type OrderedMap[K comparable, V any] struct {
 	m *orderedmap.OrderedMap[K, V]
 }
 
+func NewOrderedMap[K comparable, V any]() *OrderedMap[K, V] {
+	return &OrderedMap[K, V]{m: orderedmap.New[K, V]()}
+}
+
 func (m OrderedMap[K, V]) Get(key K) (V, bool) {
 	return m.m.Get(key)
 }

internal/database/batch/batch.go

@@ -170,7 +170,7 @@ func NewInserterWithReturn(
 	returningScanner ReturningScanner,
 ) *Inserter {
 	numColumns := len(columnNames)
-	maxBatchSize := getMaxBatchSize(numColumns, maxNumParameters)
+	maxBatchSize := GetMaxBatchSize(numColumns, maxNumParameters)
 	queryPrefix := makeQueryPrefix(tableName, columnNames)
 	querySuffix := makeQuerySuffix(numColumns, maxNumParameters)
 	onConflictSuffix := makeOnConflictSuffix(onConflictClause)
@@ -348,9 +348,9 @@ const MaxNumPostgresParameters = 32767
 // in a single insert statement.
 const MaxNumSQLiteParameters = 999
 
-// getMaxBatchSize returns the number of rows that can be inserted into a single table with the
+// GetMaxBatchSize returns the number of rows that can be inserted into a single table with the
 // given number of columns via a single insert statement.
-func getMaxBatchSize(numColumns, maxNumParameters int) int {
+func GetMaxBatchSize(numColumns, maxNumParameters int) int {
 	return (maxNumParameters / numColumns) * numColumns
 }
 
@@ -365,8 +365,10 @@ func makeQueryPrefix(tableName string, columnNames []string) string {
 	return fmt.Sprintf(`INSERT INTO "%s" (%s) VALUES `, tableName, strings.Join(quotedColumnNames, ","))
 }
 
-var querySuffixCache = map[int]string{}
-var querySuffixCacheMutex sync.Mutex
+var (
+	querySuffixCache      = map[int]string{}
+	querySuffixCacheMutex sync.Mutex
+)
 
 // makeQuerySuffix creates the suffix of the batch insert statement containing the placeholder
 // variables, e.g. `($1,$2,$3),($4,$5,$6),...`. The number of rows will be the maximum number of

internal/database/schema.json

@@ -116,6 +116,10 @@
       "Name": "func_insert_zoekt_repo",
       "Definition": "CREATE OR REPLACE FUNCTION public.func_insert_zoekt_repo()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\nBEGIN\n  INSERT INTO zoekt_repos (repo_id) VALUES (NEW.id);\n\n  RETURN NULL;\nEND;\n$function$\n"
     },
+    {
+      "Name": "func_lsif_dependency_repos_backfill",
+      "Definition": "CREATE OR REPLACE FUNCTION public.func_lsif_dependency_repos_backfill()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\n    BEGIN\n        INSERT INTO package_repo_versions (package_id, version)\n        VALUES (NEW.id, NEW.version);\n\n        RETURN NULL;\n    END;\n$function$\n"
+    },
     {
       "Name": "func_lsif_uploads_delete",
       "Definition": "CREATE OR REPLACE FUNCTION public.func_lsif_uploads_delete()\n RETURNS trigger\n LANGUAGE plpgsql\nAS $function$\n    BEGIN\n        UPDATE lsif_uploads_audit_logs\n        SET record_deleted_at = NOW()\n        WHERE upload_id IN (\n            SELECT id FROM OLD\n        );\n\n        RETURN NULL;\n    END;\n$function$\n"
@@ -814,6 +818,15 @@
       "Increment": 1,
       "CycleOption": "NO"
     },
+    {
+      "Name": "package_repo_versions_id_seq",
+      "TypeName": "bigint",
+      "StartValue": 1,
+      "MinimumValue": 1,
+      "MaximumValue": 9223372036854775807,
+      "Increment": 1,
+      "CycleOption": "NO"
+    },
     {
       "Name": "permission_sync_jobs_id_seq",
       "TypeName": "integer",
@@ -12376,10 +12389,25 @@
           "IndexDefinition": "CREATE UNIQUE INDEX lsif_dependency_repos_unique_triplet ON lsif_dependency_repos USING btree (scheme, name, version)",
           "ConstraintType": "u",
           "ConstraintDefinition": "UNIQUE (scheme, name, version)"
+        },
+        {
+          "Name": "lsif_dependency_repos_name_idx",
+          "IsPrimaryKey": false,
+          "IsUnique": false,
+          "IsExclusion": false,
+          "IsDeferrable": false,
+          "IndexDefinition": "CREATE INDEX lsif_dependency_repos_name_idx ON lsif_dependency_repos USING btree (name)",
+          "ConstraintType": "",
+          "ConstraintDefinition": ""
         }
       ],
       "Constraints": null,
-      "Triggers": []
+      "Triggers": [
+        {
+          "Name": "lsif_dependency_repos_backfill",
+          "Definition": "CREATE TRIGGER lsif_dependency_repos_backfill AFTER INSERT ON lsif_dependency_repos FOR EACH ROW WHEN (new.version \u003c\u003e '👁️temporary_sentinel_value👁️'::text) EXECUTE FUNCTION func_lsif_dependency_repos_backfill()"
+        }
+      ]
     },
     {
       "Name": "lsif_dependency_syncing_jobs",
@@ -17062,6 +17090,93 @@
       ],
       "Triggers": []
     },
+    {
+      "Name": "package_repo_versions",
+      "Comment": "",
+      "Columns": [
+        {
+          "Name": "id",
+          "Index": 1,
+          "TypeName": "bigint",
+          "IsNullable": false,
+          "Default": "nextval('package_repo_versions_id_seq'::regclass)",
+          "CharacterMaximumLength": 0,
+          "IsIdentity": false,
+          "IdentityGeneration": "",
+          "IsGenerated": "NEVER",
+          "GenerationExpression": "",
+          "Comment": ""
+        },
+        {
+          "Name": "package_id",
+          "Index": 2,
+          "TypeName": "bigint",
+          "IsNullable": false,
+          "Default": "",
+          "CharacterMaximumLength": 0,
+          "IsIdentity": false,
+          "IdentityGeneration": "",
+          "IsGenerated": "NEVER",
+          "GenerationExpression": "",
+          "Comment": ""
+        },
+        {
+          "Name": "version",
+          "Index": 3,
+          "TypeName": "text",
+          "IsNullable": false,
+          "Default": "",
+          "CharacterMaximumLength": 0,
+          "IsIdentity": false,
+          "IdentityGeneration": "",
+          "IsGenerated": "NEVER",
+          "GenerationExpression": "",
+          "Comment": ""
+        }
+      ],
+      "Indexes": [
+        {
+          "Name": "package_repo_versions_pkey",
+          "IsPrimaryKey": true,
+          "IsUnique": true,
+          "IsExclusion": false,
+          "IsDeferrable": false,
+          "IndexDefinition": "CREATE UNIQUE INDEX package_repo_versions_pkey ON package_repo_versions USING btree (id)",
+          "ConstraintType": "p",
+          "ConstraintDefinition": "PRIMARY KEY (id)"
+        },
+        {
+          "Name": "package_repo_versions_unique_version_per_package",
+          "IsPrimaryKey": false,
+          "IsUnique": true,
+          "IsExclusion": false,
+          "IsDeferrable": false,
+          "IndexDefinition": "CREATE UNIQUE INDEX package_repo_versions_unique_version_per_package ON package_repo_versions USING btree (package_id, version)",
+          "ConstraintType": "",
+          "ConstraintDefinition": ""
+        },
+        {
+          "Name": "package_repo_versions_fk_idx",
+          "IsPrimaryKey": false,
+          "IsUnique": false,
+          "IsExclusion": false,
+          "IsDeferrable": false,
+          "IndexDefinition": "CREATE INDEX package_repo_versions_fk_idx ON package_repo_versions USING btree (package_id)",
+          "ConstraintType": "",
+          "ConstraintDefinition": ""
+        }
+      ],
+      "Constraints": [
+        {
+          "Name": "package_id_fk",
+          "ConstraintType": "f",
+          "RefTableName": "lsif_dependency_repos",
+          "IsDeferrable": false,
+          "ConstraintDefinition": "FOREIGN KEY (package_id) REFERENCES lsif_dependency_repos(id) ON DELETE CASCADE"
+        }
+      ],
+      "Triggers": []
+    },
     {
       "Name": "permission_sync_jobs",
       "Comment": "",

internal/database/schema.md

@@ -1759,6 +1759,11 @@ Foreign-key constraints:
 Indexes:
     "lsif_dependency_repos_pkey" PRIMARY KEY, btree (id)
     "lsif_dependency_repos_unique_triplet" UNIQUE CONSTRAINT, btree (scheme, name, version)
+    "lsif_dependency_repos_name_idx" btree (name)
+Referenced by:
+    TABLE "package_repo_versions" CONSTRAINT "package_id_fk" FOREIGN KEY (package_id) REFERENCES lsif_dependency_repos(id) ON DELETE CASCADE
+Triggers:
+    lsif_dependency_repos_backfill AFTER INSERT ON lsif_dependency_repos FOR EACH ROW WHEN (new.version <> '👁️temporary_sentinel_value👁️'::text) EXECUTE FUNCTION func_lsif_dependency_repos_backfill()
 
 ```
 
@@ -2634,6 +2639,22 @@ Referenced by:
 
 ```
 
+# Table "public.package_repo_versions"
+```
+   Column   |  Type  | Collation | Nullable |                      Default                      
+------------+--------+-----------+----------+---------------------------------------------------
+ id         | bigint |           | not null | nextval('package_repo_versions_id_seq'::regclass)
+ package_id | bigint |           | not null | 
+ version    | text   |           | not null | 
+Indexes:
+    "package_repo_versions_pkey" PRIMARY KEY, btree (id)
+    "package_repo_versions_unique_version_per_package" UNIQUE, btree (package_id, version)
+    "package_repo_versions_fk_idx" btree (package_id)
+Foreign-key constraints:
+    "package_id_fk" FOREIGN KEY (package_id) REFERENCES lsif_dependency_repos(id) ON DELETE CASCADE
+
+```
+
 # Table "public.permission_sync_jobs"
 ```
         Column        |           Type           | Collation | Nullable |                     Default                      

internal/extsvc/rubygems/rubygems_test.go

@@ -38,8 +38,7 @@ func TestGetPackageContents(t *testing.T) {
 	ctx := context.Background()
 	client, stop := newTestHTTPClient(t)
 	defer stop()
-	dep, err := reposource.ParseRubyVersionedPackage("hola@0.1.0")
-	require.Nil(t, err)
+	dep := reposource.ParseRubyVersionedPackage("hola@0.1.0")
 	readCloser, _, err := client.GetPackageContents(ctx, dep)
 	require.Nil(t, err)
 	defer readCloser.Close()

internal/own/codeowners/proto/codeowners.pb.go

@@ -268,11 +268,11 @@ var file_codeowners_proto_rawDesc = []byte{
 	0x6e, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x18, 0x01, 0x20,
 	0x01, 0x28, 0x09, 0x52, 0x06, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65,
 	0x6d, 0x61, 0x69, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x6d, 0x61, 0x69,
-	0x6c, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
+	0x6c, 0x42, 0x42, 0x5a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
 	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x73, 0x6f, 0x75, 0x72,
 	0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
 	0x2f, 0x6f, 0x77, 0x6e, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x73, 0x2f,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

internal/repos/go_packages_test.go

@@ -14,30 +14,26 @@ import (
 
 func TestGoPackagesSource_ListRepos(t *testing.T) {
 	ctx := context.Background()
-	depsSvc := testDependenciesService(ctx, t, []dependencies.Repo{
+	depsSvc := testDependenciesService(ctx, t, []dependencies.MinimalPackageRepoRef{
 		{
-			ID:      1,
-			Scheme:  dependencies.GoPackagesScheme,
-			Name:    "github.com/foo/barbaz",
-			Version: "v0.0.1", // test that we create a repo for this module even if it's missing.
+			Scheme: dependencies.GoPackagesScheme,
+			Name:   "github.com/foo/barbaz",
+			Versions: []string{
+				"v0.0.1",
+			}, // test that we create a repo for this module even if it's missing.
 		},
 		{
-			ID:      2,
-			Scheme:  dependencies.GoPackagesScheme,
-			Name:    "github.com/gorilla/mux",
-			Version: "v1.8.0", // test deduplication with version from config
+			Scheme: dependencies.GoPackagesScheme,
+			Name:   "github.com/gorilla/mux",
+			Versions: []string{
+				"v1.8.0", // test deduplication with version from config
+				"v1.7.4", // test multiple versions of the same module
+			},
 		},
 		{
-			ID:      3,
-			Scheme:  dependencies.GoPackagesScheme,
-			Name:    "github.com/gorilla/mux",
-			Version: "v1.7.4", // test multiple versions of the same module
-		},
-		{
-			ID:      4,
-			Scheme:  dependencies.GoPackagesScheme,
-			Name:    "github.com/goware/urlx",
-			Version: "v0.3.1",
+			Scheme:   dependencies.GoPackagesScheme,
+			Name:     "github.com/goware/urlx",
+			Versions: []string{"v0.3.1"},
 		},
 	})
 

internal/repos/npm_packages_test.go

@@ -35,18 +35,30 @@ func TestGetNpmDependencyRepos(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		deps, err := depsSvc.ListDependencyRepos(ctx, dependencies.ListDependencyReposOpts{
-			Scheme: dependencies.NpmPackagesScheme,
-			Name:   reposource.PackageName(testCase.pkgName),
+		deps, _, err := depsSvc.ListPackageRepoRefs(ctx, dependencies.ListDependencyReposOpts{
+			Scheme:        dependencies.NpmPackagesScheme,
+			Name:          reposource.PackageName(testCase.pkgName),
+			ExactNameOnly: true,
 		})
-		require.Nil(t, err)
+		if err != nil {
+			t.Fatalf("unexpected error listing package repos: %v", err)
+		}
+
 		depStrs := []string{}
 		for _, dep := range deps {
 			pkg, err := reposource.ParseNpmPackageFromPackageSyntax(dep.Name)
-			require.Nil(t, err)
-			depStrs = append(depStrs,
-				(&reposource.NpmVersionedPackage{NpmPackageName: pkg, Version: dep.Version}).VersionedPackageSyntax(),
-			)
+			if err != nil {
+				t.Fatalf("unexpected error parsing package from package name: %v", err)
+			}
+
+			for _, version := range dep.Versions {
+				depStrs = append(depStrs,
+					(&reposource.NpmVersionedPackage{
+						NpmPackageName: pkg,
+						Version:        version.Version,
+					}).VersionedPackageSyntax(),
+				)
+			}
 		}
 		sort.Strings(depStrs)
 		sort.Strings(testCase.matches)
@@ -54,21 +66,27 @@ func TestGetNpmDependencyRepos(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		depStrs := []string{}
-		lastID := 0
-		for i := 0; i < len(testCase.matches); i++ {
-			deps, err := depsSvc.ListDependencyRepos(ctx, dependencies.ListDependencyReposOpts{
-				Scheme: dependencies.NpmPackagesScheme,
-				Name:   reposource.PackageName(testCase.pkgName),
-				After:  lastID,
-				Limit:  1,
-			})
-			require.Nil(t, err)
-			require.Equal(t, len(deps), 1)
-			pkg, err := reposource.ParseNpmPackageFromPackageSyntax(deps[0].Name)
-			require.Nil(t, err)
-			depStrs = append(depStrs, (&reposource.NpmVersionedPackage{NpmPackageName: pkg, Version: deps[0].Version}).VersionedPackageSyntax())
-			lastID = deps[0].ID
+		var depStrs []string
+		deps, _, err := depsSvc.ListPackageRepoRefs(ctx, dependencies.ListDependencyReposOpts{
+			Scheme:        dependencies.NpmPackagesScheme,
+			Name:          reposource.PackageName(testCase.pkgName),
+			ExactNameOnly: true,
+			Limit:         1,
+		})
+		require.Nil(t, err)
+		if len(testCase.matches) > 0 {
+			require.Equal(t, 1, len(deps))
+		} else {
+			require.Equal(t, 0, len(deps))
+			continue
+		}
+		pkg, err := reposource.ParseNpmPackageFromPackageSyntax(deps[0].Name)
+		require.Nil(t, err)
+		for _, version := range deps[0].Versions {
+			depStrs = append(depStrs, (&reposource.NpmVersionedPackage{
+				NpmPackageName: pkg,
+				Version:        version.Version,
+			}).VersionedPackageSyntax())
 		}
 		sort.Strings(depStrs)
 		sort.Strings(testCase.matches)
@@ -76,13 +94,13 @@ func TestGetNpmDependencyRepos(t *testing.T) {
 	}
 }
 
-func testDependenciesService(ctx context.Context, t *testing.T, dependencyRepos []dependencies.Repo) *dependencies.Service {
+func testDependenciesService(ctx context.Context, t *testing.T, dependencyRepos []dependencies.MinimalPackageRepoRef) *dependencies.Service {
 	t.Helper()
 	logger := logtest.Scoped(t)
 	db := database.NewDB(logger, dbtest.NewDB(logger, t))
 	depsSvc := dependencies.TestService(db, nil)
 
-	_, err := depsSvc.UpsertDependencyRepos(ctx, dependencyRepos)
+	_, _, err := depsSvc.InsertPackageRepoRefs(ctx, dependencyRepos)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
@@ -98,19 +116,19 @@ var testDependencies = []string{
 	"pkg2@0.1-abc",
 	"pkg2@1",
 }
-var testDependencyRepos = func() []dependencies.Repo {
-	dependencyRepos := []dependencies.Repo{}
-	for i, depStr := range testDependencies {
+
+var testDependencyRepos = func() []dependencies.MinimalPackageRepoRef {
+	dependencyRepos := []dependencies.MinimalPackageRepoRef{}
+	for _, depStr := range testDependencies {
 		dep, err := reposource.ParseNpmVersionedPackage(depStr)
 		if err != nil {
 			panic(err.Error())
 		}
 
-		dependencyRepos = append(dependencyRepos, dependencies.Repo{
-			ID:      i + 1,
-			Scheme:  dependencies.NpmPackagesScheme,
-			Name:    dep.PackageSyntax(),
-			Version: dep.Version,
+		dependencyRepos = append(dependencyRepos, dependencies.MinimalPackageRepoRef{
+			Scheme:   dependencies.NpmPackagesScheme,
+			Name:     dep.PackageSyntax(),
+			Versions: []string{dep.Version},
 		})
 	}
 
@@ -119,30 +137,24 @@ var testDependencyRepos = func() []dependencies.Repo {
 
 func TestNPMPackagesSource_ListRepos(t *testing.T) {
 	ctx := context.Background()
-	depsSvc := testDependenciesService(ctx, t, []dependencies.Repo{
+	depsSvc := testDependenciesService(ctx, t, []dependencies.MinimalPackageRepoRef{
 		{
-			ID:      1,
-			Scheme:  dependencies.NpmPackagesScheme,
-			Name:    "@sourcegraph/sourcegraph.proposed",
-			Version: "12.0.0", // test deduplication with version from config
+			Scheme: dependencies.NpmPackagesScheme,
+			Name:   "@sourcegraph/sourcegraph.proposed",
+			Versions: []string{
+				"12.0.0", // test deduplication with version from config
+				"12.0.1", // test deduplication with version from config
+			},
 		},
 		{
-			ID:      2,
-			Scheme:  dependencies.NpmPackagesScheme,
-			Name:    "@sourcegraph/sourcegraph.proposed",
-			Version: "12.0.1", // test deduplication with version from config
+			Scheme:   dependencies.NpmPackagesScheme,
+			Name:     "@sourcegraph/web-ext",
+			Versions: []string{"3.0.0-fork.1"},
 		},
 		{
-			ID:      3,
-			Scheme:  dependencies.NpmPackagesScheme,
-			Name:    "@sourcegraph/web-ext",
-			Version: "3.0.0-fork.1",
-		},
-		{
-			ID:      4,
-			Scheme:  dependencies.NpmPackagesScheme,
-			Name:    "fastq",
-			Version: "0.9.9", // test missing modules still create a repo.
+			Scheme:   dependencies.NpmPackagesScheme,
+			Name:     "fastq",
+			Versions: []string{"0.9.9"}, // test missing modules still create a repo.
 		},
 	})
 

internal/repos/packages.go

@@ -93,14 +93,12 @@ func (s *PackagesSource) ListRepos(ctx context.Context, results chan SourceResul
 	}()
 
 	const batchLimit = 100
-	var lastName reposource.PackageName
-	lastName = ""
+	var lastID int
 	for {
-		depRepos, err := s.depsSvc.ListDependencyRepos(ctx, dependencies.ListDependencyReposOpts{
-			Scheme:          s.scheme,
-			After:           lastName,
-			Limit:           batchLimit,
-			ExcludeVersions: true,
+		depRepos, _, err := s.depsSvc.ListPackageRepoRefs(ctx, dependencies.ListDependencyReposOpts{
+			Scheme: s.scheme,
+			After:  lastID,
+			Limit:  batchLimit,
 		})
 		if err != nil {
 			results <- SourceResult{Source: s, Err: err}
@@ -110,10 +108,10 @@ func (s *PackagesSource) ListRepos(ctx context.Context, results chan SourceResul
 			break
 		}
 
-		lastName = depRepos[len(depRepos)-1].Name
+		lastID = depRepos[len(depRepos)-1].ID
 
 		// at most batchLimit because of the limit above
-		depReposToHandle := make([]dependencies.Repo, 0, len(depRepos))
+		depReposToHandle := make([]dependencies.PackageRepoReference, 0, len(depRepos))
 		for _, depRepo := range depRepos {
 			if _, ok := handledPackages[depRepo.Name]; !ok {
 				// don't need to add to handledPackages here, as the results from
@@ -186,7 +184,7 @@ func getPackage(s packagesSource, name reposource.PackageName) (reposource.Packa
 	switch d := s.(type) {
 	// Downloading package descriptions is disabled due to performance issues, causing sync times to take >12hr.
 	// Don't re-enable the case below without fixing https://github.com/sourcegraph/sourcegraph/issues/39653.
-	//case packagesDownloadSource:
+	// case packagesDownloadSource:
 	//	return d.GetPackage(ctx, name)
 	default:
 		return d.ParsePackageFromName(name)

internal/repos/python_packages.go

@@ -53,11 +53,11 @@ func (s *pythonPackagesSource) Get(ctx context.Context, name reposource.PackageN
 }
 
 func (pythonPackagesSource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
-	return reposource.ParseVersionedPackage(dep)
+	return reposource.ParseVersionedPackage(dep), nil
 }
 
 func (pythonPackagesSource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
-	return reposource.ParsePythonPackageFromName(name)
+	return reposource.ParsePythonPackageFromName(name), nil
 }
 
 func (pythonPackagesSource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {

internal/repos/python_packages_test.go

@@ -14,31 +14,26 @@ import (
 
 func TestPythonPackagesSource_ListRepos(t *testing.T) {
 	ctx := context.Background()
-	depsSvc := testDependenciesService(ctx, t, []dependencies.Repo{
+	depsSvc := testDependenciesService(ctx, t, []dependencies.MinimalPackageRepoRef{
 		{
-			ID:      1,
-			Scheme:  dependencies.PythonPackagesScheme,
-			Name:    "requests",
-			Version: "2.27.1", // test deduplication with version from config
+			Scheme: dependencies.PythonPackagesScheme,
+			Name:   "requests",
+			Versions: []string{
+				"2.27.1", // test deduplication with version from config
+				"2.27.2", // test multiple versions of the same module
+			},
 		},
 		{
-			ID:      2,
-			Scheme:  dependencies.PythonPackagesScheme,
-			Name:    "requests",
-			Version: "2.27.2", // test multiple versions of the same module
+			Scheme:   dependencies.PythonPackagesScheme,
+			Name:     "numpy",
+			Versions: []string{"1.22.3"},
 		},
 		{
-			ID:      3,
-			Scheme:  dependencies.PythonPackagesScheme,
-			Name:    "numpy",
-			Version: "1.22.3",
+			Scheme:   dependencies.PythonPackagesScheme,
+			Name:     "lofi",
+			Versions: []string{"foobar"}, // test that we create a repo for this package even if it's missing.
 		},
-		{
-			ID:      4,
-			Scheme:  dependencies.PythonPackagesScheme,
-			Name:    "lofi",
-			Version: "foobar", // test that we create a repo for this package even if it's missing.
-		}})
+	})
 
 	svc := types.ExternalService{
 		Kind: extsvc.KindPythonPackages,

internal/repos/ruby_packages.go

@@ -45,12 +45,13 @@ type rubyPackagesSource struct {
 var _ packagesSource = &rubyPackagesSource{}
 
 func (rubyPackagesSource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
-	return reposource.ParseRubyVersionedPackage(dep)
+	return reposource.ParseRubyVersionedPackage(dep), nil
 }
 
 func (rubyPackagesSource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
-	return reposource.ParseRubyPackageFromName(name)
+	return reposource.ParseRubyPackageFromName(name), nil
 }
+
 func (rubyPackagesSource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {
 	return reposource.ParseRubyPackageFromRepoName(repoName)
 }

internal/repos/rust_packages.go

@@ -45,12 +45,13 @@ type rustPackagesSource struct {
 var _ packagesSource = &rustPackagesSource{}
 
 func (rustPackagesSource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
-	return reposource.ParseRustVersionedPackage(dep)
+	return reposource.ParseRustVersionedPackage(dep), nil
 }
 
 func (rustPackagesSource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
-	return reposource.ParseRustPackageFromName(name)
+	return reposource.ParseRustPackageFromName(name), nil
 }
+
 func (rustPackagesSource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {
 	return reposource.ParseRustPackageFromRepoName(repoName)
 }

migrations/frontend/1674669326_package_repos_separate_versions_table/down.sql

@@ -0,0 +1,33 @@
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1 FROM information_schema.tables
+        WHERE
+            table_name = 'package_repo_versions' AND
+            table_schema = current_schema()
+    ) THEN
+        WITH matched_triplets AS (
+            SELECT lr.scheme, lr.name, rpv.version
+            FROM package_repo_versions rpv
+            JOIN lsif_dependency_repos lr
+            ON rpv.package_id = lr.id
+        )
+        INSERT INTO lsif_dependency_repos (scheme, name, version)
+        SELECT * FROM matched_triplets
+        ON CONFLICT DO NOTHING;
+    END IF;
+END
+$$;
+
+DELETE FROM lsif_dependency_repos
+WHERE version = '👁️temporary_sentinel_value👁️';
+
+DROP INDEX IF EXISTS package_repo_versions_fk_idx;
+DROP INDEX IF EXISTS package_repo_versions_unique_version_per_package;
+
+DROP INDEX IF EXISTS lsif_dependency_repos_name_idx;
+
+DROP TABLE IF EXISTS package_repo_versions;
+
+DROP TRIGGER IF EXISTS lsif_dependency_repos_backfill ON lsif_dependency_repos;
+DROP FUNCTION IF EXISTS func_lsif_dependency_repos_backfill;

migrations/frontend/1674669326_package_repos_separate_versions_table/metadata.yaml

@@ -0,0 +1,2 @@
+name: package_repos_separate_versions_table
+parents: [1674480050]

migrations/frontend/1674669326_package_repos_separate_versions_table/up.sql

@@ -0,0 +1,52 @@
+-- This will be a two-step migration:
+-- 1. Create the new table and make versions column in old table nullable
+-- This is so that during a migration, an instance on vX-1 can still read & write without incorrect data.
+-- Then when the instance is on vX, it will read & write to the new+old tables but not the version
+-- column from the old table.
+-- 2. Drop version column in old table and flatten remaining duplicates
+-- The instance on vX is not using this column, and the read queries should be designed to
+-- handle both flattened and non-flattened
+
+CREATE TABLE IF NOT EXISTS package_repo_versions (
+    id BIGSERIAL PRIMARY KEY,
+    package_id BIGINT NOT NULL,
+    version TEXT NOT NULL,
+
+    CONSTRAINT package_id_fk FOREIGN KEY (package_id) REFERENCES lsif_dependency_repos (id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS package_repo_versions_fk_idx ON package_repo_versions (package_id);
+CREATE UNIQUE INDEX IF NOT EXISTS package_repo_versions_unique_version_per_package ON package_repo_versions (package_id, version);
+
+CREATE INDEX IF NOT EXISTS lsif_dependency_repos_name_idx ON lsif_dependency_repos (name);
+
+-- if any rows were inserted into lsif_dependency_repos an instance on a version older than this
+-- schema after the migration happened but before the instance was ugpraded, then we need this trigger
+-- to copy over anything added _after_ the migration but before the _instance upgrade_
+CREATE OR REPLACE FUNCTION func_lsif_dependency_repos_backfill() RETURNS TRIGGER AS $$
+    BEGIN
+        INSERT INTO package_repo_versions (package_id, version)
+        VALUES (NEW.id, NEW.version);
+
+        RETURN NULL;
+    END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS lsif_dependency_repos_backfill ON lsif_dependency_repos;
+CREATE TRIGGER lsif_dependency_repos_backfill AFTER INSERT ON lsif_dependency_repos
+FOR EACH ROW
+WHEN (NEW.version <> '👁️temporary_sentinel_value👁️')
+EXECUTE FUNCTION func_lsif_dependency_repos_backfill();
+
+-- for every existing triplet, we use the lowest ID for a given (scheme,name) tuple
+-- and insert the (version) using that ID into package_repo_versions
+INSERT INTO package_repo_versions (package_id, version)
+SELECT (
+    SELECT MIN(id)
+    FROM lsif_dependency_repos
+    WHERE
+        scheme = lr.scheme AND
+        name = lr.name
+) AS package_id, version
+FROM lsif_dependency_repos lr;
+

migrations/frontend/squashed.sql

@@ -293,6 +293,17 @@ BEGIN
 END;
 $$;
 
+CREATE FUNCTION func_lsif_dependency_repos_backfill() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+    BEGIN
+        INSERT INTO package_repo_versions (package_id, version)
+        VALUES (NEW.id, NEW.version);
+
+        RETURN NULL;
+    END;
+$$;
+
 CREATE FUNCTION func_lsif_uploads_delete() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -3285,6 +3296,21 @@ CREATE VIEW outbound_webhooks_with_event_types AS
           WHERE (outbound_webhook_event_types.outbound_webhook_id = outbound_webhooks.id))) AS event_types
    FROM outbound_webhooks;
 
+CREATE TABLE package_repo_versions (
+    id bigint NOT NULL,
+    package_id bigint NOT NULL,
+    version text NOT NULL
+);
+
+CREATE SEQUENCE package_repo_versions_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE package_repo_versions_id_seq OWNED BY package_repo_versions.id;
+
 CREATE TABLE permission_sync_jobs (
     id integer NOT NULL,
     state text DEFAULT 'queued'::text,
@@ -4172,6 +4198,8 @@ ALTER TABLE ONLY outbound_webhook_logs ALTER COLUMN id SET DEFAULT nextval('outb
 
 ALTER TABLE ONLY outbound_webhooks ALTER COLUMN id SET DEFAULT nextval('outbound_webhooks_id_seq'::regclass);
 
+ALTER TABLE ONLY package_repo_versions ALTER COLUMN id SET DEFAULT nextval('package_repo_versions_id_seq'::regclass);
+
 ALTER TABLE ONLY permission_sync_jobs ALTER COLUMN id SET DEFAULT nextval('permission_sync_jobs_id_seq'::regclass);
 
 ALTER TABLE ONLY permissions ALTER COLUMN id SET DEFAULT nextval('permissions_id_seq'::regclass);
@@ -4499,6 +4527,9 @@ ALTER TABLE ONLY outbound_webhook_logs
 ALTER TABLE ONLY outbound_webhooks
     ADD CONSTRAINT outbound_webhooks_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY package_repo_versions
+    ADD CONSTRAINT package_repo_versions_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY permission_sync_jobs
     ADD CONSTRAINT permission_sync_jobs_pkey PRIMARY KEY (id);
 
@@ -4852,6 +4883,8 @@ CREATE INDEX lsif_dependency_indexing_jobs_state ON lsif_dependency_indexing_job
 
 CREATE INDEX lsif_dependency_indexing_jobs_upload_id ON lsif_dependency_syncing_jobs USING btree (upload_id);
 
+CREATE INDEX lsif_dependency_repos_name_idx ON lsif_dependency_repos USING btree (name);
+
 CREATE INDEX lsif_dependency_syncing_jobs_state ON lsif_dependency_syncing_jobs USING btree (state);
 
 CREATE INDEX lsif_indexes_commit_last_checked_at ON lsif_indexes USING btree (commit_last_checked_at) WHERE (state <> 'deleted'::text);
@@ -4928,6 +4961,10 @@ CREATE INDEX outbound_webhook_payload_process_after_idx ON outbound_webhook_jobs
 
 CREATE INDEX outbound_webhooks_logs_status_code_idx ON outbound_webhook_logs USING btree (status_code);
 
+CREATE INDEX package_repo_versions_fk_idx ON package_repo_versions USING btree (package_id);
+
+CREATE UNIQUE INDEX package_repo_versions_unique_version_per_package ON package_repo_versions USING btree (package_id, version);
+
 CREATE INDEX permission_sync_jobs_process_after ON permission_sync_jobs USING btree (process_after);
 
 CREATE INDEX permission_sync_jobs_repository_id ON permission_sync_jobs USING btree (repository_id);
@@ -5052,6 +5089,8 @@ CREATE TRIGGER batch_spec_workspace_execution_last_dequeues_update AFTER UPDATE
 
 CREATE TRIGGER changesets_update_computed_state BEFORE INSERT OR UPDATE ON changesets FOR EACH ROW EXECUTE FUNCTION changesets_computed_state_ensure();
 
+CREATE TRIGGER lsif_dependency_repos_backfill AFTER INSERT ON lsif_dependency_repos FOR EACH ROW WHEN ((new.version <> '👁️temporary_sentinel_value👁️'::text)) EXECUTE FUNCTION func_lsif_dependency_repos_backfill();
+
 CREATE TRIGGER trig_create_zoekt_repo_on_repo_insert AFTER INSERT ON repo FOR EACH ROW EXECUTE FUNCTION func_insert_zoekt_repo();
 
 CREATE TRIGGER trig_delete_batch_change_reference_on_changesets AFTER DELETE ON batch_changes FOR EACH ROW EXECUTE FUNCTION delete_batch_change_reference_on_changesets();
@@ -5422,6 +5461,9 @@ ALTER TABLE ONLY outbound_webhooks
 ALTER TABLE ONLY outbound_webhooks
     ADD CONSTRAINT outbound_webhooks_updated_by_fkey FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY package_repo_versions
+    ADD CONSTRAINT package_id_fk FOREIGN KEY (package_id) REFERENCES lsif_dependency_repos(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY permission_sync_jobs
     ADD CONSTRAINT permission_sync_jobs_repository_id_fkey FOREIGN KEY (repository_id) REFERENCES repo(id) ON DELETE CASCADE;