Packages: add RubyGems support (#42817)

* Packages: add RubyGems support

* Add metadata.gz file to the generated directory

This makes it easier to write auto-inference rules against Ruby gems.

* Fix CI errors

* Add back empty go.mod file

client/web/src/components/externalServices/externalServices.tsx

@@ -9,6 +9,7 @@ import GitLabIcon from 'mdi-react/GitlabIcon'
 import LanguageGoIcon from 'mdi-react/LanguageGoIcon'
 import LanguageJavaIcon from 'mdi-react/LanguageJavaIcon'
 import LanguagePythonIcon from 'mdi-react/LanguagePythonIcon'
+import LanguageRubyIcon from 'mdi-react/LanguageRubyIcon'
 import LanguageRustIcon from 'mdi-react/LanguageRustIcon'
 import NpmIcon from 'mdi-react/NpmIcon'
 
@@ -30,6 +31,7 @@ import pagureSchemaJSON from '../../../../../schema/pagure.schema.json'
 import perforceSchemaJSON from '../../../../../schema/perforce.schema.json'
 import phabricatorSchemaJSON from '../../../../../schema/phabricator.schema.json'
 import pythonPackagesJSON from '../../../../../schema/python-packages.schema.json'
+import rubyPackagesSchemaJSON from '../../../../../schema/ruby-packages.schema.json'
 import rustPackagesJSON from '../../../../../schema/rust-packages.schema.json'
 import { ExternalRepositoryFields, ExternalServiceKind } from '../../graphql-operations'
 import { EditorAction } from '../../settings/EditorActionsGroup'
@@ -1399,6 +1401,44 @@ const RUST_PACKAGES = {
     editorActions: [],
 }
 
+const RUBY_PACKAGES: AddExternalServiceOptions = {
+    kind: ExternalServiceKind.RUBYPACKAGES,
+    title: 'Ruby Dependencies',
+    icon: LanguageRubyIcon,
+    jsonSchema: rubyPackagesSchemaJSON,
+    defaultDisplayName: 'Ruby Dependencies',
+    defaultConfig: `{
+  "repository": "https://rubygems.org/",
+  "dependencies": ["shopify_api@12.0.0"]
+}`,
+    instructions: (
+        <div>
+            <ol>
+                <li>
+                    The URL https://rubygems.org/ is used if the field
+                    <Code>"repository"</Code> is empty.
+                </li>
+                <li>
+                    Use the syntax <Code>"GEM_NAME@GEM_VERSION"</Code> to list a dependency for the{' '}
+                    <Code>"dependencies"</Code> field.
+                </li>
+                <li>
+                    The field <Code>"repository"</Code> is redacted because it can include <Code>admin:password</Code>{' '}
+                    credentials.
+                </li>
+                <li>
+                    See the{' '}
+                    <Link to="https://www.jfrog.com/confluence/display/JFROG/RubyGems+Repositories">
+                        Artifactory documentation for RubyGem repositories
+                    </Link>{' '}
+                    for details on how to configure an internal Artifactory repository.
+                </li>
+            </ol>
+        </div>
+    ),
+    editorActions: [],
+}
+
 export const codeHostExternalServices: Record<string, AddExternalServiceOptions> = {
     github: GITHUB_DOTCOM,
     ghe: GITHUB_ENTERPRISE,
@@ -1412,6 +1452,7 @@ export const codeHostExternalServices: Record<string, AddExternalServiceOptions>
     git: GENERIC_GIT,
     ...(window.context?.experimentalFeatures?.pythonPackages === 'enabled' ? { pythonPackages: PYTHON_PACKAGES } : {}),
     ...(window.context?.experimentalFeatures?.rustPackages === 'enabled' ? { rustPackages: RUST_PACKAGES } : {}),
+    ...(window.context?.experimentalFeatures?.rubyPackages === 'enabled' ? { rubyPackages: RUBY_PACKAGES } : {}),
     ...(window.context?.experimentalFeatures?.goPackages === 'enabled' ? { goModules: GO_MODULES } : {}),
     ...(window.context?.experimentalFeatures?.jvmPackages === 'enabled' ? { jvmPackages: JVM_PACKAGES } : {}),
     ...(window.context?.experimentalFeatures?.npmPackages === 'enabled' ? { npmPackages: NPM_PACKAGES } : {}),
@@ -1446,6 +1487,7 @@ export const defaultExternalServices: Record<ExternalServiceKind, AddExternalSer
     [ExternalServiceKind.NPMPACKAGES]: NPM_PACKAGES,
     [ExternalServiceKind.PYTHONPACKAGES]: PYTHON_PACKAGES,
     [ExternalServiceKind.RUSTPACKAGES]: RUST_PACKAGES,
+    [ExternalServiceKind.RUBYPACKAGES]: RUBY_PACKAGES,
 }
 
 export const externalRepoIcon = (

client/web/src/enterprise/batches/settings/AddCredentialModal.tsx

@@ -61,6 +61,7 @@ const scopeRequirements: Record<ExternalServiceKind, JSX.Element> = {
     [ExternalServiceKind.GOMODULES]: <span>Unsupported</span>,
     [ExternalServiceKind.PYTHONPACKAGES]: <span>Unsupported</span>,
     [ExternalServiceKind.RUSTPACKAGES]: <span>Unsupported</span>,
+    [ExternalServiceKind.RUBYPACKAGES]: <span>Unsupported</span>,
     [ExternalServiceKind.JVMPACKAGES]: <span>Unsupported</span>,
     [ExternalServiceKind.NPMPACKAGES]: <span>Unsupported</span>,
     [ExternalServiceKind.PERFORCE]: <span>Unsupported</span>,

client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx

@@ -27,6 +27,7 @@ const configInstructionLinks: Record<ExternalServiceKind, string> = {
     [ExternalServiceKind.PHABRICATOR]: 'unsupported',
     [ExternalServiceKind.PYTHONPACKAGES]: 'unsupported',
     [ExternalServiceKind.RUSTPACKAGES]: 'unsupported',
+    [ExternalServiceKind.RUBYPACKAGES]: 'unsupported',
 }
 
 export interface CodeHostSshPublicKeyProps {

client/web/src/site-admin/SiteAdminReportBugPage.tsx

@@ -23,6 +23,7 @@ import pagureSchemaJSON from '../../../../schema/pagure.schema.json'
 import perforceSchemaJSON from '../../../../schema/perforce.schema.json'
 import phabricatorSchemaJSON from '../../../../schema/phabricator.schema.json'
 import pythonPackagesSchemaJSON from '../../../../schema/python-packages.schema.json'
+import rubyPackagesSchemaJSON from '../../../../schema/ruby-packages.schema.json'
 import rustPackagesSchemaJSON from '../../../../schema/rust-packages.schema.json'
 import settingsSchemaJSON from '../../../../schema/settings.schema.json'
 import siteSchemaJSON from '../../../../schema/site.schema.json'
@@ -53,6 +54,7 @@ const externalServices: Record<ExternalServiceKind, JSONSchema> = {
     NPMPACKAGES: npmPackagesSchemaJSON,
     PYTHONPACKAGES: pythonPackagesSchemaJSON,
     RUSTPACKAGES: rustPackagesSchemaJSON,
+    RUBYPACKAGES: rubyPackagesSchemaJSON,
     OTHER: otherExternalServiceSchemaJSON,
     PERFORCE: perforceSchemaJSON,
     PHABRICATOR: phabricatorSchemaJSON,

cmd/frontend/graphqlbackend/schema.graphql

@@ -2372,6 +2372,7 @@ enum ExternalServiceKind {
     PHABRICATOR
     PYTHONPACKAGES
     RUSTPACKAGES
+    RUBYPACKAGES
 }
 
 """

cmd/gitserver/main.go

@@ -46,6 +46,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/extsvc/gomodproxy"
 	"github.com/sourcegraph/sourcegraph/internal/extsvc/npm"
 	"github.com/sourcegraph/sourcegraph/internal/extsvc/pypi"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems"
 	"github.com/sourcegraph/sourcegraph/internal/goroutine"
 	"github.com/sourcegraph/sourcegraph/internal/hostname"
 	"github.com/sourcegraph/sourcegraph/internal/httpcli"
@@ -519,6 +520,14 @@ func getVCSSyncer(
 		}
 		cli := crates.NewClient(urn, httpcli.ExternalDoer)
 		return server.NewRustPackagesSyncer(&c, depsSvc, cli), nil
+	case extsvc.TypeRubyPackages:
+		var c schema.RubyPackagesConnection
+		urn, err := extractOptions(&c)
+		if err != nil {
+			return nil, err
+		}
+		cli := rubygems.NewClient(urn, httpcli.ExternalDoer)
+		return server.NewRubyPackagesSyncer(&c, depsSvc, cli), nil
 	}
 	return &server.GitRepoSyncer{}, nil
 }

cmd/gitserver/server/vcs_syncer_ruby_packages.go

@@ -0,0 +1,150 @@
+package server
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/sourcegraph/sourcegraph/internal/api"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems"
+
+	"github.com/sourcegraph/log"
+
+	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies"
+	"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
+	"github.com/sourcegraph/sourcegraph/internal/unpack"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+	"github.com/sourcegraph/sourcegraph/schema"
+)
+
+func NewRubyPackagesSyncer(
+	connection *schema.RubyPackagesConnection,
+	svc *dependencies.Service,
+	client *rubygems.Client,
+) VCSSyncer {
+
+	repositoryURL := connection.Repository
+	if repositoryURL == "" {
+		repositoryURL = "https://rubygems.org/"
+	}
+	return &vcsPackagesSyncer{
+		logger:      log.Scoped("RubyPackagesSyncer", "sync Ruby packages"),
+		typ:         "ruby_packages",
+		scheme:      dependencies.RubyPackagesScheme,
+		placeholder: reposource.NewRubyVersionedPackage("shopify_api", "12.0.0"),
+		svc:         svc,
+		configDeps:  connection.Dependencies,
+		source:      &rubyDependencySource{repositoryURL: repositoryURL, client: client},
+	}
+}
+
+type rubyDependencySource struct {
+	repositoryURL string
+	client        *rubygems.Client
+}
+
+func (rubyDependencySource) ParseVersionedPackageFromNameAndVersion(name reposource.PackageName, version string) (reposource.VersionedPackage, error) {
+	return reposource.ParseRubyVersionedPackage(string(name) + "@" + version)
+}
+
+func (rubyDependencySource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
+	return reposource.ParseRubyVersionedPackage(dep)
+}
+
+func (rubyDependencySource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
+	return reposource.ParseRubyPackageFromName(name)
+
+}
+func (rubyDependencySource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {
+	return reposource.ParseRubyPackageFromRepoName(repoName)
+}
+
+func (s *rubyDependencySource) Download(ctx context.Context, dir string, dep reposource.VersionedPackage) error {
+	packageURL := fmt.Sprintf("%s/gems/%s-%s.gem", strings.TrimSuffix(s.repositoryURL, "/"), dep.PackageSyntax(), dep.PackageVersion())
+
+	pkg, err := s.client.Get(ctx, packageURL)
+	if err != nil {
+		return errors.Wrapf(err, "error downloading RubyGem with URL '%s'", packageURL)
+	}
+
+	if err = unpackRubyPackage(packageURL, pkg, dir); err != nil {
+		return errors.Wrapf(err, "failed to unzip ruby module from URL %s", packageURL)
+	}
+
+	return nil
+}
+
+func unpackRubyPackage(packageURL string, pkg []byte, workDir string) error {
+	r := bytes.NewReader(pkg)
+	opts := unpack.Opts{
+		SkipInvalid:    true,
+		SkipDuplicates: true,
+		Filter: func(path string, file fs.FileInfo) bool {
+			return path == "data.tar.gz" || path == "metadata.gz"
+		},
+	}
+
+	tmpDir, err := os.MkdirTemp("", "rubygems")
+	if err != nil {
+		return errors.Wrap(err, "failed to create a temporary directory")
+	}
+	defer os.RemoveAll(tmpDir)
+
+	if err := unpack.Tar(r, tmpDir, opts); err != nil {
+		return errors.Wrapf(err, "failed to tar downloaded bytes from URL %s", packageURL)
+	}
+
+	err = unpackRubyDataTarGz(packageURL, filepath.Join(tmpDir, "data.tar.gz"), workDir)
+	if err != nil {
+		return err
+	}
+	metadata, err := os.ReadFile(filepath.Join(tmpDir, "metadata.gz"))
+	if err != nil {
+		return err
+	}
+	metadataReader, err := gzip.NewReader(bytes.NewReader(metadata))
+	if err != nil {
+		return err
+	}
+	metadataBytes, err := ioutil.ReadAll(metadataReader)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(workDir, "rubygems-metadata.yml"), metadataBytes, 0644)
+}
+
+// unpackRubyDataTarGz unpacks the given `data.tar.gz` from a downloaded RubyGem.
+func unpackRubyDataTarGz(packageURL, path string, workDir string) error {
+	r, err := os.Open(path)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read file from downloaded URL %s", packageURL)
+	}
+	defer r.Close()
+	opts := unpack.Opts{
+		SkipInvalid:    true,
+		SkipDuplicates: true,
+		Filter: func(path string, file fs.FileInfo) bool {
+			size := file.Size()
+
+			const sizeLimit = 15 * 1024 * 1024
+			if size >= sizeLimit {
+				return false
+			}
+
+			_, malicious := isPotentiallyMaliciousFilepathInArchive(path, workDir)
+			return !malicious
+		},
+	}
+
+	if err := unpack.Tgz(r, workDir, opts); err != nil {
+		return err
+	}
+
+	return stripSingleOutermostDirectory(workDir)
+}

cmd/gitserver/server/vcs_syncer_rust_packages.go

@@ -45,7 +45,6 @@ func NewRustPackagesSyncer(
 	}
 }
 
-// pythonPackagesSyncer implements packagesSource
 type rustDependencySource struct {
 	client *crates.Client
 }

doc/admin/external_service/index.md

@@ -141,6 +141,7 @@ We recognize there are other code hosts including CVS, Azure Dev Ops, SVN, and m
   - [Go dependencies](go.md)
   - [npm dependencies](npm.md)
   - [Python dependencies](python.md)
+  - [Ruby dependencies](ruby.md)
 
 **Users** can configure the following public code hosts:
 

doc/admin/external_service/ruby-packages.schema.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "ruby-packages.schema.json#",
+  "title": "RubyPackagesConnection",
+  "description": "Configuration for a connection to Ruby packages",
+  "allowComments": true,
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "repository": {
+      "description": "The URL at which the maven repository can be found.",
+      "type": "string",
+      "default": ["https://rubygems.org/"],
+      "examples": ["https://rubygems.org/", "https://<server name>.jfrog.io/artifactory/api/gems/<repository key>"]
+    },
+    "rateLimit": {
+      "description": "Rate limit applied when making background API requests to the configured Ruby repository APIs.",
+      "title": "RubyRateLimit",
+      "type": "object",
+      "required": ["enabled", "requestsPerHour"],
+      "properties": {
+        "enabled": {
+          "description": "true if rate limiting is enabled.",
+          "type": "boolean",
+          "default": true
+        },
+        "requestsPerHour": {
+          "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.",
+          "type": "number",
+          "default": 8,
+          "minimum": 0
+        }
+      },
+      "default": {
+        "enabled": true,
+        "requestsPerHour": 3600
+      }
+    },
+    "dependencies": {
+      "description": "An array of strings specifying Ruby packages to mirror in Sourcegraph.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "examples": [["shopify_api@12.0.0"]]
+    }
+  }
+}

doc/admin/external_service/ruby.md

@@ -0,0 +1,57 @@
+# Ruby dependencies
+
+<aside class="experimental">
+<p>
+<span class="badge badge-experimental">Experimental</span> This feature is experimental and might change or be removed in the future. We've released it as an experimental feature to provide a preview of functionality we're working on.
+</p>
+</aside>
+
+Site admins can sync Ruby dependencies from any RubyGems repositories, including rubygems.org or an internal Artifactory, to their Sourcegraph instance so that users can search and navigate the repositories.
+
+To add Ruby dependencies to Sourcegraph you need to setup a Ruby dependencies code host:
+
+1. As *site admin*: go to **Site admin > Global settings** and enable the experimental feature by adding: `{"experimentalFeatures": {"rubyPackages": "enabled"} }`
+1. As *site admin*: go to **Site admin > Manage code hosts**
+1. Select **Ruby Dependencies**.
+1. [Configure the connection](#configuration) by following the instructions above the text field. Additional fields can be added using <kbd>Cmd/Ctrl+Space</kbd> for auto-completion. See the [configuration documentation below](#configuration).
+1. Press **Add repositories**.
+
+## Repository syncing
+
+There are two ways to sync Ruby dependency repositories.
+
+* **Indexing** (recommended): run [`scip-ruby`](https://sourcegraph.github.io/scip-ruby/) against your Ruby codebase and upload the generated index to Sourcegraph using the [src-cli](https://github.com/sourcegraph/src-cli) command `src code-intel upload`. This is usually setup to run in a CI pipeline. Sourcegraph automatically synchronizes Ruby dependency repositories based on the dependencies that are discovered by `scip-ruby`.
+* **Code host configuration**: manually list dependencies in the `"dependencies"` section of the [JSON configuration](#configuration) when creating the Ruby dependency code host. This method can be useful to verify that the credentials are picked up correctly without having to upload an index.
+
+## Credentials
+
+The `"repository"` field in the [configuration](#configuration) section is automatically redacted and can optionally include the username and password of an internal [Artifactory RubyGems](https://www.jfrog.com/confluence/display/JFROG/RubyGems+Repositories) repository.
+
+## Rate limiting
+
+By default, requests to the RubyGems repository is 8 request per second.
+
+To manually set the value, add the following to your code host configuration:
+
+```json
+"rateLimit": {
+  "enabled": true,
+  "requestsPerHour": 600.0
+}
+```
+where the `requestsPerHour` field is set based on your requirements.
+
+**Not recommended**: Rate-limiting can be turned off entirely as well.
+This increases the risk of overloading the code host.
+
+```json
+"rateLimit": {
+  "enabled": false
+}
+```
+
+## Configuration
+
+Ruby dependencies code host connections support the following configuration options, which are specified in the JSON editor in the site admin "Manage code hosts" area.
+
+<div markdown-func=jsonschemadoc jsonschemadoc:path="admin/external_service/ruby-packages.schema.json">[View page on docs.sourcegraph.com](https://docs.sourcegraph.com/integration/ruby) to see rendered content.</div>

doc/integration/index.md

@@ -15,6 +15,7 @@ Sourcegraph integrates with your other tools to help you search, navigate, and r
   - [npm dependencies](npm.md)
   - [Python dependencies](python.md)
   - [Go dependencies](go.md)
+  - [Ruby dependencies](ruby.md)
   - [Other Git repository hosts](../admin/external_service/other.md)
   - [Editor plugins](editor.md): jump to Sourcegraph from your editor
     - [Open in Editor](open_in_editor.md): jump to your editor from Sourcegraph

doc/integration/ruby.md

@@ -0,0 +1,29 @@
+# Ruby dependencies integration with Sourcegraph
+
+You can use Sourcegraph with Ruby dependencies from any RubyGems repository, including rubygems.org or an internal Artifactory.
+
+This integration makes it possible to search and navigate through the source code of published Ruby library (for example, [`shopify_api@12.0.0`](https://sourcegraph.com/rubygems/shopify_api@v12.0.0)).
+
+Feature | Supported?
+------- | ----------
+[Repository syncing](#repository-syncing) | ✅
+[Repository permissions](#repository-syncing) | ❌
+[Multiple RubyGems repositories](#multiple-ruby-dependency-code-hosts) | ❌
+
+## Setup
+
+See the "[Ruby dependencies](../admin/external_service/ruby.md)" documentation.
+
+## Repository syncing
+
+Site admins can [add Ruby dependencies to Sourcegraph](../admin/external_service/ruby.md#repository-syncing).
+
+## Repository permissions
+
+⚠ Ruby dependencies are visible by all users of the Sourcegraph instance.
+
+## Multiple Ruby dependencies code hosts
+
+⚠️ It's only possible to create one Ruby dependency code host for each Sourcegraph instance.
+
+See the issue [sourcegraph#32461](https://github.com/sourcegraph/sourcegraph/issues/32461) for more details about this limitation.

internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go

@@ -59,8 +59,9 @@ var autoIndexingEnabled = conf.CodeIntelAutoIndexingEnabled
 var schemeToExternalService = map[string]string{
 	dependencies.JVMPackagesScheme:    extsvc.KindJVMPackages,
 	dependencies.NpmPackagesScheme:    extsvc.KindNpmPackages,
-	dependencies.RustPackagesScheme:   extsvc.KindRustPackages,
 	dependencies.PythonPackagesScheme: extsvc.KindPythonPackages,
+	dependencies.RustPackagesScheme:   extsvc.KindRustPackages,
+	dependencies.RubyPackagesScheme:   extsvc.KindRubyPackages,
 }
 
 func (h *dependencySyncSchedulerHandler) Handle(ctx context.Context, logger log.Logger, record workerutil.Record) error {
@@ -244,6 +245,7 @@ func (h *dependencySyncSchedulerHandler) shouldIndexDependencies(ctx context.Con
 		upload.Indexer == "scip-typescript" ||
 		upload.Indexer == "lsif-typescript" ||
 		upload.Indexer == "scip-python" ||
+		upload.Indexer == "scip-ruby" ||
 		upload.Indexer == "rust-analyzer", nil
 }
 

internal/codeintel/dependencies/consts.go

@@ -8,4 +8,5 @@ const (
 	GoPackagesScheme     = shared.GoPackagesScheme
 	PythonPackagesScheme = shared.PythonPackagesScheme
 	RustPackagesScheme   = shared.RustPackagesScheme
+	RubyPackagesScheme   = shared.RubyPackagesScheme
 )

internal/codeintel/dependencies/shared/consts.go

@@ -6,4 +6,5 @@ const (
 	NpmPackagesScheme    = "npm"
 	PythonPackagesScheme = "python"
 	RustPackagesScheme   = "rust-analyzer"
+	RubyPackagesScheme   = "scip-ruby"
 )

internal/conf/reposource/ruby_packages.go

@@ -0,0 +1,89 @@
+package reposource
+
+import (
+	"strings"
+
+	"github.com/sourcegraph/sourcegraph/internal/api"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+const rubyPackagesPrefix = "rubygems/"
+
+type RubyVersionedPackage struct {
+	Name    PackageName
+	Version string
+}
+
+func NewRubyVersionedPackage(name PackageName, version string) *RubyVersionedPackage {
+	return &RubyVersionedPackage{
+		Name:    name,
+		Version: version,
+	}
+}
+
+// ParseRubyVersionedPackage parses a string in a '<name>(@version>)?' format into an
+// RubyVersionedPackage.
+func ParseRubyVersionedPackage(dependency string) (*RubyVersionedPackage, error) {
+	var dep RubyVersionedPackage
+	if i := strings.LastIndex(dependency, "@"); i == -1 {
+		dep.Name = PackageName(dependency)
+	} else {
+		dep.Name = PackageName(strings.TrimSpace(dependency[:i]))
+		dep.Version = strings.TrimSpace(dependency[i+1:])
+	}
+	return &dep, nil
+}
+
+func ParseRubyPackageFromName(name PackageName) (*RubyVersionedPackage, error) {
+	return ParseRubyVersionedPackage(string(name))
+}
+
+// ParseRubyPackageFromRepoName is a convenience function to parse a repo name in a
+// 'crates/<name>(@<version>)?' format into a RubyVersionedPackage.
+func ParseRubyPackageFromRepoName(name api.RepoName) (*RubyVersionedPackage, error) {
+	dependency := strings.TrimPrefix(string(name), rubyPackagesPrefix)
+	if len(dependency) == len(name) {
+		return nil, errors.Newf("invalid Ruby dependency repo name, missing %s prefix '%s'", rubyPackagesPrefix, name)
+	}
+	return ParseRubyVersionedPackage(dependency)
+}
+
+func (p *RubyVersionedPackage) Scheme() string {
+	return "scip-ruby"
+}
+
+func (p *RubyVersionedPackage) PackageSyntax() PackageName {
+	return p.Name
+}
+
+func (p *RubyVersionedPackage) VersionedPackageSyntax() string {
+	if p.Version == "" {
+		return string(p.Name)
+	}
+	return string(p.Name) + "@" + p.Version
+}
+
+func (p *RubyVersionedPackage) PackageVersion() string {
+	return p.Version
+}
+
+func (p *RubyVersionedPackage) Description() string { return "" }
+
+func (p *RubyVersionedPackage) RepoName() api.RepoName {
+	return api.RepoName(rubyPackagesPrefix + p.Name)
+}
+
+func (p *RubyVersionedPackage) GitTagFromVersion() string {
+	version := strings.TrimPrefix(p.Version, "v")
+	return "v" + version
+}
+
+func (p *RubyVersionedPackage) Less(other VersionedPackage) bool {
+	o := other.(*RubyVersionedPackage)
+
+	if p.Name == o.Name {
+		return versionGreaterThan(p.Version, o.Version)
+	}
+
+	return p.Name > o.Name
+}

internal/database/external_services.go

@@ -219,6 +219,7 @@ var ExternalServiceKinds = map[string]ExternalServiceKind{
 	extsvc.KindPhabricator:     {CodeHost: true, JSONSchema: schema.PhabricatorSchemaJSON},
 	extsvc.KindPythonPackages:  {CodeHost: true, JSONSchema: schema.PythonPackagesSchemaJSON},
 	extsvc.KindRustPackages:    {CodeHost: true, JSONSchema: schema.RustPackagesSchemaJSON},
+	extsvc.KindRubyPackages:    {CodeHost: true, JSONSchema: schema.RubyPackagesSchemaJSON},
 }
 
 // ExternalServiceKind describes a kind of external service.

internal/database/repos.go

@@ -562,6 +562,8 @@ func scanRepo(logger log.Logger, rows *sql.Rows, r *types.Repo) (err error) {
 		r.Metadata = &struct{}{}
 	case extsvc.TypeRustPackages:
 		r.Metadata = &struct{}{}
+	case extsvc.TypeRubyPackages:
+		r.Metadata = &struct{}{}
 	default:
 		logger.Warn("unknown service type", log.String("type", typ))
 		return nil

internal/extsvc/codehost.go

@@ -15,7 +15,7 @@ type CodeHost struct {
 
 func (c *CodeHost) IsPackageHost() bool {
 	switch c.ServiceType {
-	case TypeNpmPackages, TypeJVMPackages, TypeGoModules, TypePythonPackages, TypeRustPackages:
+	case TypeNpmPackages, TypeJVMPackages, TypeGoModules, TypePythonPackages, TypeRustPackages, TypeRubyPackages:
 		return true
 	}
 	return false
@@ -46,6 +46,9 @@ var (
 	RustURL      = &url.URL{Host: "crates"}
 	RustPackages = NewCodeHost(RustURL, TypeRustPackages)
 
+	RubyURL      = &url.URL{Host: "rubygems"}
+	RubyPackages = NewCodeHost(RubyURL, TypeRubyPackages)
+
 	PublicCodeHosts = []*CodeHost{
 		GitHubDotCom,
 		GitLabDotCom,
@@ -54,6 +57,7 @@ var (
 		GoModules,
 		PythonPackages,
 		RustPackages,
+		RubyPackages,
 	}
 )
 

internal/extsvc/crates/client.go

@@ -17,11 +17,6 @@ type Client struct {
 	limiter *ratelimit.InstrumentedLimiter
 }
 
-type RustFile struct {
-	Name string
-	URL  string
-}
-
 func NewClient(urn string, cli httpcli.Doer) *Client {
 	return &Client{
 		cli:     cli,

internal/extsvc/rubygems/client.go

@@ -0,0 +1,77 @@
+package rubygems
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/sourcegraph/sourcegraph/internal/httpcli"
+	"github.com/sourcegraph/sourcegraph/internal/ratelimit"
+)
+
+type Client struct {
+	cli httpcli.Doer
+
+	// Self-imposed rate-limiter.
+	limiter *ratelimit.InstrumentedLimiter
+}
+
+func NewClient(urn string, cli httpcli.Doer) *Client {
+	return &Client{
+		cli:     cli,
+		limiter: ratelimit.DefaultRegistry.Get(urn),
+	}
+}
+
+func (c *Client) Get(ctx context.Context, url string) ([]byte, error) {
+	if err := c.limiter.Wait(ctx); err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("User-Agent", "sourcegraph-rubygems-syncer (sourcegraph.com)")
+
+	b, err := c.do(req)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+type Error struct {
+	path    string
+	code    int
+	message string
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("bad response with status code %d for %s: %s", e.code, e.path, e.message)
+}
+
+func (e *Error) NotFound() bool {
+	return e.code == http.StatusNotFound
+}
+
+func (c *Client) do(req *http.Request) ([]byte, error) {
+	resp, err := c.cli.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	bs, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, &Error{path: req.URL.Path, code: resp.StatusCode, message: string(bs)}
+	}
+
+	return bs, nil
+}

internal/extsvc/types.go

@@ -125,6 +125,7 @@ const (
 	KindJVMPackages     = "JVMPACKAGES"
 	KindPythonPackages  = "PYTHONPACKAGES"
 	KindRustPackages    = "RUSTPACKAGES"
+	KindRubyPackages    = "RUBYPACKAGES"
 	KindNpmPackages     = "NPMPACKAGES"
 	KindPagure          = "PAGURE"
 	KindOther           = "OTHER"
@@ -184,9 +185,12 @@ const (
 	// TypePythonPackages is the (api.ExternalRepoSpec).ServiceType value for Python packages.
 	TypePythonPackages = "pythonPackages"
 
-	// TypeRustPackages is the (api.ExternalRepoSpec).ServiceType value for Python packages.
+	// TypeRustPackages is the (api.ExternalRepoSpec).ServiceType value for Rust packages.
 	TypeRustPackages = "rustPackages"
 
+	// TypeRubyPackages is the (api.ExternalRepoSpec).ServiceType value for Ruby packages.
+	TypeRubyPackages = "rubyPackages"
+
 	// TypeOther is the (api.ExternalRepoSpec).ServiceType value for other projects.
 	TypeOther = "other"
 )
@@ -219,6 +223,8 @@ func KindToType(kind string) string {
 		return TypePythonPackages
 	case KindRustPackages:
 		return TypeRustPackages
+	case KindRubyPackages:
+		return TypeRubyPackages
 	case KindNpmPackages:
 		return TypeNpmPackages
 	case KindGoPackages:
@@ -262,6 +268,8 @@ func TypeToKind(t string) string {
 		return KindPythonPackages
 	case TypeRustPackages:
 		return KindRustPackages
+	case TypeRubyPackages:
+		return KindRubyPackages
 	case TypeGoModules:
 		return KindGoPackages
 	case TypePagure:
@@ -282,6 +290,7 @@ var (
 	goLower     = strings.ToLower(TypeGoModules)
 	pythonLower = strings.ToLower(TypePythonPackages)
 	rustLower   = strings.ToLower(TypeRustPackages)
+	rubyLower   = strings.ToLower(TypeRubyPackages)
 )
 
 // ParseServiceType will return a ServiceType constant after doing a case insensitive match on s.
@@ -316,6 +325,8 @@ func ParseServiceType(s string) (string, bool) {
 		return TypePythonPackages, true
 	case rustLower:
 		return TypeRustPackages, true
+	case rubyLower:
+		return TypeRubyPackages, true
 	case TypePagure:
 		return TypePagure, true
 	case TypeOther:
@@ -355,6 +366,8 @@ func ParseServiceKind(s string) (string, bool) {
 		return KindPythonPackages, true
 	case KindRustPackages:
 		return KindRustPackages, true
+	case KindRubyPackages:
+		return KindRubyPackages, true
 	case KindPagure:
 		return KindPagure, true
 	case KindOther:
@@ -436,6 +449,8 @@ func getConfigPrototype(kind string) (any, error) {
 		return &schema.PythonPackagesConnection{}, nil
 	case KindRustPackages:
 		return &schema.RustPackagesConnection{}, nil
+	case KindRubyPackages:
+		return &schema.RubyPackagesConnection{}, nil
 	case KindOther:
 		return &schema.OtherExternalServiceConnection{}, nil
 	default:
@@ -616,6 +631,12 @@ func GetLimitFromConfig(kind string, config any) (rate.Limit, error) {
 		if c != nil && c.RateLimit != nil {
 			limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour)
 		}
+	case *schema.RubyPackagesConnection:
+		// The rubygems.org API allows 10 rps https://guides.rubygems.org/rubygems-org-rate-limits/
+		limit = rate.Limit(10)
+		if c != nil && c.RateLimit != nil {
+			limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour)
+		}
 	default:
 		return limit, ErrRateLimitUnsupported{codehostKind: kind}
 	}
@@ -736,6 +757,8 @@ func uniqueCodeHostIdentifier(kind string, cfg any) (string, error) {
 		return KindPythonPackages, nil
 	case *schema.RustPackagesConnection:
 		return KindRustPackages, nil
+	case *schema.RubyPackagesConnection:
+		return KindRubyPackages, nil
 	case *schema.PagureConnection:
 		rawURL = c.Url
 	default:

internal/repos/clone_url.go

@@ -98,6 +98,8 @@ func cloneURL(parsed any, logger log.Logger, kind string, repo *types.Repo) (str
 		return string(repo.Name), nil
 	case *schema.RustPackagesConnection:
 		return string(repo.Name), nil
+	case *schema.RubyPackagesConnection:
+		return string(repo.Name), nil
 	case *schema.JVMPackagesConnection:
 		if r, ok := repo.Metadata.(*reposource.MavenMetadata); ok {
 			return r.Module.CloneURL(), nil

internal/repos/ruby_packages.go

@@ -0,0 +1,56 @@
+package repos
+
+import (
+	"context"
+
+	"github.com/sourcegraph/sourcegraph/internal/api"
+	"github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies"
+	"github.com/sourcegraph/sourcegraph/internal/conf/reposource"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems"
+	"github.com/sourcegraph/sourcegraph/internal/httpcli"
+	"github.com/sourcegraph/sourcegraph/internal/jsonc"
+	"github.com/sourcegraph/sourcegraph/internal/types"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+	"github.com/sourcegraph/sourcegraph/schema"
+)
+
+// NewRubyPackagesSource returns a new rubyPackagesSource from the given external service.
+func NewRubyPackagesSource(ctx context.Context, svc *types.ExternalService, cf *httpcli.Factory) (*PackagesSource, error) {
+	rawConfig, err := svc.Config.Decrypt(ctx)
+	if err != nil {
+		return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err)
+	}
+	var c schema.RubyPackagesConnection
+	if err := jsonc.Unmarshal(rawConfig, &c); err != nil {
+		return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err)
+	}
+
+	cli, err := cf.Doer()
+	if err != nil {
+		return nil, err
+	}
+
+	return &PackagesSource{
+		svc:        svc,
+		configDeps: c.Dependencies,
+		scheme:     dependencies.RubyPackagesScheme,
+		src:        &rubyPackagesSource{client: rubygems.NewClient(svc.URN(), cli)},
+	}, nil
+}
+
+type rubyPackagesSource struct {
+	client *rubygems.Client
+}
+
+var _ packagesSource = &rubyPackagesSource{}
+
+func (rubyPackagesSource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
+	return reposource.ParseRubyVersionedPackage(dep)
+}
+
+func (rubyPackagesSource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
+	return reposource.ParseRubyPackageFromName(name)
+}
+func (rubyPackagesSource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {
+	return reposource.ParseRubyPackageFromRepoName(repoName)
+}

internal/repos/sources.go

@@ -76,6 +76,8 @@ func NewSource(ctx context.Context, logger log.Logger, db database.DB, svc *type
 		return NewPythonPackagesSource(ctx, svc, cf)
 	case extsvc.KindRustPackages:
 		return NewRustPackagesSource(ctx, svc, cf)
+	case extsvc.KindRubyPackages:
+		return NewRubyPackagesSource(ctx, svc, cf)
 	case extsvc.KindOther:
 		return NewOtherSource(ctx, svc, cf, logger.Scoped("OtherSource", ""))
 	default:

internal/types/secret.go

@@ -77,6 +77,8 @@ func (e *ExternalService) RedactedConfig(ctx context.Context) (string, error) {
 		}
 	case *schema.RustPackagesConnection:
 		// Nothing to redact
+	case *schema.RubyPackagesConnection:
+		es.redactString(c.Repository, "repository")
 	case *schema.JVMPackagesConnection:
 		if c.Maven != nil {
 			es.redactString(c.Maven.Credentials, "maven", "credentials")
@@ -174,6 +176,9 @@ func (e *ExternalService) UnredactConfig(ctx context.Context, old *ExternalServi
 		}
 	case *schema.RustPackagesConnection:
 		// Nothing to unredact
+	case *schema.RubyPackagesConnection:
+		o := oldCfg.(*schema.RubyPackagesConnection)
+		es.unredactString(c.Repository, o.Repository, "repository")
 	case *schema.JVMPackagesConnection:
 		o := oldCfg.(*schema.JVMPackagesConnection)
 		if c.Maven != nil && o.Maven != nil {

schema/ruby-packages.schema.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "ruby-packages.schema.json#",
+  "title": "RubyPackagesConnection",
+  "description": "Configuration for a connection to Ruby packages",
+  "allowComments": true,
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "repository": {
+      "description": "The URL at which the maven repository can be found.",
+      "type": "string",
+      "default": ["https://rubygems.org/"],
+      "examples": ["https://rubygems.org/", "https://<server name>.jfrog.io/artifactory/api/gems/<repository key>"]
+    },
+    "rateLimit": {
+      "description": "Rate limit applied when making background API requests to the configured Ruby repository APIs.",
+      "title": "RubyRateLimit",
+      "type": "object",
+      "required": ["enabled", "requestsPerHour"],
+      "properties": {
+        "enabled": {
+          "description": "true if rate limiting is enabled.",
+          "type": "boolean",
+          "default": true
+        },
+        "requestsPerHour": {
+          "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.",
+          "type": "number",
+          "default": 8,
+          "minimum": 0
+        }
+      },
+      "default": {
+        "enabled": true,
+        "requestsPerHour": 3600
+      }
+    },
+    "dependencies": {
+      "description": "An array of strings specifying Ruby packages to mirror in Sourcegraph.",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "examples": [["shopify_api@12.0.0"]]
+    }
+  }
+}

schema/schema.go

@@ -652,6 +652,8 @@ type ExperimentalFeatures struct {
 	Ranking *Ranking `json:"ranking,omitempty"`
 	// RateLimitAnonymous description: Configures the hourly rate limits for anonymous calls to the GraphQL API. Setting limit to 0 disables the limiter. This is only relevant if unauthenticated calls to the API are permitted.
 	RateLimitAnonymous int `json:"rateLimitAnonymous,omitempty"`
+	// RubyPackages description: Allow adding Ruby package host connections
+	RubyPackages string `json:"rubyPackages,omitempty"`
 	// RustPackages description: Allow adding Rust package code host connections
 	RustPackages string `json:"rustPackages,omitempty"`
 	// SearchIndexBranches description: A map from repository name to a list of extra revs (branch, ref, tag, commit sha, etc) to index for a repository. We always index the default branch ("HEAD") and revisions in version contexts. This allows specifying additional revisions. Sourcegraph can index up to 64 branches per repository.
@@ -1587,6 +1589,24 @@ type Responders struct {
 	Username string `json:"username,omitempty"`
 }
 
+// RubyPackagesConnection description: Configuration for a connection to Ruby packages
+type RubyPackagesConnection struct {
+	// Dependencies description: An array of strings specifying Ruby packages to mirror in Sourcegraph.
+	Dependencies []string `json:"dependencies,omitempty"`
+	// RateLimit description: Rate limit applied when making background API requests to the configured Ruby repository APIs.
+	RateLimit *RubyRateLimit `json:"rateLimit,omitempty"`
+	// Repository description: The URL at which the maven repository can be found.
+	Repository string `json:"repository,omitempty"`
+}
+
+// RubyRateLimit description: Rate limit applied when making background API requests to the configured Ruby repository APIs.
+type RubyRateLimit struct {
+	// Enabled description: true if rate limiting is enabled.
+	Enabled bool `json:"enabled"`
+	// RequestsPerHour description: Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.
+	RequestsPerHour float64 `json:"requestsPerHour"`
+}
+
 // RustPackagesConnection description: Configuration for a connection to Rust packages
 type RustPackagesConnection struct {
 	// Dependencies description: An array of strings specifying Rust packages to mirror in Sourcegraph.

schema/site.schema.json

@@ -246,6 +246,12 @@
           "enum": ["enabled", "disabled"],
           "default": "disabled"
         },
+        "rubyPackages": {
+          "description": "Allow adding Ruby package host connections",
+          "type": "string",
+          "enum": ["enabled", "disabled"],
+          "default": "disabled"
+        },
         "pagure": {
           "description": "Allow adding Pagure code host connections",
           "type": "string",

schema/stringdata.go

@@ -47,31 +47,24 @@ var GitLabSchemaJSON string
 //go:embed gitolite.schema.json
 var GitoliteSchemaJSON string
 
-// GoModulesSchemaJSON is the content of the file "go-modules.schema.json".
-//
 //go:embed go-modules.schema.json
 var GoModulesSchemaJSON string
 
-// JVMPackagesSchemaJSON is the content of the file "jvm-packages.schema.json".
-//
 //go:embed jvm-packages.schema.json
 var JVMPackagesSchemaJSON string
 
-// NpmPackagesSchemaJSON is the content of the file "npm-packages.schema.json".
-//
 //go:embed npm-packages.schema.json
 var NpmPackagesSchemaJSON string
 
-// PythonPackagesSchemaJSON is the content of the file "python-packages.schema.json".
-//
 //go:embed python-packages.schema.json
 var PythonPackagesSchemaJSON string
 
-// RustPackagesSchemaJSON is the content of the file "python-packages.schema.json".
-//
 //go:embed rust-packages.schema.json
 var RustPackagesSchemaJSON string
 
+//go:embed ruby-packages.schema.json
+var RubyPackagesSchemaJSON string
+
 // OtherExternalServiceSchemaJSON is the content of the file "other_external_service.schema.json".
 //
 //go:embed other_external_service.schema.json