diff --git a/client/web/src/components/externalServices/externalServices.tsx b/client/web/src/components/externalServices/externalServices.tsx
index 00897c8d374..80f60a4a611 100644
--- a/client/web/src/components/externalServices/externalServices.tsx
+++ b/client/web/src/components/externalServices/externalServices.tsx
@@ -9,6 +9,7 @@ import GitLabIcon from 'mdi-react/GitlabIcon'
import LanguageGoIcon from 'mdi-react/LanguageGoIcon'
import LanguageJavaIcon from 'mdi-react/LanguageJavaIcon'
import LanguagePythonIcon from 'mdi-react/LanguagePythonIcon'
+import LanguageRubyIcon from 'mdi-react/LanguageRubyIcon'
import LanguageRustIcon from 'mdi-react/LanguageRustIcon'
import NpmIcon from 'mdi-react/NpmIcon'
@@ -30,6 +31,7 @@ import pagureSchemaJSON from '../../../../../schema/pagure.schema.json'
import perforceSchemaJSON from '../../../../../schema/perforce.schema.json'
import phabricatorSchemaJSON from '../../../../../schema/phabricator.schema.json'
import pythonPackagesJSON from '../../../../../schema/python-packages.schema.json'
+import rubyPackagesSchemaJSON from '../../../../../schema/ruby-packages.schema.json'
import rustPackagesJSON from '../../../../../schema/rust-packages.schema.json'
import { ExternalRepositoryFields, ExternalServiceKind } from '../../graphql-operations'
import { EditorAction } from '../../settings/EditorActionsGroup'
@@ -1399,6 +1401,44 @@ const RUST_PACKAGES = {
editorActions: [],
}
+const RUBY_PACKAGES: AddExternalServiceOptions = {
+ kind: ExternalServiceKind.RUBYPACKAGES,
+ title: 'Ruby Dependencies',
+ icon: LanguageRubyIcon,
+ jsonSchema: rubyPackagesSchemaJSON,
+ defaultDisplayName: 'Ruby Dependencies',
+ defaultConfig: `{
+ "repository": "https://rubygems.org/",
+ "dependencies": ["shopify_api@12.0.0"]
+}`,
+ instructions: (
+
+
+ -
+ The URL https://rubygems.org/ is used if the field
+
"repository" is empty.
+
+ -
+ Use the syntax
"GEM_NAME@GEM_VERSION" to list a dependency for the{' '}
+ "dependencies" field.
+
+ -
+ The field
"repository" is redacted because it can include admin:password{' '}
+ credentials.
+
+ -
+ See the{' '}
+
+ Artifactory documentation for RubyGem repositories
+ {' '}
+ for details on how to configure an internal Artifactory repository.
+
+
+
+ ),
+ editorActions: [],
+}
+
export const codeHostExternalServices: Record = {
github: GITHUB_DOTCOM,
ghe: GITHUB_ENTERPRISE,
@@ -1412,6 +1452,7 @@ export const codeHostExternalServices: Record
git: GENERIC_GIT,
...(window.context?.experimentalFeatures?.pythonPackages === 'enabled' ? { pythonPackages: PYTHON_PACKAGES } : {}),
...(window.context?.experimentalFeatures?.rustPackages === 'enabled' ? { rustPackages: RUST_PACKAGES } : {}),
+ ...(window.context?.experimentalFeatures?.rubyPackages === 'enabled' ? { rubyPackages: RUBY_PACKAGES } : {}),
...(window.context?.experimentalFeatures?.goPackages === 'enabled' ? { goModules: GO_MODULES } : {}),
...(window.context?.experimentalFeatures?.jvmPackages === 'enabled' ? { jvmPackages: JVM_PACKAGES } : {}),
...(window.context?.experimentalFeatures?.npmPackages === 'enabled' ? { npmPackages: NPM_PACKAGES } : {}),
@@ -1446,6 +1487,7 @@ export const defaultExternalServices: Record = {
[ExternalServiceKind.GOMODULES]: Unsupported,
[ExternalServiceKind.PYTHONPACKAGES]: Unsupported,
[ExternalServiceKind.RUSTPACKAGES]: Unsupported,
+ [ExternalServiceKind.RUBYPACKAGES]: Unsupported,
[ExternalServiceKind.JVMPACKAGES]: Unsupported,
[ExternalServiceKind.NPMPACKAGES]: Unsupported,
[ExternalServiceKind.PERFORCE]: Unsupported,
diff --git a/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx b/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx
index a8eee06471f..7fccd70bda7 100644
--- a/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx
+++ b/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx
@@ -27,6 +27,7 @@ const configInstructionLinks: Record = {
[ExternalServiceKind.PHABRICATOR]: 'unsupported',
[ExternalServiceKind.PYTHONPACKAGES]: 'unsupported',
[ExternalServiceKind.RUSTPACKAGES]: 'unsupported',
+ [ExternalServiceKind.RUBYPACKAGES]: 'unsupported',
}
export interface CodeHostSshPublicKeyProps {
diff --git a/client/web/src/site-admin/SiteAdminReportBugPage.tsx b/client/web/src/site-admin/SiteAdminReportBugPage.tsx
index 876c237f3a8..04f023e330c 100644
--- a/client/web/src/site-admin/SiteAdminReportBugPage.tsx
+++ b/client/web/src/site-admin/SiteAdminReportBugPage.tsx
@@ -23,6 +23,7 @@ import pagureSchemaJSON from '../../../../schema/pagure.schema.json'
import perforceSchemaJSON from '../../../../schema/perforce.schema.json'
import phabricatorSchemaJSON from '../../../../schema/phabricator.schema.json'
import pythonPackagesSchemaJSON from '../../../../schema/python-packages.schema.json'
+import rubyPackagesSchemaJSON from '../../../../schema/ruby-packages.schema.json'
import rustPackagesSchemaJSON from '../../../../schema/rust-packages.schema.json'
import settingsSchemaJSON from '../../../../schema/settings.schema.json'
import siteSchemaJSON from '../../../../schema/site.schema.json'
@@ -53,6 +54,7 @@ const externalServices: Record = {
NPMPACKAGES: npmPackagesSchemaJSON,
PYTHONPACKAGES: pythonPackagesSchemaJSON,
RUSTPACKAGES: rustPackagesSchemaJSON,
+ RUBYPACKAGES: rubyPackagesSchemaJSON,
OTHER: otherExternalServiceSchemaJSON,
PERFORCE: perforceSchemaJSON,
PHABRICATOR: phabricatorSchemaJSON,
diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql
index 73a38fb7029..aedae054682 100755
--- a/cmd/frontend/graphqlbackend/schema.graphql
+++ b/cmd/frontend/graphqlbackend/schema.graphql
@@ -2372,6 +2372,7 @@ enum ExternalServiceKind {
PHABRICATOR
PYTHONPACKAGES
RUSTPACKAGES
+ RUBYPACKAGES
}
"""
diff --git a/cmd/gitserver/main.go b/cmd/gitserver/main.go
index 4dd9d022708..8606d4e89f0 100644
--- a/cmd/gitserver/main.go
+++ b/cmd/gitserver/main.go
@@ -46,6 +46,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/extsvc/gomodproxy"
"github.com/sourcegraph/sourcegraph/internal/extsvc/npm"
"github.com/sourcegraph/sourcegraph/internal/extsvc/pypi"
+ "github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/hostname"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
@@ -519,6 +520,14 @@ func getVCSSyncer(
}
cli := crates.NewClient(urn, httpcli.ExternalDoer)
return server.NewRustPackagesSyncer(&c, depsSvc, cli), nil
+ case extsvc.TypeRubyPackages:
+ var c schema.RubyPackagesConnection
+ urn, err := extractOptions(&c)
+ if err != nil {
+ return nil, err
+ }
+ cli := rubygems.NewClient(urn, httpcli.ExternalDoer)
+ return server.NewRubyPackagesSyncer(&c, depsSvc, cli), nil
}
return &server.GitRepoSyncer{}, nil
}
diff --git a/cmd/gitserver/server/vcs_syncer_ruby_packages.go b/cmd/gitserver/server/vcs_syncer_ruby_packages.go
new file mode 100644
index 00000000000..086f14cf5a0
--- /dev/null
+++ b/cmd/gitserver/server/vcs_syncer_ruby_packages.go
@@ -0,0 +1,150 @@
+package server
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "fmt"
+ "io/fs"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/sourcegraph/sourcegraph/internal/api"
+ "github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems"
+
+ "github.com/sourcegraph/log"
+
+ "github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies"
+ "github.com/sourcegraph/sourcegraph/internal/conf/reposource"
+ "github.com/sourcegraph/sourcegraph/internal/unpack"
+ "github.com/sourcegraph/sourcegraph/lib/errors"
+ "github.com/sourcegraph/sourcegraph/schema"
+)
+
+func NewRubyPackagesSyncer(
+ connection *schema.RubyPackagesConnection,
+ svc *dependencies.Service,
+ client *rubygems.Client,
+) VCSSyncer {
+
+ repositoryURL := connection.Repository
+ if repositoryURL == "" {
+ repositoryURL = "https://rubygems.org/"
+ }
+ return &vcsPackagesSyncer{
+ logger: log.Scoped("RubyPackagesSyncer", "sync Ruby packages"),
+ typ: "ruby_packages",
+ scheme: dependencies.RubyPackagesScheme,
+ placeholder: reposource.NewRubyVersionedPackage("shopify_api", "12.0.0"),
+ svc: svc,
+ configDeps: connection.Dependencies,
+ source: &rubyDependencySource{repositoryURL: repositoryURL, client: client},
+ }
+}
+
+type rubyDependencySource struct {
+ repositoryURL string
+ client *rubygems.Client
+}
+
+func (rubyDependencySource) ParseVersionedPackageFromNameAndVersion(name reposource.PackageName, version string) (reposource.VersionedPackage, error) {
+ return reposource.ParseRubyVersionedPackage(string(name) + "@" + version)
+}
+
+func (rubyDependencySource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
+ return reposource.ParseRubyVersionedPackage(dep)
+}
+
+func (rubyDependencySource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
+ return reposource.ParseRubyPackageFromName(name)
+
+}
+func (rubyDependencySource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {
+ return reposource.ParseRubyPackageFromRepoName(repoName)
+}
+
+func (s *rubyDependencySource) Download(ctx context.Context, dir string, dep reposource.VersionedPackage) error {
+ packageURL := fmt.Sprintf("%s/gems/%s-%s.gem", strings.TrimSuffix(s.repositoryURL, "/"), dep.PackageSyntax(), dep.PackageVersion())
+
+ pkg, err := s.client.Get(ctx, packageURL)
+ if err != nil {
+ return errors.Wrapf(err, "error downloading RubyGem with URL '%s'", packageURL)
+ }
+
+ if err = unpackRubyPackage(packageURL, pkg, dir); err != nil {
+ return errors.Wrapf(err, "failed to unzip ruby module from URL %s", packageURL)
+ }
+
+ return nil
+}
+
+func unpackRubyPackage(packageURL string, pkg []byte, workDir string) error {
+ r := bytes.NewReader(pkg)
+ opts := unpack.Opts{
+ SkipInvalid: true,
+ SkipDuplicates: true,
+ Filter: func(path string, file fs.FileInfo) bool {
+ return path == "data.tar.gz" || path == "metadata.gz"
+ },
+ }
+
+ tmpDir, err := os.MkdirTemp("", "rubygems")
+ if err != nil {
+ return errors.Wrap(err, "failed to create a temporary directory")
+ }
+ defer os.RemoveAll(tmpDir)
+
+ if err := unpack.Tar(r, tmpDir, opts); err != nil {
+ return errors.Wrapf(err, "failed to tar downloaded bytes from URL %s", packageURL)
+ }
+
+ err = unpackRubyDataTarGz(packageURL, filepath.Join(tmpDir, "data.tar.gz"), workDir)
+ if err != nil {
+ return err
+ }
+ metadata, err := os.ReadFile(filepath.Join(tmpDir, "metadata.gz"))
+ if err != nil {
+ return err
+ }
+ metadataReader, err := gzip.NewReader(bytes.NewReader(metadata))
+ if err != nil {
+ return err
+ }
+ metadataBytes, err := ioutil.ReadAll(metadataReader)
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(filepath.Join(workDir, "rubygems-metadata.yml"), metadataBytes, 0644)
+}
+
+// unpackRubyDataTarGz unpacks the given `data.tar.gz` from a downloaded RubyGem.
+func unpackRubyDataTarGz(packageURL, path string, workDir string) error {
+ r, err := os.Open(path)
+ if err != nil {
+ return errors.Wrapf(err, "failed to read file from downloaded URL %s", packageURL)
+ }
+ defer r.Close()
+ opts := unpack.Opts{
+ SkipInvalid: true,
+ SkipDuplicates: true,
+ Filter: func(path string, file fs.FileInfo) bool {
+ size := file.Size()
+
+ const sizeLimit = 15 * 1024 * 1024
+ if size >= sizeLimit {
+ return false
+ }
+
+ _, malicious := isPotentiallyMaliciousFilepathInArchive(path, workDir)
+ return !malicious
+ },
+ }
+
+ if err := unpack.Tgz(r, workDir, opts); err != nil {
+ return err
+ }
+
+ return stripSingleOutermostDirectory(workDir)
+}
diff --git a/cmd/gitserver/server/vcs_syncer_rust_packages.go b/cmd/gitserver/server/vcs_syncer_rust_packages.go
index d8c33cdf1ec..aca8ae9005c 100644
--- a/cmd/gitserver/server/vcs_syncer_rust_packages.go
+++ b/cmd/gitserver/server/vcs_syncer_rust_packages.go
@@ -45,7 +45,6 @@ func NewRustPackagesSyncer(
}
}
-// pythonPackagesSyncer implements packagesSource
type rustDependencySource struct {
client *crates.Client
}
diff --git a/doc/admin/external_service/index.md b/doc/admin/external_service/index.md
index 8f3efbb9bf2..c3f70c2bdeb 100644
--- a/doc/admin/external_service/index.md
+++ b/doc/admin/external_service/index.md
@@ -141,6 +141,7 @@ We recognize there are other code hosts including CVS, Azure Dev Ops, SVN, and m
- [Go dependencies](go.md)
- [npm dependencies](npm.md)
- [Python dependencies](python.md)
+ - [Ruby dependencies](ruby.md)
**Users** can configure the following public code hosts:
diff --git a/doc/admin/external_service/ruby-packages.schema.json b/doc/admin/external_service/ruby-packages.schema.json
new file mode 100644
index 00000000000..48a4d26084c
--- /dev/null
+++ b/doc/admin/external_service/ruby-packages.schema.json
@@ -0,0 +1,48 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "ruby-packages.schema.json#",
+ "title": "RubyPackagesConnection",
+ "description": "Configuration for a connection to Ruby packages",
+ "allowComments": true,
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "repository": {
+ "description": "The URL at which the maven repository can be found.",
+ "type": "string",
+ "default": ["https://rubygems.org/"],
+ "examples": ["https://rubygems.org/", "https://.jfrog.io/artifactory/api/gems/"]
+ },
+ "rateLimit": {
+ "description": "Rate limit applied when making background API requests to the configured Ruby repository APIs.",
+ "title": "RubyRateLimit",
+ "type": "object",
+ "required": ["enabled", "requestsPerHour"],
+ "properties": {
+ "enabled": {
+ "description": "true if rate limiting is enabled.",
+ "type": "boolean",
+ "default": true
+ },
+ "requestsPerHour": {
+ "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.",
+ "type": "number",
+ "default": 8,
+ "minimum": 0
+ }
+ },
+ "default": {
+ "enabled": true,
+ "requestsPerHour": 3600
+ }
+ },
+ "dependencies": {
+ "description": "An array of strings specifying Ruby packages to mirror in Sourcegraph.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "examples": [["shopify_api@12.0.0"]]
+ }
+ }
+}
diff --git a/doc/admin/external_service/ruby.md b/doc/admin/external_service/ruby.md
new file mode 100644
index 00000000000..576c387fb3c
--- /dev/null
+++ b/doc/admin/external_service/ruby.md
@@ -0,0 +1,57 @@
+# Ruby dependencies
+
+
+
+Site admins can sync Ruby dependencies from any RubyGems repositories, including rubygems.org or an internal Artifactory, to their Sourcegraph instance so that users can search and navigate the repositories.
+
+To add Ruby dependencies to Sourcegraph you need to setup a Ruby dependencies code host:
+
+1. As *site admin*: go to **Site admin > Global settings** and enable the experimental feature by adding: `{"experimentalFeatures": {"rubyPackages": "enabled"} }`
+1. As *site admin*: go to **Site admin > Manage code hosts**
+1. Select **Ruby Dependencies**.
+1. [Configure the connection](#configuration) by following the instructions above the text field. Additional fields can be added using Cmd/Ctrl+Space for auto-completion. See the [configuration documentation below](#configuration).
+1. Press **Add repositories**.
+
+## Repository syncing
+
+There are two ways to sync Ruby dependency repositories.
+
+* **Indexing** (recommended): run [`scip-ruby`](https://sourcegraph.github.io/scip-ruby/) against your Ruby codebase and upload the generated index to Sourcegraph using the [src-cli](https://github.com/sourcegraph/src-cli) command `src code-intel upload`. This is usually setup to run in a CI pipeline. Sourcegraph automatically synchronizes Ruby dependency repositories based on the dependencies that are discovered by `scip-ruby`.
+* **Code host configuration**: manually list dependencies in the `"dependencies"` section of the [JSON configuration](#configuration) when creating the Ruby dependency code host. This method can be useful to verify that the credentials are picked up correctly without having to upload an index.
+
+## Credentials
+
+The `"repository"` field in the [configuration](#configuration) section is automatically redacted and can optionally include the username and password of an internal [Artifactory RubyGems](https://www.jfrog.com/confluence/display/JFROG/RubyGems+Repositories) repository.
+
+## Rate limiting
+
+By default, requests to the RubyGems repository is 8 request per second.
+
+To manually set the value, add the following to your code host configuration:
+
+```json
+"rateLimit": {
+ "enabled": true,
+ "requestsPerHour": 600.0
+}
+```
+where the `requestsPerHour` field is set based on your requirements.
+
+**Not recommended**: Rate-limiting can be turned off entirely as well.
+This increases the risk of overloading the code host.
+
+```json
+"rateLimit": {
+ "enabled": false
+}
+```
+
+## Configuration
+
+Ruby dependencies code host connections support the following configuration options, which are specified in the JSON editor in the site admin "Manage code hosts" area.
+
+[View page on docs.sourcegraph.com](https://docs.sourcegraph.com/integration/ruby) to see rendered content.
diff --git a/doc/integration/index.md b/doc/integration/index.md
index 2e4856d03ab..f45c816071e 100644
--- a/doc/integration/index.md
+++ b/doc/integration/index.md
@@ -15,6 +15,7 @@ Sourcegraph integrates with your other tools to help you search, navigate, and r
- [npm dependencies](npm.md)
- [Python dependencies](python.md)
- [Go dependencies](go.md)
+ - [Ruby dependencies](ruby.md)
- [Other Git repository hosts](../admin/external_service/other.md)
- [Editor plugins](editor.md): jump to Sourcegraph from your editor
- [Open in Editor](open_in_editor.md): jump to your editor from Sourcegraph
diff --git a/doc/integration/ruby.md b/doc/integration/ruby.md
new file mode 100644
index 00000000000..ad64c7c51ba
--- /dev/null
+++ b/doc/integration/ruby.md
@@ -0,0 +1,29 @@
+# Ruby dependencies integration with Sourcegraph
+
+You can use Sourcegraph with Ruby dependencies from any RubyGems repository, including rubygems.org or an internal Artifactory.
+
+This integration makes it possible to search and navigate through the source code of published Ruby library (for example, [`shopify_api@12.0.0`](https://sourcegraph.com/rubygems/shopify_api@v12.0.0)).
+
+Feature | Supported?
+------- | ----------
+[Repository syncing](#repository-syncing) | ✅
+[Repository permissions](#repository-syncing) | ❌
+[Multiple RubyGems repositories](#multiple-ruby-dependency-code-hosts) | ❌
+
+## Setup
+
+See the "[Ruby dependencies](../admin/external_service/ruby.md)" documentation.
+
+## Repository syncing
+
+Site admins can [add Ruby dependencies to Sourcegraph](../admin/external_service/ruby.md#repository-syncing).
+
+## Repository permissions
+
+⚠ Ruby dependencies are visible by all users of the Sourcegraph instance.
+
+## Multiple Ruby dependencies code hosts
+
+⚠️ It's only possible to create one Ruby dependency code host for each Sourcegraph instance.
+
+See the issue [sourcegraph#32461](https://github.com/sourcegraph/sourcegraph/issues/32461) for more details about this limitation.
diff --git a/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go b/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go
index 3316e6c1c4f..0395acdd859 100644
--- a/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go
+++ b/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go
@@ -59,8 +59,9 @@ var autoIndexingEnabled = conf.CodeIntelAutoIndexingEnabled
var schemeToExternalService = map[string]string{
dependencies.JVMPackagesScheme: extsvc.KindJVMPackages,
dependencies.NpmPackagesScheme: extsvc.KindNpmPackages,
- dependencies.RustPackagesScheme: extsvc.KindRustPackages,
dependencies.PythonPackagesScheme: extsvc.KindPythonPackages,
+ dependencies.RustPackagesScheme: extsvc.KindRustPackages,
+ dependencies.RubyPackagesScheme: extsvc.KindRubyPackages,
}
func (h *dependencySyncSchedulerHandler) Handle(ctx context.Context, logger log.Logger, record workerutil.Record) error {
@@ -244,6 +245,7 @@ func (h *dependencySyncSchedulerHandler) shouldIndexDependencies(ctx context.Con
upload.Indexer == "scip-typescript" ||
upload.Indexer == "lsif-typescript" ||
upload.Indexer == "scip-python" ||
+ upload.Indexer == "scip-ruby" ||
upload.Indexer == "rust-analyzer", nil
}
diff --git a/internal/codeintel/dependencies/consts.go b/internal/codeintel/dependencies/consts.go
index 946d358d143..179ed984912 100644
--- a/internal/codeintel/dependencies/consts.go
+++ b/internal/codeintel/dependencies/consts.go
@@ -8,4 +8,5 @@ const (
GoPackagesScheme = shared.GoPackagesScheme
PythonPackagesScheme = shared.PythonPackagesScheme
RustPackagesScheme = shared.RustPackagesScheme
+ RubyPackagesScheme = shared.RubyPackagesScheme
)
diff --git a/internal/codeintel/dependencies/shared/consts.go b/internal/codeintel/dependencies/shared/consts.go
index 2d02e659cc5..4a94d165080 100644
--- a/internal/codeintel/dependencies/shared/consts.go
+++ b/internal/codeintel/dependencies/shared/consts.go
@@ -6,4 +6,5 @@ const (
NpmPackagesScheme = "npm"
PythonPackagesScheme = "python"
RustPackagesScheme = "rust-analyzer"
+ RubyPackagesScheme = "scip-ruby"
)
diff --git a/internal/conf/reposource/ruby_packages.go b/internal/conf/reposource/ruby_packages.go
new file mode 100644
index 00000000000..8f8e773d0e5
--- /dev/null
+++ b/internal/conf/reposource/ruby_packages.go
@@ -0,0 +1,89 @@
+package reposource
+
+import (
+ "strings"
+
+ "github.com/sourcegraph/sourcegraph/internal/api"
+ "github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+const rubyPackagesPrefix = "rubygems/"
+
+type RubyVersionedPackage struct {
+ Name PackageName
+ Version string
+}
+
+func NewRubyVersionedPackage(name PackageName, version string) *RubyVersionedPackage {
+ return &RubyVersionedPackage{
+ Name: name,
+ Version: version,
+ }
+}
+
+// ParseRubyVersionedPackage parses a string in a '(@version>)?' format into an
+// RubyVersionedPackage.
+func ParseRubyVersionedPackage(dependency string) (*RubyVersionedPackage, error) {
+ var dep RubyVersionedPackage
+ if i := strings.LastIndex(dependency, "@"); i == -1 {
+ dep.Name = PackageName(dependency)
+ } else {
+ dep.Name = PackageName(strings.TrimSpace(dependency[:i]))
+ dep.Version = strings.TrimSpace(dependency[i+1:])
+ }
+ return &dep, nil
+}
+
+func ParseRubyPackageFromName(name PackageName) (*RubyVersionedPackage, error) {
+ return ParseRubyVersionedPackage(string(name))
+}
+
+// ParseRubyPackageFromRepoName is a convenience function to parse a repo name in a
+// 'crates/(@)?' format into a RubyVersionedPackage.
+func ParseRubyPackageFromRepoName(name api.RepoName) (*RubyVersionedPackage, error) {
+ dependency := strings.TrimPrefix(string(name), rubyPackagesPrefix)
+ if len(dependency) == len(name) {
+ return nil, errors.Newf("invalid Ruby dependency repo name, missing %s prefix '%s'", rubyPackagesPrefix, name)
+ }
+ return ParseRubyVersionedPackage(dependency)
+}
+
+func (p *RubyVersionedPackage) Scheme() string {
+ return "scip-ruby"
+}
+
+func (p *RubyVersionedPackage) PackageSyntax() PackageName {
+ return p.Name
+}
+
+func (p *RubyVersionedPackage) VersionedPackageSyntax() string {
+ if p.Version == "" {
+ return string(p.Name)
+ }
+ return string(p.Name) + "@" + p.Version
+}
+
+func (p *RubyVersionedPackage) PackageVersion() string {
+ return p.Version
+}
+
+func (p *RubyVersionedPackage) Description() string { return "" }
+
+func (p *RubyVersionedPackage) RepoName() api.RepoName {
+ return api.RepoName(rubyPackagesPrefix + p.Name)
+}
+
+func (p *RubyVersionedPackage) GitTagFromVersion() string {
+ version := strings.TrimPrefix(p.Version, "v")
+ return "v" + version
+}
+
+func (p *RubyVersionedPackage) Less(other VersionedPackage) bool {
+ o := other.(*RubyVersionedPackage)
+
+ if p.Name == o.Name {
+ return versionGreaterThan(p.Version, o.Version)
+ }
+
+ return p.Name > o.Name
+}
diff --git a/internal/database/external_services.go b/internal/database/external_services.go
index c704b881ccb..9613e3a2700 100644
--- a/internal/database/external_services.go
+++ b/internal/database/external_services.go
@@ -219,6 +219,7 @@ var ExternalServiceKinds = map[string]ExternalServiceKind{
extsvc.KindPhabricator: {CodeHost: true, JSONSchema: schema.PhabricatorSchemaJSON},
extsvc.KindPythonPackages: {CodeHost: true, JSONSchema: schema.PythonPackagesSchemaJSON},
extsvc.KindRustPackages: {CodeHost: true, JSONSchema: schema.RustPackagesSchemaJSON},
+ extsvc.KindRubyPackages: {CodeHost: true, JSONSchema: schema.RubyPackagesSchemaJSON},
}
// ExternalServiceKind describes a kind of external service.
diff --git a/internal/database/repos.go b/internal/database/repos.go
index 0704cdbf128..bc77f48829c 100644
--- a/internal/database/repos.go
+++ b/internal/database/repos.go
@@ -562,6 +562,8 @@ func scanRepo(logger log.Logger, rows *sql.Rows, r *types.Repo) (err error) {
r.Metadata = &struct{}{}
case extsvc.TypeRustPackages:
r.Metadata = &struct{}{}
+ case extsvc.TypeRubyPackages:
+ r.Metadata = &struct{}{}
default:
logger.Warn("unknown service type", log.String("type", typ))
return nil
diff --git a/internal/extsvc/codehost.go b/internal/extsvc/codehost.go
index 754e7b2e896..75762abdf1d 100644
--- a/internal/extsvc/codehost.go
+++ b/internal/extsvc/codehost.go
@@ -15,7 +15,7 @@ type CodeHost struct {
func (c *CodeHost) IsPackageHost() bool {
switch c.ServiceType {
- case TypeNpmPackages, TypeJVMPackages, TypeGoModules, TypePythonPackages, TypeRustPackages:
+ case TypeNpmPackages, TypeJVMPackages, TypeGoModules, TypePythonPackages, TypeRustPackages, TypeRubyPackages:
return true
}
return false
@@ -46,6 +46,9 @@ var (
RustURL = &url.URL{Host: "crates"}
RustPackages = NewCodeHost(RustURL, TypeRustPackages)
+ RubyURL = &url.URL{Host: "rubygems"}
+ RubyPackages = NewCodeHost(RubyURL, TypeRubyPackages)
+
PublicCodeHosts = []*CodeHost{
GitHubDotCom,
GitLabDotCom,
@@ -54,6 +57,7 @@ var (
GoModules,
PythonPackages,
RustPackages,
+ RubyPackages,
}
)
diff --git a/internal/extsvc/crates/client.go b/internal/extsvc/crates/client.go
index 991046ced56..70814a545d9 100644
--- a/internal/extsvc/crates/client.go
+++ b/internal/extsvc/crates/client.go
@@ -17,11 +17,6 @@ type Client struct {
limiter *ratelimit.InstrumentedLimiter
}
-type RustFile struct {
- Name string
- URL string
-}
-
func NewClient(urn string, cli httpcli.Doer) *Client {
return &Client{
cli: cli,
diff --git a/internal/extsvc/rubygems/client.go b/internal/extsvc/rubygems/client.go
new file mode 100644
index 00000000000..0de670e4f28
--- /dev/null
+++ b/internal/extsvc/rubygems/client.go
@@ -0,0 +1,77 @@
+package rubygems
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+
+ "github.com/sourcegraph/sourcegraph/internal/httpcli"
+ "github.com/sourcegraph/sourcegraph/internal/ratelimit"
+)
+
+type Client struct {
+ cli httpcli.Doer
+
+ // Self-imposed rate-limiter.
+ limiter *ratelimit.InstrumentedLimiter
+}
+
+func NewClient(urn string, cli httpcli.Doer) *Client {
+ return &Client{
+ cli: cli,
+ limiter: ratelimit.DefaultRegistry.Get(urn),
+ }
+}
+
+func (c *Client) Get(ctx context.Context, url string) ([]byte, error) {
+ if err := c.limiter.Wait(ctx); err != nil {
+ return nil, err
+ }
+
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Add("User-Agent", "sourcegraph-rubygems-syncer (sourcegraph.com)")
+
+ b, err := c.do(req)
+ if err != nil {
+ return nil, err
+ }
+ return b, nil
+}
+
+type Error struct {
+ path string
+ code int
+ message string
+}
+
+func (e *Error) Error() string {
+ return fmt.Sprintf("bad response with status code %d for %s: %s", e.code, e.path, e.message)
+}
+
+func (e *Error) NotFound() bool {
+ return e.code == http.StatusNotFound
+}
+
+func (c *Client) do(req *http.Request) ([]byte, error) {
+ resp, err := c.cli.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ defer resp.Body.Close()
+
+ bs, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, &Error{path: req.URL.Path, code: resp.StatusCode, message: string(bs)}
+ }
+
+ return bs, nil
+}
diff --git a/internal/extsvc/types.go b/internal/extsvc/types.go
index 514273237e5..667fa971859 100644
--- a/internal/extsvc/types.go
+++ b/internal/extsvc/types.go
@@ -125,6 +125,7 @@ const (
KindJVMPackages = "JVMPACKAGES"
KindPythonPackages = "PYTHONPACKAGES"
KindRustPackages = "RUSTPACKAGES"
+ KindRubyPackages = "RUBYPACKAGES"
KindNpmPackages = "NPMPACKAGES"
KindPagure = "PAGURE"
KindOther = "OTHER"
@@ -184,9 +185,12 @@ const (
// TypePythonPackages is the (api.ExternalRepoSpec).ServiceType value for Python packages.
TypePythonPackages = "pythonPackages"
- // TypeRustPackages is the (api.ExternalRepoSpec).ServiceType value for Python packages.
+ // TypeRustPackages is the (api.ExternalRepoSpec).ServiceType value for Rust packages.
TypeRustPackages = "rustPackages"
+ // TypeRubyPackages is the (api.ExternalRepoSpec).ServiceType value for Ruby packages.
+ TypeRubyPackages = "rubyPackages"
+
// TypeOther is the (api.ExternalRepoSpec).ServiceType value for other projects.
TypeOther = "other"
)
@@ -219,6 +223,8 @@ func KindToType(kind string) string {
return TypePythonPackages
case KindRustPackages:
return TypeRustPackages
+ case KindRubyPackages:
+ return TypeRubyPackages
case KindNpmPackages:
return TypeNpmPackages
case KindGoPackages:
@@ -262,6 +268,8 @@ func TypeToKind(t string) string {
return KindPythonPackages
case TypeRustPackages:
return KindRustPackages
+ case TypeRubyPackages:
+ return KindRubyPackages
case TypeGoModules:
return KindGoPackages
case TypePagure:
@@ -282,6 +290,7 @@ var (
goLower = strings.ToLower(TypeGoModules)
pythonLower = strings.ToLower(TypePythonPackages)
rustLower = strings.ToLower(TypeRustPackages)
+ rubyLower = strings.ToLower(TypeRubyPackages)
)
// ParseServiceType will return a ServiceType constant after doing a case insensitive match on s.
@@ -316,6 +325,8 @@ func ParseServiceType(s string) (string, bool) {
return TypePythonPackages, true
case rustLower:
return TypeRustPackages, true
+ case rubyLower:
+ return TypeRubyPackages, true
case TypePagure:
return TypePagure, true
case TypeOther:
@@ -355,6 +366,8 @@ func ParseServiceKind(s string) (string, bool) {
return KindPythonPackages, true
case KindRustPackages:
return KindRustPackages, true
+ case KindRubyPackages:
+ return KindRubyPackages, true
case KindPagure:
return KindPagure, true
case KindOther:
@@ -436,6 +449,8 @@ func getConfigPrototype(kind string) (any, error) {
return &schema.PythonPackagesConnection{}, nil
case KindRustPackages:
return &schema.RustPackagesConnection{}, nil
+ case KindRubyPackages:
+ return &schema.RubyPackagesConnection{}, nil
case KindOther:
return &schema.OtherExternalServiceConnection{}, nil
default:
@@ -616,6 +631,12 @@ func GetLimitFromConfig(kind string, config any) (rate.Limit, error) {
if c != nil && c.RateLimit != nil {
limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour)
}
+ case *schema.RubyPackagesConnection:
+ // The rubygems.org API allows 10 rps https://guides.rubygems.org/rubygems-org-rate-limits/
+ limit = rate.Limit(10)
+ if c != nil && c.RateLimit != nil {
+ limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour)
+ }
default:
return limit, ErrRateLimitUnsupported{codehostKind: kind}
}
@@ -736,6 +757,8 @@ func uniqueCodeHostIdentifier(kind string, cfg any) (string, error) {
return KindPythonPackages, nil
case *schema.RustPackagesConnection:
return KindRustPackages, nil
+ case *schema.RubyPackagesConnection:
+ return KindRubyPackages, nil
case *schema.PagureConnection:
rawURL = c.Url
default:
diff --git a/internal/repos/clone_url.go b/internal/repos/clone_url.go
index 12c11977bb2..13fdf218643 100644
--- a/internal/repos/clone_url.go
+++ b/internal/repos/clone_url.go
@@ -98,6 +98,8 @@ func cloneURL(parsed any, logger log.Logger, kind string, repo *types.Repo) (str
return string(repo.Name), nil
case *schema.RustPackagesConnection:
return string(repo.Name), nil
+ case *schema.RubyPackagesConnection:
+ return string(repo.Name), nil
case *schema.JVMPackagesConnection:
if r, ok := repo.Metadata.(*reposource.MavenMetadata); ok {
return r.Module.CloneURL(), nil
diff --git a/internal/repos/ruby_packages.go b/internal/repos/ruby_packages.go
new file mode 100644
index 00000000000..f09dfa85677
--- /dev/null
+++ b/internal/repos/ruby_packages.go
@@ -0,0 +1,56 @@
+package repos
+
+import (
+ "context"
+
+ "github.com/sourcegraph/sourcegraph/internal/api"
+ "github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies"
+ "github.com/sourcegraph/sourcegraph/internal/conf/reposource"
+ "github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems"
+ "github.com/sourcegraph/sourcegraph/internal/httpcli"
+ "github.com/sourcegraph/sourcegraph/internal/jsonc"
+ "github.com/sourcegraph/sourcegraph/internal/types"
+ "github.com/sourcegraph/sourcegraph/lib/errors"
+ "github.com/sourcegraph/sourcegraph/schema"
+)
+
+// NewRubyPackagesSource returns a new rubyPackagesSource from the given external service.
+func NewRubyPackagesSource(ctx context.Context, svc *types.ExternalService, cf *httpcli.Factory) (*PackagesSource, error) {
+ rawConfig, err := svc.Config.Decrypt(ctx)
+ if err != nil {
+ return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err)
+ }
+ var c schema.RubyPackagesConnection
+ if err := jsonc.Unmarshal(rawConfig, &c); err != nil {
+ return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err)
+ }
+
+ cli, err := cf.Doer()
+ if err != nil {
+ return nil, err
+ }
+
+ return &PackagesSource{
+ svc: svc,
+ configDeps: c.Dependencies,
+ scheme: dependencies.RubyPackagesScheme,
+ src: &rubyPackagesSource{client: rubygems.NewClient(svc.URN(), cli)},
+ }, nil
+}
+
+type rubyPackagesSource struct {
+ client *rubygems.Client
+}
+
+var _ packagesSource = &rubyPackagesSource{}
+
+func (rubyPackagesSource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) {
+ return reposource.ParseRubyVersionedPackage(dep)
+}
+
+func (rubyPackagesSource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) {
+ return reposource.ParseRubyPackageFromName(name)
+}
+func (rubyPackagesSource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) {
+ return reposource.ParseRubyPackageFromRepoName(repoName)
+}
diff --git a/internal/repos/sources.go b/internal/repos/sources.go
index ff3fd8fd3ac..e6b38ca99c8 100644
--- a/internal/repos/sources.go
+++ b/internal/repos/sources.go
@@ -76,6 +76,8 @@ func NewSource(ctx context.Context, logger log.Logger, db database.DB, svc *type
return NewPythonPackagesSource(ctx, svc, cf)
case extsvc.KindRustPackages:
return NewRustPackagesSource(ctx, svc, cf)
+ case extsvc.KindRubyPackages:
+ return NewRubyPackagesSource(ctx, svc, cf)
case extsvc.KindOther:
return NewOtherSource(ctx, svc, cf, logger.Scoped("OtherSource", ""))
default:
diff --git a/internal/types/secret.go b/internal/types/secret.go
index 3125b4c9c9c..8c6c22da54f 100644
--- a/internal/types/secret.go
+++ b/internal/types/secret.go
@@ -77,6 +77,8 @@ func (e *ExternalService) RedactedConfig(ctx context.Context) (string, error) {
}
case *schema.RustPackagesConnection:
// Nothing to redact
+ case *schema.RubyPackagesConnection:
+ es.redactString(c.Repository, "repository")
case *schema.JVMPackagesConnection:
if c.Maven != nil {
es.redactString(c.Maven.Credentials, "maven", "credentials")
@@ -174,6 +176,9 @@ func (e *ExternalService) UnredactConfig(ctx context.Context, old *ExternalServi
}
case *schema.RustPackagesConnection:
// Nothing to unredact
+ case *schema.RubyPackagesConnection:
+ o := oldCfg.(*schema.RubyPackagesConnection)
+ es.unredactString(c.Repository, o.Repository, "repository")
case *schema.JVMPackagesConnection:
o := oldCfg.(*schema.JVMPackagesConnection)
if c.Maven != nil && o.Maven != nil {
diff --git a/schema/ruby-packages.schema.json b/schema/ruby-packages.schema.json
new file mode 100644
index 00000000000..48a4d26084c
--- /dev/null
+++ b/schema/ruby-packages.schema.json
@@ -0,0 +1,48 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "ruby-packages.schema.json#",
+ "title": "RubyPackagesConnection",
+ "description": "Configuration for a connection to Ruby packages",
+ "allowComments": true,
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "repository": {
+ "description": "The URL at which the maven repository can be found.",
+ "type": "string",
+ "default": ["https://rubygems.org/"],
+ "examples": ["https://rubygems.org/", "https://.jfrog.io/artifactory/api/gems/"]
+ },
+ "rateLimit": {
+ "description": "Rate limit applied when making background API requests to the configured Ruby repository APIs.",
+ "title": "RubyRateLimit",
+ "type": "object",
+ "required": ["enabled", "requestsPerHour"],
+ "properties": {
+ "enabled": {
+ "description": "true if rate limiting is enabled.",
+ "type": "boolean",
+ "default": true
+ },
+ "requestsPerHour": {
+ "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.",
+ "type": "number",
+ "default": 8,
+ "minimum": 0
+ }
+ },
+ "default": {
+ "enabled": true,
+ "requestsPerHour": 3600
+ }
+ },
+ "dependencies": {
+ "description": "An array of strings specifying Ruby packages to mirror in Sourcegraph.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "examples": [["shopify_api@12.0.0"]]
+ }
+ }
+}
diff --git a/schema/schema.go b/schema/schema.go
index 5a8186b858a..61f0c79dd55 100644
--- a/schema/schema.go
+++ b/schema/schema.go
@@ -652,6 +652,8 @@ type ExperimentalFeatures struct {
Ranking *Ranking `json:"ranking,omitempty"`
// RateLimitAnonymous description: Configures the hourly rate limits for anonymous calls to the GraphQL API. Setting limit to 0 disables the limiter. This is only relevant if unauthenticated calls to the API are permitted.
RateLimitAnonymous int `json:"rateLimitAnonymous,omitempty"`
+ // RubyPackages description: Allow adding Ruby package host connections
+ RubyPackages string `json:"rubyPackages,omitempty"`
// RustPackages description: Allow adding Rust package code host connections
RustPackages string `json:"rustPackages,omitempty"`
// SearchIndexBranches description: A map from repository name to a list of extra revs (branch, ref, tag, commit sha, etc) to index for a repository. We always index the default branch ("HEAD") and revisions in version contexts. This allows specifying additional revisions. Sourcegraph can index up to 64 branches per repository.
@@ -1587,6 +1589,24 @@ type Responders struct {
Username string `json:"username,omitempty"`
}
+// RubyPackagesConnection description: Configuration for a connection to Ruby packages
+type RubyPackagesConnection struct {
+ // Dependencies description: An array of strings specifying Ruby packages to mirror in Sourcegraph.
+ Dependencies []string `json:"dependencies,omitempty"`
+ // RateLimit description: Rate limit applied when making background API requests to the configured Ruby repository APIs.
+ RateLimit *RubyRateLimit `json:"rateLimit,omitempty"`
+ // Repository description: The URL at which the maven repository can be found.
+ Repository string `json:"repository,omitempty"`
+}
+
+// RubyRateLimit description: Rate limit applied when making background API requests to the configured Ruby repository APIs.
+type RubyRateLimit struct {
+ // Enabled description: true if rate limiting is enabled.
+ Enabled bool `json:"enabled"`
+ // RequestsPerHour description: Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.
+ RequestsPerHour float64 `json:"requestsPerHour"`
+}
+
// RustPackagesConnection description: Configuration for a connection to Rust packages
type RustPackagesConnection struct {
// Dependencies description: An array of strings specifying Rust packages to mirror in Sourcegraph.
diff --git a/schema/site.schema.json b/schema/site.schema.json
index 830840a2ea4..db17eadedf5 100644
--- a/schema/site.schema.json
+++ b/schema/site.schema.json
@@ -246,6 +246,12 @@
"enum": ["enabled", "disabled"],
"default": "disabled"
},
+ "rubyPackages": {
+ "description": "Allow adding Ruby package host connections",
+ "type": "string",
+ "enum": ["enabled", "disabled"],
+ "default": "disabled"
+ },
"pagure": {
"description": "Allow adding Pagure code host connections",
"type": "string",
diff --git a/schema/stringdata.go b/schema/stringdata.go
index b69b6d0d34b..bb8c84a0f65 100644
--- a/schema/stringdata.go
+++ b/schema/stringdata.go
@@ -47,31 +47,24 @@ var GitLabSchemaJSON string
//go:embed gitolite.schema.json
var GitoliteSchemaJSON string
-// GoModulesSchemaJSON is the content of the file "go-modules.schema.json".
-//
//go:embed go-modules.schema.json
var GoModulesSchemaJSON string
-// JVMPackagesSchemaJSON is the content of the file "jvm-packages.schema.json".
-//
//go:embed jvm-packages.schema.json
var JVMPackagesSchemaJSON string
-// NpmPackagesSchemaJSON is the content of the file "npm-packages.schema.json".
-//
//go:embed npm-packages.schema.json
var NpmPackagesSchemaJSON string
-// PythonPackagesSchemaJSON is the content of the file "python-packages.schema.json".
-//
//go:embed python-packages.schema.json
var PythonPackagesSchemaJSON string
-// RustPackagesSchemaJSON is the content of the file "python-packages.schema.json".
-//
//go:embed rust-packages.schema.json
var RustPackagesSchemaJSON string
+//go:embed ruby-packages.schema.json
+var RubyPackagesSchemaJSON string
+
// OtherExternalServiceSchemaJSON is the content of the file "other_external_service.schema.json".
//
//go:embed other_external_service.schema.json