diff --git a/client/web/src/components/externalServices/externalServices.tsx b/client/web/src/components/externalServices/externalServices.tsx index 00897c8d374..80f60a4a611 100644 --- a/client/web/src/components/externalServices/externalServices.tsx +++ b/client/web/src/components/externalServices/externalServices.tsx @@ -9,6 +9,7 @@ import GitLabIcon from 'mdi-react/GitlabIcon' import LanguageGoIcon from 'mdi-react/LanguageGoIcon' import LanguageJavaIcon from 'mdi-react/LanguageJavaIcon' import LanguagePythonIcon from 'mdi-react/LanguagePythonIcon' +import LanguageRubyIcon from 'mdi-react/LanguageRubyIcon' import LanguageRustIcon from 'mdi-react/LanguageRustIcon' import NpmIcon from 'mdi-react/NpmIcon' @@ -30,6 +31,7 @@ import pagureSchemaJSON from '../../../../../schema/pagure.schema.json' import perforceSchemaJSON from '../../../../../schema/perforce.schema.json' import phabricatorSchemaJSON from '../../../../../schema/phabricator.schema.json' import pythonPackagesJSON from '../../../../../schema/python-packages.schema.json' +import rubyPackagesSchemaJSON from '../../../../../schema/ruby-packages.schema.json' import rustPackagesJSON from '../../../../../schema/rust-packages.schema.json' import { ExternalRepositoryFields, ExternalServiceKind } from '../../graphql-operations' import { EditorAction } from '../../settings/EditorActionsGroup' @@ -1399,6 +1401,44 @@ const RUST_PACKAGES = { editorActions: [], } +const RUBY_PACKAGES: AddExternalServiceOptions = { + kind: ExternalServiceKind.RUBYPACKAGES, + title: 'Ruby Dependencies', + icon: LanguageRubyIcon, + jsonSchema: rubyPackagesSchemaJSON, + defaultDisplayName: 'Ruby Dependencies', + defaultConfig: `{ + "repository": "https://rubygems.org/", + "dependencies": ["shopify_api@12.0.0"] +}`, + instructions: ( +
+
    +
  1. + The URL https://rubygems.org/ is used if the field + "repository" is empty. +
  2. +
  3. + Use the syntax "GEM_NAME@GEM_VERSION" to list a dependency for the{' '} + "dependencies" field. +
  4. +
  5. + The field "repository" is redacted because it can include admin:password{' '} + credentials. +
  6. +
  7. + See the{' '} + + Artifactory documentation for RubyGem repositories + {' '} + for details on how to configure an internal Artifactory repository. +
  8. +
+
+ ), + editorActions: [], +} + export const codeHostExternalServices: Record = { github: GITHUB_DOTCOM, ghe: GITHUB_ENTERPRISE, @@ -1412,6 +1452,7 @@ export const codeHostExternalServices: Record git: GENERIC_GIT, ...(window.context?.experimentalFeatures?.pythonPackages === 'enabled' ? { pythonPackages: PYTHON_PACKAGES } : {}), ...(window.context?.experimentalFeatures?.rustPackages === 'enabled' ? { rustPackages: RUST_PACKAGES } : {}), + ...(window.context?.experimentalFeatures?.rubyPackages === 'enabled' ? { rubyPackages: RUBY_PACKAGES } : {}), ...(window.context?.experimentalFeatures?.goPackages === 'enabled' ? { goModules: GO_MODULES } : {}), ...(window.context?.experimentalFeatures?.jvmPackages === 'enabled' ? { jvmPackages: JVM_PACKAGES } : {}), ...(window.context?.experimentalFeatures?.npmPackages === 'enabled' ? { npmPackages: NPM_PACKAGES } : {}), @@ -1446,6 +1487,7 @@ export const defaultExternalServices: Record = { [ExternalServiceKind.GOMODULES]: Unsupported, [ExternalServiceKind.PYTHONPACKAGES]: Unsupported, [ExternalServiceKind.RUSTPACKAGES]: Unsupported, + [ExternalServiceKind.RUBYPACKAGES]: Unsupported, [ExternalServiceKind.JVMPACKAGES]: Unsupported, [ExternalServiceKind.NPMPACKAGES]: Unsupported, [ExternalServiceKind.PERFORCE]: Unsupported, diff --git a/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx b/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx index a8eee06471f..7fccd70bda7 100644 --- a/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx +++ b/client/web/src/enterprise/batches/settings/CodeHostSshPublicKey.tsx @@ -27,6 +27,7 @@ const configInstructionLinks: Record = { [ExternalServiceKind.PHABRICATOR]: 'unsupported', [ExternalServiceKind.PYTHONPACKAGES]: 'unsupported', [ExternalServiceKind.RUSTPACKAGES]: 'unsupported', + [ExternalServiceKind.RUBYPACKAGES]: 'unsupported', } export interface CodeHostSshPublicKeyProps { diff --git a/client/web/src/site-admin/SiteAdminReportBugPage.tsx b/client/web/src/site-admin/SiteAdminReportBugPage.tsx index 876c237f3a8..04f023e330c 100644 --- a/client/web/src/site-admin/SiteAdminReportBugPage.tsx +++ b/client/web/src/site-admin/SiteAdminReportBugPage.tsx @@ -23,6 +23,7 @@ import pagureSchemaJSON from '../../../../schema/pagure.schema.json' import perforceSchemaJSON from '../../../../schema/perforce.schema.json' import phabricatorSchemaJSON from '../../../../schema/phabricator.schema.json' import pythonPackagesSchemaJSON from '../../../../schema/python-packages.schema.json' +import rubyPackagesSchemaJSON from '../../../../schema/ruby-packages.schema.json' import rustPackagesSchemaJSON from '../../../../schema/rust-packages.schema.json' import settingsSchemaJSON from '../../../../schema/settings.schema.json' import siteSchemaJSON from '../../../../schema/site.schema.json' @@ -53,6 +54,7 @@ const externalServices: Record = { NPMPACKAGES: npmPackagesSchemaJSON, PYTHONPACKAGES: pythonPackagesSchemaJSON, RUSTPACKAGES: rustPackagesSchemaJSON, + RUBYPACKAGES: rubyPackagesSchemaJSON, OTHER: otherExternalServiceSchemaJSON, PERFORCE: perforceSchemaJSON, PHABRICATOR: phabricatorSchemaJSON, diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql index 73a38fb7029..aedae054682 100755 --- a/cmd/frontend/graphqlbackend/schema.graphql +++ b/cmd/frontend/graphqlbackend/schema.graphql @@ -2372,6 +2372,7 @@ enum ExternalServiceKind { PHABRICATOR PYTHONPACKAGES RUSTPACKAGES + RUBYPACKAGES } """ diff --git a/cmd/gitserver/main.go b/cmd/gitserver/main.go index 4dd9d022708..8606d4e89f0 100644 --- a/cmd/gitserver/main.go +++ b/cmd/gitserver/main.go @@ -46,6 +46,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/extsvc/gomodproxy" "github.com/sourcegraph/sourcegraph/internal/extsvc/npm" "github.com/sourcegraph/sourcegraph/internal/extsvc/pypi" + "github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems" "github.com/sourcegraph/sourcegraph/internal/goroutine" "github.com/sourcegraph/sourcegraph/internal/hostname" "github.com/sourcegraph/sourcegraph/internal/httpcli" @@ -519,6 +520,14 @@ func getVCSSyncer( } cli := crates.NewClient(urn, httpcli.ExternalDoer) return server.NewRustPackagesSyncer(&c, depsSvc, cli), nil + case extsvc.TypeRubyPackages: + var c schema.RubyPackagesConnection + urn, err := extractOptions(&c) + if err != nil { + return nil, err + } + cli := rubygems.NewClient(urn, httpcli.ExternalDoer) + return server.NewRubyPackagesSyncer(&c, depsSvc, cli), nil } return &server.GitRepoSyncer{}, nil } diff --git a/cmd/gitserver/server/vcs_syncer_ruby_packages.go b/cmd/gitserver/server/vcs_syncer_ruby_packages.go new file mode 100644 index 00000000000..086f14cf5a0 --- /dev/null +++ b/cmd/gitserver/server/vcs_syncer_ruby_packages.go @@ -0,0 +1,150 @@ +package server + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems" + + "github.com/sourcegraph/log" + + "github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies" + "github.com/sourcegraph/sourcegraph/internal/conf/reposource" + "github.com/sourcegraph/sourcegraph/internal/unpack" + "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/schema" +) + +func NewRubyPackagesSyncer( + connection *schema.RubyPackagesConnection, + svc *dependencies.Service, + client *rubygems.Client, +) VCSSyncer { + + repositoryURL := connection.Repository + if repositoryURL == "" { + repositoryURL = "https://rubygems.org/" + } + return &vcsPackagesSyncer{ + logger: log.Scoped("RubyPackagesSyncer", "sync Ruby packages"), + typ: "ruby_packages", + scheme: dependencies.RubyPackagesScheme, + placeholder: reposource.NewRubyVersionedPackage("shopify_api", "12.0.0"), + svc: svc, + configDeps: connection.Dependencies, + source: &rubyDependencySource{repositoryURL: repositoryURL, client: client}, + } +} + +type rubyDependencySource struct { + repositoryURL string + client *rubygems.Client +} + +func (rubyDependencySource) ParseVersionedPackageFromNameAndVersion(name reposource.PackageName, version string) (reposource.VersionedPackage, error) { + return reposource.ParseRubyVersionedPackage(string(name) + "@" + version) +} + +func (rubyDependencySource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) { + return reposource.ParseRubyVersionedPackage(dep) +} + +func (rubyDependencySource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) { + return reposource.ParseRubyPackageFromName(name) + +} +func (rubyDependencySource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) { + return reposource.ParseRubyPackageFromRepoName(repoName) +} + +func (s *rubyDependencySource) Download(ctx context.Context, dir string, dep reposource.VersionedPackage) error { + packageURL := fmt.Sprintf("%s/gems/%s-%s.gem", strings.TrimSuffix(s.repositoryURL, "/"), dep.PackageSyntax(), dep.PackageVersion()) + + pkg, err := s.client.Get(ctx, packageURL) + if err != nil { + return errors.Wrapf(err, "error downloading RubyGem with URL '%s'", packageURL) + } + + if err = unpackRubyPackage(packageURL, pkg, dir); err != nil { + return errors.Wrapf(err, "failed to unzip ruby module from URL %s", packageURL) + } + + return nil +} + +func unpackRubyPackage(packageURL string, pkg []byte, workDir string) error { + r := bytes.NewReader(pkg) + opts := unpack.Opts{ + SkipInvalid: true, + SkipDuplicates: true, + Filter: func(path string, file fs.FileInfo) bool { + return path == "data.tar.gz" || path == "metadata.gz" + }, + } + + tmpDir, err := os.MkdirTemp("", "rubygems") + if err != nil { + return errors.Wrap(err, "failed to create a temporary directory") + } + defer os.RemoveAll(tmpDir) + + if err := unpack.Tar(r, tmpDir, opts); err != nil { + return errors.Wrapf(err, "failed to tar downloaded bytes from URL %s", packageURL) + } + + err = unpackRubyDataTarGz(packageURL, filepath.Join(tmpDir, "data.tar.gz"), workDir) + if err != nil { + return err + } + metadata, err := os.ReadFile(filepath.Join(tmpDir, "metadata.gz")) + if err != nil { + return err + } + metadataReader, err := gzip.NewReader(bytes.NewReader(metadata)) + if err != nil { + return err + } + metadataBytes, err := ioutil.ReadAll(metadataReader) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(workDir, "rubygems-metadata.yml"), metadataBytes, 0644) +} + +// unpackRubyDataTarGz unpacks the given `data.tar.gz` from a downloaded RubyGem. +func unpackRubyDataTarGz(packageURL, path string, workDir string) error { + r, err := os.Open(path) + if err != nil { + return errors.Wrapf(err, "failed to read file from downloaded URL %s", packageURL) + } + defer r.Close() + opts := unpack.Opts{ + SkipInvalid: true, + SkipDuplicates: true, + Filter: func(path string, file fs.FileInfo) bool { + size := file.Size() + + const sizeLimit = 15 * 1024 * 1024 + if size >= sizeLimit { + return false + } + + _, malicious := isPotentiallyMaliciousFilepathInArchive(path, workDir) + return !malicious + }, + } + + if err := unpack.Tgz(r, workDir, opts); err != nil { + return err + } + + return stripSingleOutermostDirectory(workDir) +} diff --git a/cmd/gitserver/server/vcs_syncer_rust_packages.go b/cmd/gitserver/server/vcs_syncer_rust_packages.go index d8c33cdf1ec..aca8ae9005c 100644 --- a/cmd/gitserver/server/vcs_syncer_rust_packages.go +++ b/cmd/gitserver/server/vcs_syncer_rust_packages.go @@ -45,7 +45,6 @@ func NewRustPackagesSyncer( } } -// pythonPackagesSyncer implements packagesSource type rustDependencySource struct { client *crates.Client } diff --git a/doc/admin/external_service/index.md b/doc/admin/external_service/index.md index 8f3efbb9bf2..c3f70c2bdeb 100644 --- a/doc/admin/external_service/index.md +++ b/doc/admin/external_service/index.md @@ -141,6 +141,7 @@ We recognize there are other code hosts including CVS, Azure Dev Ops, SVN, and m - [Go dependencies](go.md) - [npm dependencies](npm.md) - [Python dependencies](python.md) + - [Ruby dependencies](ruby.md) **Users** can configure the following public code hosts: diff --git a/doc/admin/external_service/ruby-packages.schema.json b/doc/admin/external_service/ruby-packages.schema.json new file mode 100644 index 00000000000..48a4d26084c --- /dev/null +++ b/doc/admin/external_service/ruby-packages.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "ruby-packages.schema.json#", + "title": "RubyPackagesConnection", + "description": "Configuration for a connection to Ruby packages", + "allowComments": true, + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The URL at which the maven repository can be found.", + "type": "string", + "default": ["https://rubygems.org/"], + "examples": ["https://rubygems.org/", "https://.jfrog.io/artifactory/api/gems/"] + }, + "rateLimit": { + "description": "Rate limit applied when making background API requests to the configured Ruby repository APIs.", + "title": "RubyRateLimit", + "type": "object", + "required": ["enabled", "requestsPerHour"], + "properties": { + "enabled": { + "description": "true if rate limiting is enabled.", + "type": "boolean", + "default": true + }, + "requestsPerHour": { + "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.", + "type": "number", + "default": 8, + "minimum": 0 + } + }, + "default": { + "enabled": true, + "requestsPerHour": 3600 + } + }, + "dependencies": { + "description": "An array of strings specifying Ruby packages to mirror in Sourcegraph.", + "type": "array", + "items": { + "type": "string" + }, + "examples": [["shopify_api@12.0.0"]] + } + } +} diff --git a/doc/admin/external_service/ruby.md b/doc/admin/external_service/ruby.md new file mode 100644 index 00000000000..576c387fb3c --- /dev/null +++ b/doc/admin/external_service/ruby.md @@ -0,0 +1,57 @@ +# Ruby dependencies + + + +Site admins can sync Ruby dependencies from any RubyGems repositories, including rubygems.org or an internal Artifactory, to their Sourcegraph instance so that users can search and navigate the repositories. + +To add Ruby dependencies to Sourcegraph you need to setup a Ruby dependencies code host: + +1. As *site admin*: go to **Site admin > Global settings** and enable the experimental feature by adding: `{"experimentalFeatures": {"rubyPackages": "enabled"} }` +1. As *site admin*: go to **Site admin > Manage code hosts** +1. Select **Ruby Dependencies**. +1. [Configure the connection](#configuration) by following the instructions above the text field. Additional fields can be added using Cmd/Ctrl+Space for auto-completion. See the [configuration documentation below](#configuration). +1. Press **Add repositories**. + +## Repository syncing + +There are two ways to sync Ruby dependency repositories. + +* **Indexing** (recommended): run [`scip-ruby`](https://sourcegraph.github.io/scip-ruby/) against your Ruby codebase and upload the generated index to Sourcegraph using the [src-cli](https://github.com/sourcegraph/src-cli) command `src code-intel upload`. This is usually setup to run in a CI pipeline. Sourcegraph automatically synchronizes Ruby dependency repositories based on the dependencies that are discovered by `scip-ruby`. +* **Code host configuration**: manually list dependencies in the `"dependencies"` section of the [JSON configuration](#configuration) when creating the Ruby dependency code host. This method can be useful to verify that the credentials are picked up correctly without having to upload an index. + +## Credentials + +The `"repository"` field in the [configuration](#configuration) section is automatically redacted and can optionally include the username and password of an internal [Artifactory RubyGems](https://www.jfrog.com/confluence/display/JFROG/RubyGems+Repositories) repository. + +## Rate limiting + +By default, requests to the RubyGems repository is 8 request per second. + +To manually set the value, add the following to your code host configuration: + +```json +"rateLimit": { + "enabled": true, + "requestsPerHour": 600.0 +} +``` +where the `requestsPerHour` field is set based on your requirements. + +**Not recommended**: Rate-limiting can be turned off entirely as well. +This increases the risk of overloading the code host. + +```json +"rateLimit": { + "enabled": false +} +``` + +## Configuration + +Ruby dependencies code host connections support the following configuration options, which are specified in the JSON editor in the site admin "Manage code hosts" area. + +
[View page on docs.sourcegraph.com](https://docs.sourcegraph.com/integration/ruby) to see rendered content.
diff --git a/doc/integration/index.md b/doc/integration/index.md index 2e4856d03ab..f45c816071e 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -15,6 +15,7 @@ Sourcegraph integrates with your other tools to help you search, navigate, and r - [npm dependencies](npm.md) - [Python dependencies](python.md) - [Go dependencies](go.md) + - [Ruby dependencies](ruby.md) - [Other Git repository hosts](../admin/external_service/other.md) - [Editor plugins](editor.md): jump to Sourcegraph from your editor - [Open in Editor](open_in_editor.md): jump to your editor from Sourcegraph diff --git a/doc/integration/ruby.md b/doc/integration/ruby.md new file mode 100644 index 00000000000..ad64c7c51ba --- /dev/null +++ b/doc/integration/ruby.md @@ -0,0 +1,29 @@ +# Ruby dependencies integration with Sourcegraph + +You can use Sourcegraph with Ruby dependencies from any RubyGems repository, including rubygems.org or an internal Artifactory. + +This integration makes it possible to search and navigate through the source code of published Ruby library (for example, [`shopify_api@12.0.0`](https://sourcegraph.com/rubygems/shopify_api@v12.0.0)). + +Feature | Supported? +------- | ---------- +[Repository syncing](#repository-syncing) | ✅ +[Repository permissions](#repository-syncing) | ❌ +[Multiple RubyGems repositories](#multiple-ruby-dependency-code-hosts) | ❌ + +## Setup + +See the "[Ruby dependencies](../admin/external_service/ruby.md)" documentation. + +## Repository syncing + +Site admins can [add Ruby dependencies to Sourcegraph](../admin/external_service/ruby.md#repository-syncing). + +## Repository permissions + +⚠ Ruby dependencies are visible by all users of the Sourcegraph instance. + +## Multiple Ruby dependencies code hosts + +⚠️ It's only possible to create one Ruby dependency code host for each Sourcegraph instance. + +See the issue [sourcegraph#32461](https://github.com/sourcegraph/sourcegraph/issues/32461) for more details about this limitation. diff --git a/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go b/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go index 3316e6c1c4f..0395acdd859 100644 --- a/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go +++ b/internal/codeintel/autoindexing/internal/background/job_dependency_sync_scheduler.go @@ -59,8 +59,9 @@ var autoIndexingEnabled = conf.CodeIntelAutoIndexingEnabled var schemeToExternalService = map[string]string{ dependencies.JVMPackagesScheme: extsvc.KindJVMPackages, dependencies.NpmPackagesScheme: extsvc.KindNpmPackages, - dependencies.RustPackagesScheme: extsvc.KindRustPackages, dependencies.PythonPackagesScheme: extsvc.KindPythonPackages, + dependencies.RustPackagesScheme: extsvc.KindRustPackages, + dependencies.RubyPackagesScheme: extsvc.KindRubyPackages, } func (h *dependencySyncSchedulerHandler) Handle(ctx context.Context, logger log.Logger, record workerutil.Record) error { @@ -244,6 +245,7 @@ func (h *dependencySyncSchedulerHandler) shouldIndexDependencies(ctx context.Con upload.Indexer == "scip-typescript" || upload.Indexer == "lsif-typescript" || upload.Indexer == "scip-python" || + upload.Indexer == "scip-ruby" || upload.Indexer == "rust-analyzer", nil } diff --git a/internal/codeintel/dependencies/consts.go b/internal/codeintel/dependencies/consts.go index 946d358d143..179ed984912 100644 --- a/internal/codeintel/dependencies/consts.go +++ b/internal/codeintel/dependencies/consts.go @@ -8,4 +8,5 @@ const ( GoPackagesScheme = shared.GoPackagesScheme PythonPackagesScheme = shared.PythonPackagesScheme RustPackagesScheme = shared.RustPackagesScheme + RubyPackagesScheme = shared.RubyPackagesScheme ) diff --git a/internal/codeintel/dependencies/shared/consts.go b/internal/codeintel/dependencies/shared/consts.go index 2d02e659cc5..4a94d165080 100644 --- a/internal/codeintel/dependencies/shared/consts.go +++ b/internal/codeintel/dependencies/shared/consts.go @@ -6,4 +6,5 @@ const ( NpmPackagesScheme = "npm" PythonPackagesScheme = "python" RustPackagesScheme = "rust-analyzer" + RubyPackagesScheme = "scip-ruby" ) diff --git a/internal/conf/reposource/ruby_packages.go b/internal/conf/reposource/ruby_packages.go new file mode 100644 index 00000000000..8f8e773d0e5 --- /dev/null +++ b/internal/conf/reposource/ruby_packages.go @@ -0,0 +1,89 @@ +package reposource + +import ( + "strings" + + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +const rubyPackagesPrefix = "rubygems/" + +type RubyVersionedPackage struct { + Name PackageName + Version string +} + +func NewRubyVersionedPackage(name PackageName, version string) *RubyVersionedPackage { + return &RubyVersionedPackage{ + Name: name, + Version: version, + } +} + +// ParseRubyVersionedPackage parses a string in a '(@version>)?' format into an +// RubyVersionedPackage. +func ParseRubyVersionedPackage(dependency string) (*RubyVersionedPackage, error) { + var dep RubyVersionedPackage + if i := strings.LastIndex(dependency, "@"); i == -1 { + dep.Name = PackageName(dependency) + } else { + dep.Name = PackageName(strings.TrimSpace(dependency[:i])) + dep.Version = strings.TrimSpace(dependency[i+1:]) + } + return &dep, nil +} + +func ParseRubyPackageFromName(name PackageName) (*RubyVersionedPackage, error) { + return ParseRubyVersionedPackage(string(name)) +} + +// ParseRubyPackageFromRepoName is a convenience function to parse a repo name in a +// 'crates/(@)?' format into a RubyVersionedPackage. +func ParseRubyPackageFromRepoName(name api.RepoName) (*RubyVersionedPackage, error) { + dependency := strings.TrimPrefix(string(name), rubyPackagesPrefix) + if len(dependency) == len(name) { + return nil, errors.Newf("invalid Ruby dependency repo name, missing %s prefix '%s'", rubyPackagesPrefix, name) + } + return ParseRubyVersionedPackage(dependency) +} + +func (p *RubyVersionedPackage) Scheme() string { + return "scip-ruby" +} + +func (p *RubyVersionedPackage) PackageSyntax() PackageName { + return p.Name +} + +func (p *RubyVersionedPackage) VersionedPackageSyntax() string { + if p.Version == "" { + return string(p.Name) + } + return string(p.Name) + "@" + p.Version +} + +func (p *RubyVersionedPackage) PackageVersion() string { + return p.Version +} + +func (p *RubyVersionedPackage) Description() string { return "" } + +func (p *RubyVersionedPackage) RepoName() api.RepoName { + return api.RepoName(rubyPackagesPrefix + p.Name) +} + +func (p *RubyVersionedPackage) GitTagFromVersion() string { + version := strings.TrimPrefix(p.Version, "v") + return "v" + version +} + +func (p *RubyVersionedPackage) Less(other VersionedPackage) bool { + o := other.(*RubyVersionedPackage) + + if p.Name == o.Name { + return versionGreaterThan(p.Version, o.Version) + } + + return p.Name > o.Name +} diff --git a/internal/database/external_services.go b/internal/database/external_services.go index c704b881ccb..9613e3a2700 100644 --- a/internal/database/external_services.go +++ b/internal/database/external_services.go @@ -219,6 +219,7 @@ var ExternalServiceKinds = map[string]ExternalServiceKind{ extsvc.KindPhabricator: {CodeHost: true, JSONSchema: schema.PhabricatorSchemaJSON}, extsvc.KindPythonPackages: {CodeHost: true, JSONSchema: schema.PythonPackagesSchemaJSON}, extsvc.KindRustPackages: {CodeHost: true, JSONSchema: schema.RustPackagesSchemaJSON}, + extsvc.KindRubyPackages: {CodeHost: true, JSONSchema: schema.RubyPackagesSchemaJSON}, } // ExternalServiceKind describes a kind of external service. diff --git a/internal/database/repos.go b/internal/database/repos.go index 0704cdbf128..bc77f48829c 100644 --- a/internal/database/repos.go +++ b/internal/database/repos.go @@ -562,6 +562,8 @@ func scanRepo(logger log.Logger, rows *sql.Rows, r *types.Repo) (err error) { r.Metadata = &struct{}{} case extsvc.TypeRustPackages: r.Metadata = &struct{}{} + case extsvc.TypeRubyPackages: + r.Metadata = &struct{}{} default: logger.Warn("unknown service type", log.String("type", typ)) return nil diff --git a/internal/extsvc/codehost.go b/internal/extsvc/codehost.go index 754e7b2e896..75762abdf1d 100644 --- a/internal/extsvc/codehost.go +++ b/internal/extsvc/codehost.go @@ -15,7 +15,7 @@ type CodeHost struct { func (c *CodeHost) IsPackageHost() bool { switch c.ServiceType { - case TypeNpmPackages, TypeJVMPackages, TypeGoModules, TypePythonPackages, TypeRustPackages: + case TypeNpmPackages, TypeJVMPackages, TypeGoModules, TypePythonPackages, TypeRustPackages, TypeRubyPackages: return true } return false @@ -46,6 +46,9 @@ var ( RustURL = &url.URL{Host: "crates"} RustPackages = NewCodeHost(RustURL, TypeRustPackages) + RubyURL = &url.URL{Host: "rubygems"} + RubyPackages = NewCodeHost(RubyURL, TypeRubyPackages) + PublicCodeHosts = []*CodeHost{ GitHubDotCom, GitLabDotCom, @@ -54,6 +57,7 @@ var ( GoModules, PythonPackages, RustPackages, + RubyPackages, } ) diff --git a/internal/extsvc/crates/client.go b/internal/extsvc/crates/client.go index 991046ced56..70814a545d9 100644 --- a/internal/extsvc/crates/client.go +++ b/internal/extsvc/crates/client.go @@ -17,11 +17,6 @@ type Client struct { limiter *ratelimit.InstrumentedLimiter } -type RustFile struct { - Name string - URL string -} - func NewClient(urn string, cli httpcli.Doer) *Client { return &Client{ cli: cli, diff --git a/internal/extsvc/rubygems/client.go b/internal/extsvc/rubygems/client.go new file mode 100644 index 00000000000..0de670e4f28 --- /dev/null +++ b/internal/extsvc/rubygems/client.go @@ -0,0 +1,77 @@ +package rubygems + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/sourcegraph/sourcegraph/internal/httpcli" + "github.com/sourcegraph/sourcegraph/internal/ratelimit" +) + +type Client struct { + cli httpcli.Doer + + // Self-imposed rate-limiter. + limiter *ratelimit.InstrumentedLimiter +} + +func NewClient(urn string, cli httpcli.Doer) *Client { + return &Client{ + cli: cli, + limiter: ratelimit.DefaultRegistry.Get(urn), + } +} + +func (c *Client) Get(ctx context.Context, url string) ([]byte, error) { + if err := c.limiter.Wait(ctx); err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Add("User-Agent", "sourcegraph-rubygems-syncer (sourcegraph.com)") + + b, err := c.do(req) + if err != nil { + return nil, err + } + return b, nil +} + +type Error struct { + path string + code int + message string +} + +func (e *Error) Error() string { + return fmt.Sprintf("bad response with status code %d for %s: %s", e.code, e.path, e.message) +} + +func (e *Error) NotFound() bool { + return e.code == http.StatusNotFound +} + +func (c *Client) do(req *http.Request) ([]byte, error) { + resp, err := c.cli.Do(req) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + bs, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, &Error{path: req.URL.Path, code: resp.StatusCode, message: string(bs)} + } + + return bs, nil +} diff --git a/internal/extsvc/types.go b/internal/extsvc/types.go index 514273237e5..667fa971859 100644 --- a/internal/extsvc/types.go +++ b/internal/extsvc/types.go @@ -125,6 +125,7 @@ const ( KindJVMPackages = "JVMPACKAGES" KindPythonPackages = "PYTHONPACKAGES" KindRustPackages = "RUSTPACKAGES" + KindRubyPackages = "RUBYPACKAGES" KindNpmPackages = "NPMPACKAGES" KindPagure = "PAGURE" KindOther = "OTHER" @@ -184,9 +185,12 @@ const ( // TypePythonPackages is the (api.ExternalRepoSpec).ServiceType value for Python packages. TypePythonPackages = "pythonPackages" - // TypeRustPackages is the (api.ExternalRepoSpec).ServiceType value for Python packages. + // TypeRustPackages is the (api.ExternalRepoSpec).ServiceType value for Rust packages. TypeRustPackages = "rustPackages" + // TypeRubyPackages is the (api.ExternalRepoSpec).ServiceType value for Ruby packages. + TypeRubyPackages = "rubyPackages" + // TypeOther is the (api.ExternalRepoSpec).ServiceType value for other projects. TypeOther = "other" ) @@ -219,6 +223,8 @@ func KindToType(kind string) string { return TypePythonPackages case KindRustPackages: return TypeRustPackages + case KindRubyPackages: + return TypeRubyPackages case KindNpmPackages: return TypeNpmPackages case KindGoPackages: @@ -262,6 +268,8 @@ func TypeToKind(t string) string { return KindPythonPackages case TypeRustPackages: return KindRustPackages + case TypeRubyPackages: + return KindRubyPackages case TypeGoModules: return KindGoPackages case TypePagure: @@ -282,6 +290,7 @@ var ( goLower = strings.ToLower(TypeGoModules) pythonLower = strings.ToLower(TypePythonPackages) rustLower = strings.ToLower(TypeRustPackages) + rubyLower = strings.ToLower(TypeRubyPackages) ) // ParseServiceType will return a ServiceType constant after doing a case insensitive match on s. @@ -316,6 +325,8 @@ func ParseServiceType(s string) (string, bool) { return TypePythonPackages, true case rustLower: return TypeRustPackages, true + case rubyLower: + return TypeRubyPackages, true case TypePagure: return TypePagure, true case TypeOther: @@ -355,6 +366,8 @@ func ParseServiceKind(s string) (string, bool) { return KindPythonPackages, true case KindRustPackages: return KindRustPackages, true + case KindRubyPackages: + return KindRubyPackages, true case KindPagure: return KindPagure, true case KindOther: @@ -436,6 +449,8 @@ func getConfigPrototype(kind string) (any, error) { return &schema.PythonPackagesConnection{}, nil case KindRustPackages: return &schema.RustPackagesConnection{}, nil + case KindRubyPackages: + return &schema.RubyPackagesConnection{}, nil case KindOther: return &schema.OtherExternalServiceConnection{}, nil default: @@ -616,6 +631,12 @@ func GetLimitFromConfig(kind string, config any) (rate.Limit, error) { if c != nil && c.RateLimit != nil { limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour) } + case *schema.RubyPackagesConnection: + // The rubygems.org API allows 10 rps https://guides.rubygems.org/rubygems-org-rate-limits/ + limit = rate.Limit(10) + if c != nil && c.RateLimit != nil { + limit = limitOrInf(c.RateLimit.Enabled, c.RateLimit.RequestsPerHour) + } default: return limit, ErrRateLimitUnsupported{codehostKind: kind} } @@ -736,6 +757,8 @@ func uniqueCodeHostIdentifier(kind string, cfg any) (string, error) { return KindPythonPackages, nil case *schema.RustPackagesConnection: return KindRustPackages, nil + case *schema.RubyPackagesConnection: + return KindRubyPackages, nil case *schema.PagureConnection: rawURL = c.Url default: diff --git a/internal/repos/clone_url.go b/internal/repos/clone_url.go index 12c11977bb2..13fdf218643 100644 --- a/internal/repos/clone_url.go +++ b/internal/repos/clone_url.go @@ -98,6 +98,8 @@ func cloneURL(parsed any, logger log.Logger, kind string, repo *types.Repo) (str return string(repo.Name), nil case *schema.RustPackagesConnection: return string(repo.Name), nil + case *schema.RubyPackagesConnection: + return string(repo.Name), nil case *schema.JVMPackagesConnection: if r, ok := repo.Metadata.(*reposource.MavenMetadata); ok { return r.Module.CloneURL(), nil diff --git a/internal/repos/ruby_packages.go b/internal/repos/ruby_packages.go new file mode 100644 index 00000000000..f09dfa85677 --- /dev/null +++ b/internal/repos/ruby_packages.go @@ -0,0 +1,56 @@ +package repos + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/api" + "github.com/sourcegraph/sourcegraph/internal/codeintel/dependencies" + "github.com/sourcegraph/sourcegraph/internal/conf/reposource" + "github.com/sourcegraph/sourcegraph/internal/extsvc/rubygems" + "github.com/sourcegraph/sourcegraph/internal/httpcli" + "github.com/sourcegraph/sourcegraph/internal/jsonc" + "github.com/sourcegraph/sourcegraph/internal/types" + "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/schema" +) + +// NewRubyPackagesSource returns a new rubyPackagesSource from the given external service. +func NewRubyPackagesSource(ctx context.Context, svc *types.ExternalService, cf *httpcli.Factory) (*PackagesSource, error) { + rawConfig, err := svc.Config.Decrypt(ctx) + if err != nil { + return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err) + } + var c schema.RubyPackagesConnection + if err := jsonc.Unmarshal(rawConfig, &c); err != nil { + return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err) + } + + cli, err := cf.Doer() + if err != nil { + return nil, err + } + + return &PackagesSource{ + svc: svc, + configDeps: c.Dependencies, + scheme: dependencies.RubyPackagesScheme, + src: &rubyPackagesSource{client: rubygems.NewClient(svc.URN(), cli)}, + }, nil +} + +type rubyPackagesSource struct { + client *rubygems.Client +} + +var _ packagesSource = &rubyPackagesSource{} + +func (rubyPackagesSource) ParseVersionedPackageFromConfiguration(dep string) (reposource.VersionedPackage, error) { + return reposource.ParseRubyVersionedPackage(dep) +} + +func (rubyPackagesSource) ParsePackageFromName(name reposource.PackageName) (reposource.Package, error) { + return reposource.ParseRubyPackageFromName(name) +} +func (rubyPackagesSource) ParsePackageFromRepoName(repoName api.RepoName) (reposource.Package, error) { + return reposource.ParseRubyPackageFromRepoName(repoName) +} diff --git a/internal/repos/sources.go b/internal/repos/sources.go index ff3fd8fd3ac..e6b38ca99c8 100644 --- a/internal/repos/sources.go +++ b/internal/repos/sources.go @@ -76,6 +76,8 @@ func NewSource(ctx context.Context, logger log.Logger, db database.DB, svc *type return NewPythonPackagesSource(ctx, svc, cf) case extsvc.KindRustPackages: return NewRustPackagesSource(ctx, svc, cf) + case extsvc.KindRubyPackages: + return NewRubyPackagesSource(ctx, svc, cf) case extsvc.KindOther: return NewOtherSource(ctx, svc, cf, logger.Scoped("OtherSource", "")) default: diff --git a/internal/types/secret.go b/internal/types/secret.go index 3125b4c9c9c..8c6c22da54f 100644 --- a/internal/types/secret.go +++ b/internal/types/secret.go @@ -77,6 +77,8 @@ func (e *ExternalService) RedactedConfig(ctx context.Context) (string, error) { } case *schema.RustPackagesConnection: // Nothing to redact + case *schema.RubyPackagesConnection: + es.redactString(c.Repository, "repository") case *schema.JVMPackagesConnection: if c.Maven != nil { es.redactString(c.Maven.Credentials, "maven", "credentials") @@ -174,6 +176,9 @@ func (e *ExternalService) UnredactConfig(ctx context.Context, old *ExternalServi } case *schema.RustPackagesConnection: // Nothing to unredact + case *schema.RubyPackagesConnection: + o := oldCfg.(*schema.RubyPackagesConnection) + es.unredactString(c.Repository, o.Repository, "repository") case *schema.JVMPackagesConnection: o := oldCfg.(*schema.JVMPackagesConnection) if c.Maven != nil && o.Maven != nil { diff --git a/schema/ruby-packages.schema.json b/schema/ruby-packages.schema.json new file mode 100644 index 00000000000..48a4d26084c --- /dev/null +++ b/schema/ruby-packages.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "ruby-packages.schema.json#", + "title": "RubyPackagesConnection", + "description": "Configuration for a connection to Ruby packages", + "allowComments": true, + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The URL at which the maven repository can be found.", + "type": "string", + "default": ["https://rubygems.org/"], + "examples": ["https://rubygems.org/", "https://.jfrog.io/artifactory/api/gems/"] + }, + "rateLimit": { + "description": "Rate limit applied when making background API requests to the configured Ruby repository APIs.", + "title": "RubyRateLimit", + "type": "object", + "required": ["enabled", "requestsPerHour"], + "properties": { + "enabled": { + "description": "true if rate limiting is enabled.", + "type": "boolean", + "default": true + }, + "requestsPerHour": { + "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.", + "type": "number", + "default": 8, + "minimum": 0 + } + }, + "default": { + "enabled": true, + "requestsPerHour": 3600 + } + }, + "dependencies": { + "description": "An array of strings specifying Ruby packages to mirror in Sourcegraph.", + "type": "array", + "items": { + "type": "string" + }, + "examples": [["shopify_api@12.0.0"]] + } + } +} diff --git a/schema/schema.go b/schema/schema.go index 5a8186b858a..61f0c79dd55 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -652,6 +652,8 @@ type ExperimentalFeatures struct { Ranking *Ranking `json:"ranking,omitempty"` // RateLimitAnonymous description: Configures the hourly rate limits for anonymous calls to the GraphQL API. Setting limit to 0 disables the limiter. This is only relevant if unauthenticated calls to the API are permitted. RateLimitAnonymous int `json:"rateLimitAnonymous,omitempty"` + // RubyPackages description: Allow adding Ruby package host connections + RubyPackages string `json:"rubyPackages,omitempty"` // RustPackages description: Allow adding Rust package code host connections RustPackages string `json:"rustPackages,omitempty"` // SearchIndexBranches description: A map from repository name to a list of extra revs (branch, ref, tag, commit sha, etc) to index for a repository. We always index the default branch ("HEAD") and revisions in version contexts. This allows specifying additional revisions. Sourcegraph can index up to 64 branches per repository. @@ -1587,6 +1589,24 @@ type Responders struct { Username string `json:"username,omitempty"` } +// RubyPackagesConnection description: Configuration for a connection to Ruby packages +type RubyPackagesConnection struct { + // Dependencies description: An array of strings specifying Ruby packages to mirror in Sourcegraph. + Dependencies []string `json:"dependencies,omitempty"` + // RateLimit description: Rate limit applied when making background API requests to the configured Ruby repository APIs. + RateLimit *RubyRateLimit `json:"rateLimit,omitempty"` + // Repository description: The URL at which the maven repository can be found. + Repository string `json:"repository,omitempty"` +} + +// RubyRateLimit description: Rate limit applied when making background API requests to the configured Ruby repository APIs. +type RubyRateLimit struct { + // Enabled description: true if rate limiting is enabled. + Enabled bool `json:"enabled"` + // RequestsPerHour description: Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1. + RequestsPerHour float64 `json:"requestsPerHour"` +} + // RustPackagesConnection description: Configuration for a connection to Rust packages type RustPackagesConnection struct { // Dependencies description: An array of strings specifying Rust packages to mirror in Sourcegraph. diff --git a/schema/site.schema.json b/schema/site.schema.json index 830840a2ea4..db17eadedf5 100644 --- a/schema/site.schema.json +++ b/schema/site.schema.json @@ -246,6 +246,12 @@ "enum": ["enabled", "disabled"], "default": "disabled" }, + "rubyPackages": { + "description": "Allow adding Ruby package host connections", + "type": "string", + "enum": ["enabled", "disabled"], + "default": "disabled" + }, "pagure": { "description": "Allow adding Pagure code host connections", "type": "string", diff --git a/schema/stringdata.go b/schema/stringdata.go index b69b6d0d34b..bb8c84a0f65 100644 --- a/schema/stringdata.go +++ b/schema/stringdata.go @@ -47,31 +47,24 @@ var GitLabSchemaJSON string //go:embed gitolite.schema.json var GitoliteSchemaJSON string -// GoModulesSchemaJSON is the content of the file "go-modules.schema.json". -// //go:embed go-modules.schema.json var GoModulesSchemaJSON string -// JVMPackagesSchemaJSON is the content of the file "jvm-packages.schema.json". -// //go:embed jvm-packages.schema.json var JVMPackagesSchemaJSON string -// NpmPackagesSchemaJSON is the content of the file "npm-packages.schema.json". -// //go:embed npm-packages.schema.json var NpmPackagesSchemaJSON string -// PythonPackagesSchemaJSON is the content of the file "python-packages.schema.json". -// //go:embed python-packages.schema.json var PythonPackagesSchemaJSON string -// RustPackagesSchemaJSON is the content of the file "python-packages.schema.json". -// //go:embed rust-packages.schema.json var RustPackagesSchemaJSON string +//go:embed ruby-packages.schema.json +var RubyPackagesSchemaJSON string + // OtherExternalServiceSchemaJSON is the content of the file "other_external_service.schema.json". // //go:embed other_external_service.schema.json