diff --git a/cmd/gitserver/server/vcs_syncer_jvm_packages.go b/cmd/gitserver/server/vcs_syncer_jvm_packages.go index 1960df3cdc0..18f09cef551 100644 --- a/cmd/gitserver/server/vcs_syncer_jvm_packages.go +++ b/cmd/gitserver/server/vcs_syncer_jvm_packages.go @@ -42,11 +42,6 @@ func NewJVMPackagesSyncer(connection *schema.JVMPackagesConnection, svc *depende panic(fmt.Sprintf("expected placeholder package to parse but got %v", err)) } - var configDeps []string - if connection.Maven != nil { - configDeps = connection.Maven.Dependencies - } - chandle := coursier.NewCoursierHandle(observation.NewContext(log.Scoped("gitserver.jvmsyncer", "")), cacheDir) return &vcsPackagesSyncer{ @@ -55,7 +50,7 @@ func NewJVMPackagesSyncer(connection *schema.JVMPackagesConnection, svc *depende scheme: dependencies.JVMPackagesScheme, placeholder: placeholder, svc: svc, - configDeps: configDeps, + configDeps: connection.Maven.Dependencies, source: &jvmPackagesSyncer{ coursier: chandle, config: connection, diff --git a/cmd/gitserver/server/vcs_syncer_jvm_packages_test.go b/cmd/gitserver/server/vcs_syncer_jvm_packages_test.go index f194914629f..3b0e144c9cb 100644 --- a/cmd/gitserver/server/vcs_syncer_jvm_packages_test.go +++ b/cmd/gitserver/server/vcs_syncer_jvm_packages_test.go @@ -132,7 +132,7 @@ func TestNoMaliciousFiles(t *testing.T) { cacheDir := filepath.Join(dir, "cache") s := jvmPackagesSyncer{ coursier: coursier.NewCoursierHandle(&observation.TestContext, cacheDir), - config: &schema.JVMPackagesConnection{Maven: &schema.Maven{Dependencies: []string{}}}, + config: &schema.JVMPackagesConnection{Maven: schema.Maven{Dependencies: []string{}}}, fetch: func(ctx context.Context, config *schema.JVMPackagesConnection, dependency *reposource.MavenVersionedPackage) (sourceCodeJarPath string, err error) { jarPath := path.Join(dir, "sampletext.zip") createMaliciousJar(t, jarPath) @@ -202,7 +202,7 @@ func TestJVMCloneCommand(t *testing.T) { depsSvc := dependencies.TestService(database.NewDB(logger, dbtest.NewDB(logger, t))) cacheDir := filepath.Join(dir, "cache") - s := NewJVMPackagesSyncer(&schema.JVMPackagesConnection{Maven: &schema.Maven{Dependencies: []string{}}}, depsSvc, cacheDir).(*vcsPackagesSyncer) + s := NewJVMPackagesSyncer(&schema.JVMPackagesConnection{Maven: schema.Maven{Dependencies: []string{}}}, depsSvc, cacheDir).(*vcsPackagesSyncer) bareGitDirectory := path.Join(dir, "git") s.runCloneCommand(t, examplePackageUrl, bareGitDirectory, []string{exampleVersionedPackage}) diff --git a/doc/admin/external_service/go.md b/doc/admin/external_service/go.md index 492e2a17264..b64dda7431c 100644 --- a/doc/admin/external_service/go.md +++ b/doc/admin/external_service/go.md @@ -39,6 +39,7 @@ based on a default internal limit. ([source](https://github.com/sourcegraph/sour "requestsPerHour": 57600.0 } ``` + where the `requestsPerHour` field is set based on your requirements. **Not recommended**: Rate-limiting can be turned off entirely as well. diff --git a/doc/admin/external_service/index.md b/doc/admin/external_service/index.md index 052fafec4a5..7b16cea0051 100644 --- a/doc/admin/external_service/index.md +++ b/doc/admin/external_service/index.md @@ -170,11 +170,13 @@ We recognize there are other code hosts including CVS, SVN, and many more. Today - [Non-Git code hosts](non-git.md) - [Perforce](../repo/perforce.md) - [Plastic SCM](../repo/plasticscm.md) - - [JVM dependencies](jvm.md) - - [Go dependencies](go.md) - - [npm dependencies](npm.md) - - [Python dependencies](python.md) - - [Ruby dependencies](ruby.md) + - [Package repository hosts](package-repos.md) + - [JVM dependencies](jvm.md) + - [Go dependencies](go.md) + - [npm dependencies](npm.md) + - [Python dependencies](python.md) + - [Ruby dependencies](ruby.md) + - [Rust dependencies](rust.md) **Users** can configure the following public code hosts: diff --git a/doc/admin/external_service/jvm.md b/doc/admin/external_service/jvm.md index 388845ebd7e..22ac93f9c19 100644 --- a/doc/admin/external_service/jvm.md +++ b/doc/admin/external_service/jvm.md @@ -26,7 +26,7 @@ There are two ways to sync JVM dependency repositories. ## Credentials Sourcegraph uses [Coursier](https://get-coursier.io/) to resolve JVM dependencies. -Use the `"credentials"` section of the JSON configuration to provide usernames and passwords to access your Maven repository. See the Coursier documentation about [inline credentials](https://get-coursier.io/docs/other-credentials#inline) to learn more about how to format the `"credentials"` configuration. +Use the `"credentials"` section of the JSON configuration to provide usernames and passwords to access your Maven repository. See the Coursier documentation about [inline credentials](https://get-coursier.io/docs/other-credentials#inline) to learn more about how to format the `"credentials"` configuration, or the example displayed at the bottom of the page. ## Rate limiting @@ -38,9 +38,10 @@ To manually set the value, add the following to your code host configuration: ```json "rateLimit": { "enabled": true, - "requestsPerHour": 600.0 + "requestsPerHour": 600 } ``` + where the `requestsPerHour` field is set based on your requirements. **Not recommended**: Rate-limiting can be turned off entirely as well. @@ -56,4 +57,4 @@ This increases the risk of overloading the code host. JVM dependencies code host connections support the following configuration options, which are specified in the JSON editor in the site admin "Manage code hosts" area. -
[View page on docs.sourcegraph.com](https://docs.sourcegraph.com/integration/jvm) to see rendered content.
+
[View page on docs.sourcegraph.com](https://docs.sourcegraph.com/admin/external_service/jvm) to see rendered content.
diff --git a/doc/admin/external_service/npm.md b/doc/admin/external_service/npm.md index 1e9a6c1e992..ef3b1bbca9a 100644 --- a/doc/admin/external_service/npm.md +++ b/doc/admin/external_service/npm.md @@ -34,9 +34,10 @@ By default, requests to the npm registry will be rate-limited based on a default ```json "rateLimit": { "enabled": true, - "requestsPerHour": 3000.0 + "requestsPerHour": 3000 } ``` + where the `requestsPerHour` field is set based on your requirements. **Not recommended**: Rate-limiting can be turned off entirely as well. diff --git a/doc/admin/external_service/package-repos.md b/doc/admin/external_service/package-repos.md new file mode 100644 index 00000000000..942eda1ecf2 --- /dev/null +++ b/doc/admin/external_service/package-repos.md @@ -0,0 +1,53 @@ +# Package repositories + + + +Sourcegraph package repos can synchronize dependency sources (Rust crates, JVM libraries, Node.js packages, Ruby gems, and more) from public and private artifact hosts (such as NPM, Artifactory etc). + +## Enable package repositories + +Package repositories can be enabled on a per-ecosystem level in your [site configuration](/admin/config/site_config), for example: + +```json +{ + // ... + "experimentalFeatures": { + "jvmPackages": "enabled", + "goPackagse": "enabled", + "npmPackages": "enabled", + "pythonPackagse": "disabled", + "rubyPackages": "disabled", + "rustPacakges": "enabled" + } + // ... +} +``` + +## Repository syncing + +There are generally two ways of syncing package repositories to the Sourcegraph instance. + +1. **Indexing** (recommended): package repositories can be added to the Sourcegraph instance when they are referenced in [code graph data uploads](/code_navigation/explanations/uploads). Code graph indexers can derive package repository references during indexing, which will be used on upload to sync them to your instance. An external service for the given ecosystem must be created in order for these referenced package repositories to be synchronized. +2. **Code host configuration**: each package repository external service supports manually defining versions of packages to sync. See the page specific to the ecosystem you wish to configure. This method can be useful to verify that the credentials are picked up correctly without having to upload an index, as we'll as to poke holes in the filters (detailed below) if necessary. + +## Filters + +Package repository filters allow you to limit the package repositories and versions that are synced to the Sourcegraph instance. Using glob patterns, you can block or allow certain package repositories or versions on a per-ecosystem level. + +There are two kinds of filters: + +1. **Package name filters**: these filters match a glob pattern against a package repository's name, and will apply to all versions of matching package repositories. +2. **Package version filters**: these filters match a glob pattern against versions for a specific package repository. It cannot apply to multiple package repositories. + +Filters can also have one of two behaviours: + +1. **Block filters**: package repositories or versions matching these filters won't be synced. +2. **Allow filters**: only package repositories or versions matching these filters (that don't match a block filter) will be synced. + +Having no configured _allow_ filters implicitly allows everything. _Block_ filters have priority over _allow_ filters (a blocked package or version will not be synced even if it is explicitly allowed). + +Package repository filter updates may require a few minutes to propagate through the system. The blocked status of a package repository will be updated in the UI and the relevant external service may add or remove repositories after syncing (visible in the site-admin page for the external service). diff --git a/doc/admin/external_service/python.md b/doc/admin/external_service/python.md index 8256029e55a..374727c2a50 100644 --- a/doc/admin/external_service/python.md +++ b/doc/admin/external_service/python.md @@ -35,9 +35,10 @@ By default, requests to the Python package mirrors will be rate-limited based on ```json "rateLimit": { "enabled": true, - "requestsPerHour": 57600.0 + "requestsPerHour": 57600 } ``` + where the `requestsPerHour` field is set based on your requirements. **Not recommended**: Rate-limiting can be turned off entirely as well. diff --git a/doc/admin/external_service/ruby.md b/doc/admin/external_service/ruby.md index 576c387fb3c..5aeeda6443c 100644 --- a/doc/admin/external_service/ruby.md +++ b/doc/admin/external_service/ruby.md @@ -36,9 +36,10 @@ To manually set the value, add the following to your code host configuration: ```json "rateLimit": { "enabled": true, - "requestsPerHour": 600.0 + "requestsPerHour": 600 } ``` + where the `requestsPerHour` field is set based on your requirements. **Not recommended**: Rate-limiting can be turned off entirely as well. diff --git a/doc/admin/external_service/rust-packages.schema.json b/doc/admin/external_service/rust-packages.schema.json new file mode 120000 index 00000000000..2dbe961d90d --- /dev/null +++ b/doc/admin/external_service/rust-packages.schema.json @@ -0,0 +1 @@ +../../../schema/rust-packages.schema.json \ No newline at end of file diff --git a/doc/admin/external_service/rust.md b/doc/admin/external_service/rust.md new file mode 100644 index 00000000000..2a26ba56bf3 --- /dev/null +++ b/doc/admin/external_service/rust.md @@ -0,0 +1,58 @@ +# Rust dependencies + + + +Site admins can sync Rust dependencies from any Cargo repository, including crates.io or an internal Artifactory, to their Sourcegraph instance so that users can search and navigate their dependencies. + +To add Rust dependencies to Sourcegraph you need to setup a Rust dependencies code host: + +1. As *site admin*: go to **Site admin > Global settings** and enable the experimental feature by adding: `{"experimentalFeatures": {"rustPackages": "enabled"} }` +1. As *site admin*: go to **Site admin > Manage code hosts** +1. Select **Rust Dependencies**. +1. [Configure the connection](#configuration) by following the instructions above the text field. Additional fields can be added using Cmd/Ctrl+Space for auto-completion. See the [configuration documentation below](#configuration). +1. Press **Add repositories**. + +## Repository syncing + +There are two ways to sync Rust dependency repositories. + +* **Indexing** (recommended): run [`scip-rust`](https://sourcegraph.github.io/scip-rust/) against your Rust codebase and upload the generated index to Sourcegraph using the [src-cli](https://github.com/sourcegraph/src-cli) command `src code-intel upload`. This is usually setup to run in a CI pipeline. Sourcegraph automatically synchronizes Rust dependency repositories based on the dependencies that are discovered by `scip-rust`. +* **Code host configuration**: manually list dependencies in the `"dependencies"` section of the [JSON configuration](#configuration) when creating the Rust dependency code host. This method can be useful to verify that the credentials are picked up correctly without having to upload an index. + +## Credentials + +The `"repository"` field in the [configuration](#configuration) section is automatically redacted and can optionally include the username and password of an internal [Artifactory Cargo](https://www.jfrog.com/confluence/display/JFROG/Cargo+Repositories) repository. + +## Rate limiting + +By default, requests to the Cargo repository is 8 request per second. + +To manually set the value, add the following to your code host configuration: + +```json +"rateLimit": { + "enabled": true, + "requestsPerHour": 600 +} +``` + +where the `requestsPerHour` field is set based on your requirements. + +**Not recommended**: Rate-limiting can be turned off entirely as well. +This increases the risk of overloading the code host. + +```json +"rateLimit": { + "enabled": false +} +``` + +## Configuration + +Rust dependencies code host connections support the following configuration options, which are specified in the JSON editor in the site admin "Manage code hosts" area. + +
[View page on docs.sourcegraph.com](https://docs.sourcegraph.com/integration/rust) to see rendered content.
diff --git a/doc/integration/go.md b/doc/integration/go.md index 55d12a74eef..ffe1fcfccd9 100644 --- a/doc/integration/go.md +++ b/doc/integration/go.md @@ -1,12 +1,14 @@ # Go dependencies integration with Sourcegraph You can use Sourcegraph with Go modules from any Go module proxy, including open source code from proxy.golang.org or a private proxy such as [Athens](https://github.com/gomods/athens). + This integration makes it possible to search and navigate through the source code of published Go modules (for example, [`gorilla/mux@v1.8.0`](https://sourcegraph.com/go/github.com/gorilla/mux@v1.8.0)). Feature | Supported? ------- | ---------- [Repository syncing](#repository-syncing) | ✅ [Repository permissions](#repository-syncing) | ❌ +[Multiple Go repositories code hosts](#multiple-go-dependencies-code-hosts) | ❌ ## Setup @@ -19,3 +21,9 @@ Site admins can [add Go packages to Sourcegraph](../admin/external_service/go.md ## Repository permissions ⚠️ Go dependency repositories are visible by all users of the Sourcegraph instance. + +## Multiple Go dependencies code hosts + +⚠️ It's only possible to create one Ruby dependency code host for each Sourcegraph instance. + +See the issue [sourcegraph#32461](https://github.com/sourcegraph/sourcegraph/issues/32461) for more details about this limitation. diff --git a/doc/integration/index.md b/doc/integration/index.md index ae2a479be77..89b8b3f6248 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -5,25 +5,26 @@ Sourcegraph integrates with your other tools to help you search, navigate, and r - [GraphQL API](../api/graphql/index.md): create custom tools using Sourcegraph data - [Browser extension](browser_extension/index.md): go-to-definitions and hovers in your code host and code reviews - Code hosts - - [GitHub](github.md) - - [GitLab](gitlab.md) - - [Bitbucket Cloud](bitbucket_cloud.md) - - [Bitbucket Server / Bitbucket Data Center](bitbucket_server.md) + - [GitHub](github.md) + - [GitLab](gitlab.md) + - [Bitbucket Cloud](bitbucket_cloud.md) + - [Bitbucket Server / Bitbucket Data Center](bitbucket_server.md) + - [Package repository hosts](../admin/external_service/package-repos.md) - [JVM dependencies](jvm.md) - [npm dependencies](npm.md) - [Python dependencies](python.md) - [Go dependencies](go.md) - [Ruby dependencies](ruby.md) - - [Other Git repository hosts](../admin/external_service/other.md) - - [Editor plugins](editor.md): jump to Sourcegraph from your editor - - [Open in Editor](open_in_editor.md): jump to your editor from Sourcegraph - - [Search shortcuts](browser_extension/how-tos/browser_search_engine.md): quickly search from your browser - - Launcher extensions - - [Sourcegraph for Raycast](https://www.raycast.com/bobheadxi/sourcegraph) (unofficial): search code, browse - notebooks, and manage batch changes from the Raycast launcher + - [Other Git repository hosts](../admin/external_service/other.md) +- [Editor plugins](editor.md): jump to Sourcegraph from your editor + - [Open in Editor](open_in_editor.md): jump to your editor from Sourcegraph + - [Search shortcuts](browser_extension/how-tos/browser_search_engine.md): quickly search from your browser +- Launcher extensions + - [Sourcegraph for Raycast](https://www.raycast.com/bobheadxi/sourcegraph) (unofficial): search code, browse + notebooks, and manage batch changes from the Raycast launcher - IDE Extensions - - [JetBrains](jetbrains/index.md) - - [Manual testing](jetbrains/manual_testing.md) - - [VS Code](vscode/index.md) + - [JetBrains](jetbrains/index.md) + - [Manual testing](jetbrains/manual_testing.md) + - [VS Code](vscode/index.md) ![GitHub pull request integration](https://storage.googleapis.com/sourcegraph-assets/code-graph/docs/github-pr.png) diff --git a/doc/integration/jvm.md b/doc/integration/jvm.md index 6fa81ae3e5a..b23f7aff2b0 100644 --- a/doc/integration/jvm.md +++ b/doc/integration/jvm.md @@ -8,7 +8,7 @@ Feature | Supported? ------- | ---------- [Repository syncing](#repository-syncing) | ✅ [Repository permissions](#repository-syncing) | ❌ -[Multiple JVM dependencies code hosts](#multiple-jvm-dependency-code-hosts) | ❌ +[Multiple JVM dependencies code hosts](#multiple-jvm-dependencies-code-hosts) | ❌ ## Setup diff --git a/doc/integration/npm.md b/doc/integration/npm.md index 4d849ff4934..9260e025274 100644 --- a/doc/integration/npm.md +++ b/doc/integration/npm.md @@ -7,7 +7,7 @@ Feature | Supported? ------- | ---------- [Repository syncing](#repository-syncing) | ✅ [Repository permissions](#repository-syncing) | ❌ -[Multiple npm dependencies code hosts](#multiple-npm-dependency-code-hosts) | ❌ +[Multiple npm dependencies code hosts](#multiple-npm-dependencies-code-hosts) | ❌ ## Setup diff --git a/doc/integration/python.md b/doc/integration/python.md index 8c3942288d8..383d894f902 100644 --- a/doc/integration/python.md +++ b/doc/integration/python.md @@ -1,12 +1,14 @@ # Python dependencies integration with Sourcegraph You can use Sourcegraph with Python packages from any Python package mirror, including open source code from pypi.org or a private mirror such as [Nexus](https://www.sonatype.com/products/nexus-repository). + This integration makes it possible to search and navigate through the source code of published Python packages (for example, [`numpy@v1.19.5`](https://sourcegraph.com/python/numpy@v1.19.5)). Feature | Supported? ------- | ---------- [Repository syncing](#repository-syncing) | ✅ [Repository permissions](#repository-permissions) | ❌ +[Multiple Python repositories code hosts](#multiple-python-dependencies-code-hosts) | ❌ ## Setup @@ -19,3 +21,9 @@ Site admins can [add Python packages to Sourcegraph](../admin/external_service/p ## Repository permissions ⚠️ Python dependency repositories are visible by all users of the Sourcegraph instance. + +## Multiple Python dependencies code hosts + +⚠️ It's only possible to create one Python dependency code host for each Sourcegraph instance. + +See the issue [sourcegraph#32461](https://github.com/sourcegraph/sourcegraph/issues/32461) for more details about this limitation. diff --git a/doc/integration/ruby.md b/doc/integration/ruby.md index ad64c7c51ba..de4b399c421 100644 --- a/doc/integration/ruby.md +++ b/doc/integration/ruby.md @@ -8,7 +8,7 @@ Feature | Supported? ------- | ---------- [Repository syncing](#repository-syncing) | ✅ [Repository permissions](#repository-syncing) | ❌ -[Multiple RubyGems repositories](#multiple-ruby-dependency-code-hosts) | ❌ +[Multiple RubyGems repositories code hosts](#multiple-ruby-dependencies-code-hosts) | ❌ ## Setup diff --git a/doc/integration/rust.md b/doc/integration/rust.md new file mode 100644 index 00000000000..ec87e4ee8bb --- /dev/null +++ b/doc/integration/rust.md @@ -0,0 +1,29 @@ +# Rust dependencies integration with Sourcegraph + +You can use Sourcegraph with Rust dependencies from any Cargo repository, including crates.io or an internal Artifactory. + +This integration makes it possible to search and navigate through the source code of published Rust crates (for example, [`serde@1.0.158`](https://sourcegraph.com/crates/serde@v1.0.158)). + +Feature | Supported? +------- | ---------- +[Repository syncing](#repository-syncing) | ✅ +[Repository permissions](#repository-syncing) | ❌ +[Multiple Cargo repositories code hosts](#multiple-rust-dependencies-code-hosts) | ❌ + +## Setup + +See the "[Rust dependencies](../admin/external_service/Rust.md)" documentation. + +## Repository syncing + +Site admins can [add Rust dependencies to Sourcegraph](../admin/external_service/rust.md#repository-syncing). + +## Repository permissions + +⚠ Rust dependencies are visible by all users of the Sourcegraph instance. + +## Multiple Rust dependencies code hosts + +⚠️ It's only possible to create one Rust dependency code host for each Sourcegraph instance. + +See the issue [sourcegraph#32461](https://github.com/sourcegraph/sourcegraph/issues/32461) for more details about this limitation. diff --git a/internal/extsvc/types.go b/internal/extsvc/types.go index eb5203d6e5a..82c824c0ea5 100644 --- a/internal/extsvc/types.go +++ b/internal/extsvc/types.go @@ -635,7 +635,7 @@ func GetLimitFromConfig(kind string, config any) (rate.Limit, error) { } case *schema.JVMPackagesConnection: limit = rate.Limit(2) - if c != nil && c.Maven != nil && c.Maven.RateLimit != nil { + if c != nil && c.Maven.RateLimit != nil { limit = limitOrInf(c.Maven.RateLimit.Enabled, c.Maven.RateLimit.RequestsPerHour) } case *schema.PagureConnection: diff --git a/internal/repos/jvm_packages.go b/internal/repos/jvm_packages.go index cab15beb315..2fd6c4098fb 100644 --- a/internal/repos/jvm_packages.go +++ b/internal/repos/jvm_packages.go @@ -24,14 +24,9 @@ func NewJVMPackagesSource(ctx context.Context, svc *types.ExternalService) (*Pac return nil, errors.Errorf("external service id=%d config error: %s", svc.ID, err) } - var configDeps []string - if c.Maven != nil { - configDeps = c.Maven.Dependencies - } - return &PackagesSource{ svc: svc, - configDeps: configDeps, + configDeps: c.Maven.Dependencies, scheme: dependencies.JVMPackagesScheme, src: &jvmPackagesSource{config: &c}, }, nil diff --git a/internal/types/secret.go b/internal/types/secret.go index 66a51ef74b6..c3b1531c2b4 100644 --- a/internal/types/secret.go +++ b/internal/types/secret.go @@ -94,9 +94,7 @@ func (e *ExternalService) RedactedConfig(ctx context.Context) (string, error) { case *schema.RubyPackagesConnection: es.redactString(c.Repository, "repository") case *schema.JVMPackagesConnection: - if c.Maven != nil { - es.redactString(c.Maven.Credentials, "maven", "credentials") - } + es.redactString(c.Maven.Credentials, "maven", "credentials") case *schema.PagureConnection: es.redactString(c.Token, "token") case *schema.NpmPackagesConnection: @@ -229,28 +227,25 @@ func (e *ExternalService) UnredactConfig(ctx context.Context, old *ExternalServi es.unredactString(c.Repository, o.Repository, "repository") case *schema.JVMPackagesConnection: o := oldCfg.(*schema.JVMPackagesConnection) - if c.Maven != nil && o.Maven != nil { - // credentials didn't change check if repositories did - if c.Maven.Credentials == RedactedSecret { - oldRepos := o.Maven.Repositories - sort.Strings(oldRepos) + // credentials didn't change check if repositories did + if c.Maven.Credentials == RedactedSecret { + oldRepos := o.Maven.Repositories + sort.Strings(oldRepos) - newRepos := c.Maven.Repositories - sort.Strings(newRepos) + newRepos := c.Maven.Repositories + sort.Strings(newRepos) - // if we only remove a known repo, it's fine - if len(newRepos) < len(oldRepos) { - for _, r := range newRepos { - // we have a new repo in the list, return error - if !slices.Contains(oldRepos, r) { - return errCodeHostIdentityChanged{"repositories", "credentials"} - } + // if we only remove a known repo, it's fine + if len(newRepos) < len(oldRepos) { + for _, r := range newRepos { + // we have a new repo in the list, return error + if !slices.Contains(oldRepos, r) { + return errCodeHostIdentityChanged{"repositories", "credentials"} } - } else if !slices.Equal(oldRepos, newRepos) { - return errCodeHostIdentityChanged{"repositories", "credentials"} } + } else if !slices.Equal(oldRepos, newRepos) { + return errCodeHostIdentityChanged{"repositories", "credentials"} } - } es.unredactString(c.Maven.Credentials, o.Maven.Credentials, "maven", "credentials") case *schema.PagureConnection: @@ -397,6 +392,7 @@ func (es *edits) unredactURLs(new, old []string) (err error) { return nil } + func (es *edits) unredactURL(new, old string, path ...any) error { if new == "" || old == "" { return nil diff --git a/internal/types/secret_test.go b/internal/types/secret_test.go index 6e36579812e..264bab48f32 100644 --- a/internal/types/secret_test.go +++ b/internal/types/secret_test.go @@ -87,8 +87,8 @@ func TestExternalService_RedactedConfig(t *testing.T) { }, { kind: extsvc.KindJVMPackages, - in: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Dependencies: []string{"baz"}}}, - out: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: RedactedSecret, Dependencies: []string{"baz"}}}, + in: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Dependencies: []string{"baz"}}}, + out: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: RedactedSecret, Dependencies: []string{"baz"}}}, }, { kind: extsvc.KindNpmPackages, @@ -296,28 +296,28 @@ func TestExternalService_UnredactConfig(t *testing.T) { }, { kind: extsvc.KindJVMPackages, - old: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Dependencies: []string{"baz"}}}, - in: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: RedactedSecret, Dependencies: []string{"bar"}}}, - out: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Dependencies: []string{"bar"}}}, + old: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Dependencies: []string{"baz"}}}, + in: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: RedactedSecret, Dependencies: []string{"bar"}}}, + out: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Dependencies: []string{"bar"}}}, }, { kind: extsvc.KindJVMPackages, - old: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Repositories: []string{"foo", "baz"}}}, - in: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: RedactedSecret, Repositories: []string{"bar"}}}, - out: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Repositories: []string{"bar"}}}, + old: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Repositories: []string{"foo", "baz"}}}, + in: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: RedactedSecret, Repositories: []string{"bar"}}}, + out: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Repositories: []string{"bar"}}}, wantErr: errCodeHostIdentityChanged{"repositories", "credentials"}, }, { kind: extsvc.KindJVMPackages, - old: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Repositories: []string{"foo", "baz"}}}, - in: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: RedactedSecret, Repositories: []string{"baz", "foo"}}}, - out: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Repositories: []string{"baz", "foo"}}}, + old: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Repositories: []string{"foo", "baz"}}}, + in: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: RedactedSecret, Repositories: []string{"baz", "foo"}}}, + out: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Repositories: []string{"baz", "foo"}}}, }, { kind: extsvc.KindJVMPackages, - old: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Repositories: []string{"foo", "baz"}}}, - in: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: RedactedSecret, Repositories: []string{"baz"}}}, - out: schema.JVMPackagesConnection{Maven: &schema.Maven{Credentials: "foobar", Repositories: []string{"baz"}}}, + old: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Repositories: []string{"foo", "baz"}}}, + in: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: RedactedSecret, Repositories: []string{"baz"}}}, + out: schema.JVMPackagesConnection{Maven: schema.Maven{Credentials: "foobar", Repositories: []string{"baz"}}}, }, { kind: extsvc.KindNpmPackages, diff --git a/schema/jvm-packages.schema.json b/schema/jvm-packages.schema.json index 9f0d5f347bb..1df07d8d267 100644 --- a/schema/jvm-packages.schema.json +++ b/schema/jvm-packages.schema.json @@ -6,6 +6,7 @@ "allowComments": true, "type": "object", "additionalProperties": false, + "required": ["maven"], "properties": { "maven": { "description": "Configuration for resolving from Maven repositories.", @@ -25,7 +26,7 @@ ] }, "credentials": { - "description": "Contents of a coursier.credentials file needed for accessing the Maven repositories.", + "description": "Contents of a coursier.credentials file needed for accessing the Maven repositories. See the 'Inline' section at https://get-coursier.io/docs/other-credentials#inline for more details.", "type": "string" }, "rateLimit": { @@ -42,7 +43,7 @@ "requestsPerHour": { "description": "Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.", "type": "number", - "default": 5000, + "default": 7200, "minimum": 0 } }, @@ -60,6 +61,28 @@ }, "examples": [["groupID:artifactID"], ["org.apache.commons:commons-csv", "com.google.guava:guava"]] } + }, + "examples": [ + { + "maven": { + "repositories": ["central", "http://repo.extendedclip.com/content/repositories/public/"], + "rateLimit": { + "enabled": true, + "requestsPerHour": 3600 + }, + "credentials": "jfrog.frogger.com myuser:supersecret\njfrogmirror.frogger.com(basicauth realm) frog:detective", + "dependencies": ["com.sourcegraph:semanticdb-kotlinc:0.1.1"] + } + } + ], + "default": { + "repositories": [], + "credentials": "", + "rateLimit": { + "enabled": true, + "requestsPerHour": 7200 + }, + "dependencies": [] } } } diff --git a/schema/rust-packages.schema.json b/schema/rust-packages.schema.json index 6eb1357596f..ef77954125b 100644 --- a/schema/rust-packages.schema.json +++ b/schema/rust-packages.schema.json @@ -8,7 +8,7 @@ "additionalProperties": false, "properties": { "indexRepositoryName": { - "description": "Name of the git repository containing the crates.io index. Empty by default, which means no syncing happens. Updating this setting does not trigger a sync immediately, you must wait until the next scheduled sync for the value to get picked up.", + "description": "Name of the git repository containing the crates.io index. Only set if you intend to sync every crate from the index. Updating this setting does not trigger a sync immediately, you must wait until the next scheduled sync for the value to get picked up.", "title": "RustIndexRepositoryName", "type": "string", "examples": ["github.com/crates.io/index"] diff --git a/schema/schema.go b/schema/schema.go index 7b33f789ba7..379e87506a8 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -1353,7 +1353,7 @@ type InsightSeries struct { // JVMPackagesConnection description: Configuration for a connection to a JVM packages repository. type JVMPackagesConnection struct { // Maven description: Configuration for resolving from Maven repositories. - Maven *Maven `json:"maven,omitempty"` + Maven Maven `json:"maven"` } // Log description: Configuration for logging and alerting, including to external services. @@ -1366,7 +1366,7 @@ type Log struct { // Maven description: Configuration for resolving from Maven repositories. type Maven struct { - // Credentials description: Contents of a coursier.credentials file needed for accessing the Maven repositories. + // Credentials description: Contents of a coursier.credentials file needed for accessing the Maven repositories. See the 'Inline' section at https://get-coursier.io/docs/other-credentials#inline for more details. Credentials string `json:"credentials,omitempty"` // Dependencies description: An array of artifact "groupID:artifactID:version" strings specifying which Maven artifacts to mirror on Sourcegraph. Dependencies []string `json:"dependencies,omitempty"` @@ -1865,7 +1865,7 @@ type RubyRateLimit struct { type RustPackagesConnection struct { // Dependencies description: An array of strings specifying Rust packages to mirror in Sourcegraph. Dependencies []string `json:"dependencies,omitempty"` - // IndexRepositoryName description: Name of the git repository containing the crates.io index. Empty by default, which means no syncing happens. Updating this setting does not trigger a sync immediately, you must wait until the next scheduled sync for the value to get picked up. + // IndexRepositoryName description: Name of the git repository containing the crates.io index. Only set if you intend to sync every crate from the index. Updating this setting does not trigger a sync immediately, you must wait until the next scheduled sync for the value to get picked up. IndexRepositoryName string `json:"indexRepositoryName,omitempty"` // IndexRepositorySyncInterval description: How frequently to sync with the index repository. String formatted as a Go time.Duration. The Sourcegraph server needs to be restarted to pick up a new value of this configuration option. IndexRepositorySyncInterval string `json:"indexRepositorySyncInterval,omitempty"`