doc: dependencies search beta (#32612)

This commit adds documentation for the dependencies search beta feature. It also changes the feature flag to be enabled by default.

Co-authored-by: benvenker <bvemails@gmail.com>
Co-authored-by: Ólafur Páll Geirsson <olafurpg@gmail.com>

CHANGELOG.md

@@ -25,6 +25,7 @@ All notable changes to Sourcegraph are documented in this file.
 - Pings now include IDE extensions usage metrics [#32000](https://github.com/sourcegraph/sourcegraph/pull/32000)
 - New EventSource type: `IDEEXTENSION` for IDE extensions-related events [#32000](https://github.com/sourcegraph/sourcegraph/pull/32000)
 - Code Monitoring now has a Logs tab enabled as a [beta feature](https://docs.sourcegraph.com/admin/beta_and_experimental_features). This lets you see recent runs of your code monitors and determine if any notifications were sent or if there were any errors during the run. [#32292](https://github.com/sourcegraph/sourcegraph/pull/32292)
+- New `repo:dependencies(...)` predicate allows you to [search through the dependencies of your repositories](https://docs.sourcegraph.com/code_search/how-to/dependencies_search). This feature is currently in beta and only npm package repositories are supported with dependencies from `package-lock.json` and `yarn.lock` files. [#32405](https://github.com/sourcegraph/sourcegraph/issues/32405)
 
 ### Changed
 

client/web/src/repo/tree/TreePage.tsx

@@ -326,7 +326,7 @@ export const TreePage: React.FunctionComponent<Props> = ({
         </div>
     )
 
-    const dependenciesSearchEnabled = window.context?.experimentalFeatures?.dependenciesSearch ?? false
+    const dependenciesSearchEnabled = window.context?.experimentalFeatures?.dependenciesSearch !== 'disabled'
     const repoDepsSearchQueryURL = buildSearchURLQuery(
         `repo:deps(${repoFilterForRepoRevision(repo.name, false, revision)})`,
         SearchPatternType.literal,

dev/gqltest/search_test.go

@@ -1360,30 +1360,7 @@ func testDependenciesSearch(client, streamClient searchClient) func(*testing.T)
 	return func(t *testing.T) {
 		t.Helper()
 
-		cfg, err := client.SiteConfiguration()
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		oldConfig := *cfg
-
-		if cfg.ExperimentalFeatures == nil {
-			cfg.ExperimentalFeatures = &schema.ExperimentalFeatures{}
-		}
-
-		if cfg.ExperimentalFeatures.NpmPackages != "enabled" {
-			cfg.ExperimentalFeatures.NpmPackages = "enabled"
-		}
-
-		if !cfg.ExperimentalFeatures.DependenciesSearch {
-			cfg.ExperimentalFeatures.DependenciesSearch = true
-		}
-
-		if err = client.UpdateSiteConfiguration(cfg); err != nil {
-			t.Fatal(err)
-		}
-
-		_, err = client.AddExternalService(gqltestutil.AddExternalServiceInput{
+		_, err := client.AddExternalService(gqltestutil.AddExternalServiceInput{
 			Kind:        extsvc.KindNpmPackages,
 			DisplayName: "gqltest-npm-search",
 			Config: mustMarshalJSONString(&schema.NpmPackagesConnection{
@@ -1397,13 +1374,6 @@ func testDependenciesSearch(client, streamClient searchClient) func(*testing.T)
 			t.Fatal(err)
 		}
 
-		// Set up a npm external service to test dependencies search
-		t.Cleanup(func() {
-			if err := client.UpdateSiteConfiguration(&oldConfig); err != nil {
-				t.Fatal(err)
-			}
-		})
-
 		err = client.WaitForReposToBeCloned("npm/urql")
 		if err != nil {
 			t.Fatal(err)

doc/code_search/how-to/dependencies_search.md

@@ -0,0 +1,35 @@
+# Dependencies search <span class="badge badge-beta">beta</span>
+
+Dependencies search is a code search feature that lets you search through the dependencies of your repositories.
+
+
+<img src="https://storage.googleapis.com/sourcegraph-assets/docs/images/code_search/dependencies-search-usage.png" style="margin-left:0;margin-right:0;"/>
+
+### Setup
+
+Configure a package host connection for each kind of dependency you want to search over. Currently only [npm dependencies](../../integration/npm.md) are supported.
+
+### Use cases
+
+Resolve an incident faster by [quickly finding where an error comes from](https://sourcegraph.com/search?q=context:global+repo:deps%28%5Egithub%5C.com/sourcegraph/sourcegraph%24%403.37%29+Observable+cannot+be+called+as+a+function&patternType=literal) and then understanding the code around it by leveraging [code intelligence](../../code_intelligence/explanations/features.md).<br/>
+
+```sgquery
+r:deps(^github\.com/sourcegraph/sourcegraph$@3.37) Observable cannot be called as a function
+```
+
+### Compatibility
+
+The following table outlines the kinds of dependency repositories that dependency search supports how it finds those dependencies in your repositories.
+
+Kind | How | Supported
+---- | ------ | ---------
+[npm](../../integration/npm.md) | `package-lock.json` | ✅
+[npm](../../integration/npm.md) | `yarn.lock` | ✅
+[JVM](../../integration/jvm.md) | `gradle.lockfile` | ❌
+[JVM](../../integration/jvm.md) | `pom.xml` | ❌
+Go | `go.sum` | ❌
+Python | `poetry.lock` | ❌
+
+### Reference
+
+- [`repo:dependencies(...)`](../reference/language.md#repo-dependencies)

doc/code_search/index.md

@@ -51,6 +51,7 @@ Sourcegraph code search helps developers perform these tasks more quickly and ef
   - Curate [saved searches](explanations/features.md#saved-searches) for yourself or your org.
   - Use [Code monitoring](../code_monitoring/index.md) to set up notifications for code changes that match a query.
   - View [language statistics](explanations/features.md#statistics) for search results.
+  - Search through [your dependencies](how-to/dependencies_search.md).
 - [Search details](explanations/search_details.md)
 - [Sourcegraph Cloud](explanations/sourcegraph_cloud.md)
 - [Search tips](explanations/tips.md)

doc/code_search/reference/language.md

@@ -618,7 +618,8 @@ ComplexDiagram(
         Terminal("contains.content(...)", {href: "#repo-contains-content"}),
         Terminal("contains.file(...)", {href: "#repo-contains-file"}),
         Terminal("contains(...)", {href: "#repo-contains-file-and-content"}),
-        Terminal("contains.commit.after(...)", {href: "#repo-contains-commit-after"}))).addTo();
+        Terminal("contains.commit.after(...)", {href: "#repo-contains-commit-after"}),
+        Terminal("dependencies(...)", {href: "#repo-dependencies"}))).addTo();
 </script>
 
 ### Repo contains file
@@ -675,13 +676,33 @@ ComplexDiagram(
     Terminal(")")).addTo();
 </script>
 
-Search only inside repositories that contain a a commit after some specified
+Search only inside repositories that contain a commit after some specified
 time. See [git date formats](https://github.com/git/git/blob/master/Documentation/date-formats.txt)
 for accepted formats. Use this to filter out stale repositories that don’t contain
 commits past the specified time frame. This parameter is experimental.
 
 **Example:** [`repo:contains.commit.after(1 month ago)` ↗](https://sourcegraph.com/search?q=repo:.*sourcegraph.*+repo:contains.commit.after%281+month+ago%29&patternType=literal)
 
+### Repo dependencies
+
+<script>
+ComplexDiagram(
+    Choice(0,
+        Terminal("dependencies:"),
+        Terminal("deps:")),
+    Terminal("("),
+    Sequence(
+        Terminal("regexp", {href: "#regular-expression"}),
+        Terminal("@"),
+        Terminal("revision", {href: "#revision"})
+    ),
+    Terminal(")")).addTo();
+</script>
+
+Search only inside dependencies of repositories matching the given `regex@rev:a:b:c` input.
+
+**Example:** [`repo:dependencies(^github\.com/sourcegraph/sourcegraph$@3.36:3.35) count:all` ↗](https://sourcegraph.com/search?q=context:global+repo:dependencies%28%5Egithub%5C.com/sourcegraph/sourcegraph%24%403.36:3.35%29+count:all&patternType=literal)
+
 ## Built-in file predicate
 
 <script>

doc/code_search/reference/queries.md

@@ -89,7 +89,8 @@ The following keywords can be used on all searches (using [RE2 syntax](https://g
 | **archived:yes, archived:only** | The yes option, includes archived repositories. The only option, filters results to only archived repositories. Results in archived repositories are excluded by default. | [`repo:sourcegraph/ archived:only`](https://sourcegraph.com/search?q=repo:%5Egithub.com/sourcegraph/+archived:only) |
 | **repo:contains.file(...)** | Conditionally search inside repositories only if they contain a file path matching the regular expression. See [built-in predicates](language.md#built-in-predicate) for more. | [`repo:contains.file(\.py) file:Dockerfile pip`](https://sourcegraph.com/search?q=repo:.*sourcegraph.*+repo:contains.file%28%5C.py%29+file:Dockerfile+pip&patternType=literal) |
 | **-repohasfile:regexp-pattern** | Exclude results from repositories that contain a matching file. This keyword is a pure filter, so it requires at least one other search term in the query. Note: this filter currently only works on text matches and file path matches. | [`-repohasfile:Dockerfile docker`](https://sourcegraph.com/search?q=-repohasfile:Dockerfile+docker) |
-| **repo:contains.commit.after(...)** | (Experimental) Filter out stale repositories that don't contain commits past the specified time frame. | [`repo:contains.commit.after(yesterday)`](https://sourcegraph.com/search?q=repo:.*sourcegraph.*+repo:contains.commit.after%28yesterday%29&patternType=literal) <br> [`repo:contains.commit.after(june 25 2017)`](https://sourcegraph.com/search?q=repo:.*sourcegraph.*+repo:contains.commit.after%28june+25+2017%29&patternType=literal) |
+| **repo:contains.commit.after(...)** | Filter out stale repositories that don't contain commits past the specified time frame. | [`repo:contains.commit.after(yesterday)`](https://sourcegraph.com/search?q=repo:.*sourcegraph.*+repo:contains.commit.after%28yesterday%29&patternType=literal) <br> [`repo:contains.commit.after(june 25 2017)`](https://sourcegraph.com/search?q=repo:.*sourcegraph.*+repo:contains.commit.after%28june+25+2017%29&patternType=literal) |
+| **repo:dependencies(...)** <br> _alias: r:deps(...)_ | (Beta) Search the dependencies of the repositories matching the provided `regexp@rev:a:b:c`. | [`repo:deps(^github\.com/sourcegraph/sourcegraph$@3.36:3.35)`](https://sourcegraph.com/search?q=context:global+repo:deps%28%5Egithub%5C.com/sourcegraph/sourcegraph%24%403.36:3.35%29+count:all&patternType=literal) <br>  |
 | **file:contains(...)** | Conditionally search files only if they contain contents that match the provided regex pattern. | [`file:contains(Copyright) Sourcegraph`](https://sourcegraph.com/search?q=context:global+file:contains%28Copyright%29+Sourcegraph&patternType=literal) |
 | **count:_N_,<br> count:all**<br/> | Retrieve <em>N</em> results. By default, Sourcegraph stops searching early and returns if it finds a full page of results. This is desirable for most interactive searches. To wait for all results, use **count:all**. | [`count:1000 function`](https://sourcegraph.com/search?q=count:1000+repo:sourcegraph/sourcegraph$+function) <br> [`count:all err`](https://sourcegraph.com/search?q=repo:github.com/sourcegraph/sourcegraph+err+count:all&patternType=literal) |
 | **timeout:_go-duration-value_**<br/> | Customizes the timeout for searches. The value of the parameter is a string that can be parsed by the [Go time package's `ParseDuration`](https://golang.org/pkg/time/#ParseDuration) (e.g. 10s, 100ms). By default, the timeout is set to 10 seconds, and the search will optimize for returning results as soon as possible. The timeout value cannot be set longer than 1 minute. When provided, the search is given the full timeout to complete. | [`repo:^github.com/sourcegraph timeout:15s func count:10000`](https://sourcegraph.com/search?q=repo:%5Egithub.com/sourcegraph/+timeout:15s+func+count:10000) |

doc/code_search/tutorials/examples.md

@@ -52,6 +52,12 @@ repo:@*refs/heads/:^master type:diff after:"1 week ago" (eslint-disable)
 file:package.json type:diff after:"1 week ago"
 ```
 
+[Search an error message from your dependencies](https://sourcegraph.com/search?q=context:global+repo:deps%28%5Egithub%5C.com/sourcegraph/sourcegraph%24%403.37%29+Observable+cannot+be+called+as+a+function&patternType=literal)<br/>
+
+```sgquery
+repo:deps(^github\.com/sourcegraph/sourcegraph$@3.37) Observable cannot be called as a function
+```
+
 [Files that are Apache licensed](https://sourcegraph.com/search?q=licensed+to+the+apache+software+foundation+select:file&patternType=literal)<br/>
 
 ```sgquery

doc/integration/npm.md

@@ -12,9 +12,10 @@ Feature | Supported?
 
 ## Repository syncing
 
-There are two ways to sync npm dependency repositories.
+There are three ways to sync npm dependency repositories.
 
 * **LSIF** (recommended): run [`lsif-node`](https://github.com/sourcegraph/lsif-node) against your JS/TS codebase and upload the generated index to Sourcegraph using the  [src-cli](https://github.com/sourcegraph/src-cli) command `src lsif upload`. Sourcegraph automatically synchronizes npm dependency repositories based on the dependencies that are discovered by `lsif-node`.
+* **Dependencies search**: Sourcegraph automatically synchronizes npm dependency repositories that are in `package-lock.json` or `yarn.lock` files during a [dependencies search](../code_search/how-to/dependencies_search.md).
 * **Code host configuration**: manually list dependencies in the `"dependencies"` section of the JSON configuration when creating the npm dependency code host. This method can be useful to verify that the credentials are picked up correctly without having to upload LSIF.
 
 ## Credentials

internal/conf/computed.go

@@ -252,6 +252,14 @@ func StructuralSearchEnabled() bool {
 	return val == "enabled"
 }
 
+func DependeciesSearchEnabled() bool {
+	val := ExperimentalFeatures().DependenciesSearch
+	if val == "" {
+		return true
+	}
+	return val == "enabled"
+}
+
 func ExperimentalFeatures() schema.ExperimentalFeatures {
 	val := Get().ExperimentalFeatures
 	if val == nil {

internal/search/repos/repos.go

@@ -529,7 +529,7 @@ func (r *Resolver) dependencies(ctx context.Context, op *search.RepoOptions) (_
 		tr.Finish()
 	}()
 
-	if !conf.ExperimentalFeatures().DependenciesSearch {
+	if !conf.DependeciesSearchEnabled() {
 		return nil, nil, errors.Errorf("support for `repo:dependencies()` is disabled in site config (`experimentalFeatures.dependenciesSearch`)")
 	}
 

schema/schema.go

@@ -570,7 +570,7 @@ type ExperimentalFeatures struct {
 	// DebugLog description: Turns on debug logging for specific debugging scenarios.
 	DebugLog *DebugLog `json:"debug.log,omitempty"`
 	// DependenciesSearch description: Enables support for repo:dependencies predicate queries.
-	DependenciesSearch bool `json:"dependenciesSearch,omitempty"`
+	DependenciesSearch string `json:"dependenciesSearch,omitempty"`
 	// EnableGitServerCommandExecFilter description: Enable filtering of all exec commands on gitserver based on a pre-defined allowlist
 	EnableGitServerCommandExecFilter bool `json:"enableGitServerCommandExecFilter,omitempty"`
 	// EnableGithubInternalRepoVisibility description: Enable support for visilibity of internal Github repositories

schema/site.schema.json

@@ -97,8 +97,9 @@
         },
         "dependenciesSearch": {
           "description": "Enables support for repo:dependencies predicate queries.",
-          "type": "boolean",
-          "default": false
+          "type": "string",
+          "enum": ["enabled", "disabled"],
+          "default": "enabled"
         },
         "andOrQuery": {
           "description": "DEPRECATED: Interpret a search input query as an and/or query.",