Codenav: use new occurrences API for symbol definitions (#63217)

This integrates the new occurrences API into the Svelte webapp. This
fixes a number of issues where the syntax highlighting data is not an
accurate way to determine hoverable tokens. It is currently behind the
setting `experimentalFeatures.enablePreciseOccurrences`

client/shared/src/codeintel/scip.ts

@@ -20,6 +20,18 @@ export interface JsonOccurrence {
     symbolRoles?: number
 }
 
+// Copied from https://github.com/sourcegraph/scip/blob/62966697fbeaccaaf87dab3870c85048d801ca68/scip.proto#L500
+export enum SymbolRole {
+    Unspecified = 0,
+    Definition = 1,
+    Import = 2,
+    WriteAccess = 4,
+    ReadAccess = 8,
+    Generated = 16,
+    Test = 32,
+    ForwardDefinition = 64,
+}
+
 export class Position implements sourcegraph.Position {
     constructor(public readonly line: number, public readonly character: number) {}
 
@@ -168,7 +180,7 @@ export class Occurrence {
 // non-overlapping occurrences.  The most narrow occurrence "wins", meaning that
 // when two ranges overlap, we pick the syntax kind of the occurrence with the
 // shortest distance between start/end.
-function nonOverlappingOccurrences(occurrences: Occurrence[]): Occurrence[] {
+export function nonOverlappingOccurrences(occurrences: Occurrence[]): Occurrence[] {
     // NOTE: we can't guarantee that the occurrences are sorted from the server
     // or after splitting multiline occurrences into single-line occurrences.
     const stack: Occurrence[] = occurrences.sort((a, b) => a.range.compare(b.range)).reverse()

client/web-sveltekit/src/lib/CodeMirrorBlob.svelte

@@ -129,6 +129,11 @@
     import { EditorView } from '@codemirror/view'
     import { createEventDispatcher, onMount } from 'svelte'
 
+    import {
+        codeGraphData as codeGraphDataFacet,
+        type CodeGraphData,
+    } from '@sourcegraph/web/src/repo/blob/codemirror/codeintel/occurrences'
+
     import { browser } from '$app/environment'
     import { goto } from '$app/navigation'
     import type { LineOrPositionOrRange } from '$lib/common'
@@ -167,6 +172,7 @@
 
     export let blobInfo: BlobInfo
     export let highlights: string
+    export let codeGraphData: CodeGraphData[] = []
     export let wrapLines: boolean = false
     export let selectedLines: LineOrPositionOrRange | null = null
     export let codeIntelAPI: CodeIntelAPI | null
@@ -197,6 +203,7 @@
         blameDataExtension: null,
         blameColumnExtension: null,
         searchExtension: null,
+        codeGraph: null,
     })
     const useFileSearch = createLocalWritable('blob.overrideBrowserFindOnPage', true)
     registerHotkey({
@@ -249,6 +256,7 @@
         : null
     $: lineWrapping = wrapLines ? EditorView.lineWrapping : null
     $: syntaxHighlighting = highlights ? syntaxHighlight.of({ content: blobInfo.content, lsif: highlights }) : null
+    $: codeGraph = codeGraphDataFacet.of(codeGraphData)
     $: staticHighlightExtension = staticHighlights(staticHighlightRanges)
     $: searchExtension = search({
         overrideBrowserFindInPageShortcut: $useFileSearch,
@@ -282,6 +290,7 @@
             codeIntelExtension,
             lineWrapping,
             syntaxHighlighting,
+            codeGraph,
             staticHighlightExtension,
             blameDataExtension,
             searchExtension,
@@ -320,6 +329,7 @@
                     codeIntelExtension,
                     lineWrapping,
                     syntaxHighlighting,
+                    codeGraph,
                     staticHighlightExtension,
                     blameDataExtension,
                     blameColumnExtension,

client/web-sveltekit/src/routes/[...repo=reporev]/(validrev)/(code)/-/blob/[...path]/+page.ts

@@ -1,19 +1,31 @@
 import { BehaviorSubject, concatMap, from, map } from 'rxjs'
 
+import {
+    Occurrence,
+    Range,
+    Position,
+    SymbolRole,
+    nonOverlappingOccurrences,
+} from '@sourcegraph/shared/src/codeintel/scip'
 import { type BlameHunkData, fetchBlameHunksMemoized } from '@sourcegraph/web/src/repo/blame/shared'
+import type { CodeGraphData } from '@sourcegraph/web/src/repo/blob/codemirror/codeintel/occurrences'
 
 import { SourcegraphURL } from '$lib/common'
-import { getGraphQLClient, mapOrThrow } from '$lib/graphql'
+import { getGraphQLClient, mapOrThrow, type GraphQLClient } from '$lib/graphql'
+import { SymbolRole as GraphQLSymbolRole } from '$lib/graphql-types'
 import { resolveRevision } from '$lib/repo/utils'
 import { parseRepoRevision } from '$lib/shared'
 import { assertNonNullable } from '$lib/utils'
 
 import type { PageLoad, PageLoadEvent } from './$types'
+import type { FileViewCodeGraphData } from './FileView.gql'
 import {
     BlobDiffViewCommitQuery,
     BlobFileViewBlobQuery,
+    BlobFileViewCodeGraphDataQuery,
     BlobFileViewCommitQuery_revisionOverride,
     BlobFileViewHighlightedFileQuery,
+    BlobViewCodeGraphDataNextPage,
 } from './page.gql'
 
 function loadDiffView({ params, url }: PageLoadEvent) {
@@ -39,6 +51,92 @@ function loadDiffView({ params, url }: PageLoadEvent) {
     }
 }
 
+async function fetchCodeGraphData(
+    client: GraphQLClient,
+    repoName: string,
+    resolvedRevision: string,
+    path: string
+): Promise<CodeGraphData[]> {
+    async function fetchAllOccurrences(codeGraphDatum: FileViewCodeGraphData): Promise<FileViewCodeGraphData> {
+        while (codeGraphDatum.occurrences?.pageInfo?.hasNextPage) {
+            const response = await client.query(BlobViewCodeGraphDataNextPage, {
+                codeGraphDataID: codeGraphDatum.id,
+                after: codeGraphDatum.occurrences?.pageInfo?.endCursor ?? '',
+            })
+            if (response.error) {
+                throw new Error('failed to hydrate paginated occurrences', { cause: response.error })
+            }
+            if (response.data?.node?.__typename !== 'CodeGraphData') {
+                throw new Error('unexpected node')
+            }
+            codeGraphDatum.occurrences = {
+                nodes: [...codeGraphDatum.occurrences.nodes, ...(response.data.node.occurrences?.nodes ?? [])],
+                pageInfo: response.data.node.occurrences?.pageInfo ?? {
+                    __typename: 'PageInfo',
+                    hasNextPage: false,
+                    endCursor: null,
+                },
+            }
+        }
+        return codeGraphDatum
+    }
+
+    function translateRole(graphQLRole: GraphQLSymbolRole): SymbolRole {
+        switch (graphQLRole) {
+            case GraphQLSymbolRole.DEFINITION:
+                return SymbolRole.Definition
+            case GraphQLSymbolRole.REFERENCE:
+                // The REFERENCE role from the API is just the negation of the
+                // DEFINITION role, so simply do not set the definition bit.
+                return SymbolRole.Unspecified
+            case GraphQLSymbolRole.FORWARD_DEFINITION:
+                return SymbolRole.ForwardDefinition
+            default:
+                return SymbolRole.Unspecified
+        }
+    }
+
+    const response = await client.query(BlobFileViewCodeGraphDataQuery, {
+        repoName,
+        revspec: resolvedRevision,
+        path,
+    })
+    if (response.error) {
+        throw new Error('failed fetching code graph data', { cause: response.error })
+    }
+
+    const rawCodeGraphData = response.data?.repository?.commit?.blob?.codeGraphData
+    if (!rawCodeGraphData) {
+        return []
+    }
+    // Fetch any additional pages of occurrences
+    const hydratedCodeGraphData = await Promise.all([...rawCodeGraphData.map(fetchAllOccurrences)])
+
+    return hydratedCodeGraphData.map(({ provenance, toolInfo, commit, occurrences }) => {
+        const overlapping =
+            occurrences?.nodes?.map(
+                occ =>
+                    new Occurrence(
+                        new Range(
+                            new Position(occ.range.start.line, occ.range.start.character),
+                            new Position(occ.range.end.line, occ.range.end.character)
+                        ),
+                        undefined,
+                        occ.symbol ?? undefined,
+                        occ.roles?.map(translateRole).reduce((acc, role) => acc | role, 0)
+                    )
+            ) ?? []
+        const nonOverlapping = nonOverlappingOccurrences([...overlapping])
+        return {
+            provenance,
+            toolInfo,
+            commit,
+            occurrences: overlapping,
+            nonOverlappingOccurrences: nonOverlapping,
+        }
+    })
+}
+
 async function loadFileView({ parent, params, url }: PageLoadEvent) {
     const client = getGraphQLClient()
     const revisionOverride = url.searchParams.get('rev')
@@ -95,6 +193,14 @@ async function loadFileView({ parent, params, url }: PageLoadEvent) {
                 })
             )
             .then(mapOrThrow(result => result.data?.repository?.commit?.blob?.highlight ?? null)),
+        codeGraphData: resolvedRevision.then(async resolvedRevision => {
+            console.log((await parent()).settings.experimentalFeatures)
+            if ((await parent()).settings.experimentalFeatures?.enablePreciseOccurrences ?? false) {
+                return fetchCodeGraphData(client, repoName, resolvedRevision, filePath)
+            } else {
+                return []
+            }
+        }),
         // We can ignore the error because if the revision doesn't exist, other queries will fail as well
         revisionOverride: revisionOverride
             ? await client

client/web-sveltekit/src/routes/[...repo=reporev]/(validrev)/(code)/-/blob/[...path]/FileView.gql

@@ -30,3 +30,38 @@ fragment FileViewCommit on GitCommit {
     canonicalURL
     abbreviatedOID
 }
+
+fragment FileViewCodeGraphData on CodeGraphData {
+    id
+    provenance
+    commit
+    toolInfo {
+        name
+        version
+    }
+
+    occurrences(first: 10000) {
+        nodes {
+            ...FileViewOccurrence
+        }
+        pageInfo {
+            endCursor
+            hasNextPage
+        }
+    }
+}
+
+fragment FileViewOccurrence on SCIPOccurrence {
+    symbol
+        range {
+            start {
+                line
+                    character
+            }
+            end {
+                line
+                    character
+            }
+        }
+    roles
+}

client/web-sveltekit/src/routes/[...repo=reporev]/(validrev)/(code)/-/blob/[...path]/FileView.svelte

@@ -8,6 +8,7 @@
     import { writable } from 'svelte/store'
 
     import { noOpTelemetryRecorder } from '@sourcegraph/shared/src/telemetry'
+    import type { CodeGraphData } from '@sourcegraph/web/src/repo/blob/codemirror/codeintel/occurrences'
 
     import { goto, preloadData, afterNavigate } from '$app/navigation'
     import { page } from '$app/stores'
@@ -49,9 +50,11 @@
     const lineWrap = writable<boolean>(false)
     const blobLoader = createPromiseStore<Awaited<PageData['blob']>>()
     const highlightsLoader = createPromiseStore<Awaited<PageData['highlights']>>()
+    const codeGraphDataLoader = createPromiseStore<Awaited<PageData['codeGraphData']>>()
 
     let blob: FileViewGitBlob | null = null
     let highlights: FileViewHighlightedFile | null = null
+    let codeGraphData: CodeGraphData[] | null = null
     let cmblob: CodeMirrorBlob | null = null
     let initialScrollPosition: ScrollSnapshot | null = null
     let selectedPosition: LineOrPositionOrRange | null = null
@@ -67,12 +70,14 @@
     } = data)
     $: blobLoader.set(data.blob)
     $: highlightsLoader.set(data.highlights)
+    $: codeGraphDataLoader.set(data.codeGraphData)
 
     $: if (!$blobLoader.pending) {
         // Only update highlights and position after the file content has been loaded.
         // While the file content is loading we show the previous file content.
         blob = $blobLoader.value ?? null
         highlights = $highlightsLoader.pending ? null : $highlightsLoader.value ?? null
+        codeGraphData = $codeGraphDataLoader.pending ? null : $codeGraphDataLoader.value ?? null
         selectedPosition = data.lineOrPosition
     }
     $: fileLoadingError = !$blobLoader.pending && $blobLoader.error
@@ -268,6 +273,7 @@
                     filePath,
                 }}
                 highlights={highlights?.lsif ?? ''}
+                codeGraphData={codeGraphData ?? undefined}
                 showBlame={showBlameView}
                 blameData={$blameData}
                 wrapLines={$lineWrap}

client/web-sveltekit/src/routes/[...repo=reporev]/(validrev)/(code)/-/blob/[...path]/page.gql

@@ -48,6 +48,36 @@ query BlobFileViewHighlightedFileQuery(
     }
 }
 
+query BlobFileViewCodeGraphDataQuery($repoName: String!, $revspec: String!, $path: String!) {
+    repository(name: $repoName) {
+        id
+        commit(rev: $revspec) {
+            id
+            blob(path: $path) {
+                codeGraphData {
+                    ...FileViewCodeGraphData
+                }
+            }
+        }
+    }
+}
+
+query BlobViewCodeGraphDataNextPage($codeGraphDataID: ID!, $after: String!) {
+    node(id: $codeGraphDataID) {
+        ...on CodeGraphData {
+            occurrences(first: 10000, after: $after){
+                nodes {
+                    ...FileViewOccurrence
+                }
+                pageInfo {
+                    endCursor
+                    hasNextPage
+                }
+            }
+        }
+    }
+}
+
 query BlobFileViewCommitQuery_revisionOverride($repoName: String!, $revspec: String!) {
     repository(name: $repoName) {
         commit(rev: $revspec) {

client/web/BUILD.bazel

@@ -1291,6 +1291,7 @@ ts_project(
         "src/repo/blob/codemirror/codeintel/hover.ts",
         "src/repo/blob/codemirror/codeintel/keybindings.ts",
         "src/repo/blob/codemirror/codeintel/modifier-key.ts",
+        "src/repo/blob/codemirror/codeintel/occurrences.ts",
         "src/repo/blob/codemirror/codeintel/pin.ts",
         "src/repo/blob/codemirror/codeintel/token-selection.ts",
         "src/repo/blob/codemirror/codeintel/tooltips.ts",

client/web/src/repo/blob/codemirror/codeintel/api.ts

@@ -21,8 +21,7 @@ import type { WebHoverOverlayProps } from '../../../../components/WebHoverOverla
 import { syntaxHighlight } from '../highlight'
 import {
     contains,
-    isInteractiveOccurrence,
-    occurrenceAt,
+    interactiveOccurrenceAt,
     positionAtCmPosition,
     rangeToCmSelection,
     closestOccurrenceByCharacter,
@@ -138,10 +137,7 @@ export class CodeIntelAPIAdapter {
             return fromCache
         }
 
-        let occurrence = occurrenceAt(state, offset) ?? null
-        if (occurrence && !isInteractiveOccurrence(occurrence)) {
-            occurrence = null
-        }
+        const occurrence = interactiveOccurrenceAt(state, offset) ?? null
         const range = occurrence ? rangeToCmSelection(state.doc, occurrence.range) : null
         for (let i = range?.from ?? offset, to = range?.to ?? offset; i <= to; i++) {
             this.occurrenceCache.set(offset, { occurrence, range })
@@ -275,10 +271,7 @@ export class CodeIntelAPIAdapter {
                               .join('\n\n----\n\n')
                               .trimEnd()
                 const precise = isPrecise(result)
-                if (!precise && markdownContents.length > 0 && !isInteractiveOccurrence(occurrence)) {
-                    return null
-                }
-                if (markdownContents === '' && isInteractiveOccurrence(occurrence)) {
+                if (markdownContents === '') {
                     markdownContents = 'No hover information available'
                 }
                 return markdownContents

client/web/src/repo/blob/codemirror/codeintel/occurrences.ts

@@ -0,0 +1,25 @@
+import { Facet } from '@codemirror/state'
+
+import { Occurrence } from '@sourcegraph/shared/src/codeintel/scip'
+
+export interface CodeGraphData {
+    provenance: string
+    commit: string
+    toolInfo: {
+        name: string | null
+        version: string | null
+    } | null
+    // The raw occurrences as returned by the API. Guaranteed to be sorted.
+    occurrences: Occurrence[]
+    // The same as occurrences, but flattened so there are no overlapping
+    // ranges. Guaranteed to be sorted.
+    nonOverlappingOccurrences: Occurrence[]
+}
+
+// A facet that contains the precise code graph data from the occurrences API.
+// It just retains the most recent contribution. At some point, we should
+// probably extend this to be able to accept contributions from multiple
+// sources.
+export const codeGraphData = Facet.define<CodeGraphData[], CodeGraphData[]>({
+    combine: values => values[0] ?? [],
+})

client/web/src/repo/blob/codemirror/occurrence-utils.ts

@@ -3,6 +3,7 @@ import { EditorSelection, type Text, type EditorState, type SelectionRange } fro
 import type { Range } from '@sourcegraph/extension-api-types'
 import { Occurrence, Position, Range as ScipRange, SyntaxKind } from '@sourcegraph/shared/src/codeintel/scip'
 
+import { CodeGraphData, codeGraphData } from './codeintel/occurrences'
 import { type HighlightIndex, syntaxHighlight } from './highlight'
 
 /**
@@ -25,7 +26,7 @@ const INTERACTIVE_OCCURRENCE_KINDS = new Set([
     SyntaxKind.IdentifierAttribute,
 ])
 
-export const isInteractiveOccurrence = (occurrence: Occurrence): boolean => {
+const isInteractiveOccurrence = (occurrence: Occurrence): boolean => {
     if (!occurrence.kind) {
         return false
     }
@@ -33,23 +34,31 @@ export const isInteractiveOccurrence = (occurrence: Occurrence): boolean => {
     return INTERACTIVE_OCCURRENCE_KINDS.has(occurrence.kind)
 }
 
-export function occurrenceAt(state: EditorState, offset: number): Occurrence | undefined {
-    // First we try to get an occurrence from syntax highlighting data.
-    const fromHighlighting = highlightingOccurrenceAtPosition(state, offset)
-    if (fromHighlighting) {
+export function interactiveOccurrenceAt(state: EditorState, offset: number): Occurrence | undefined {
+    const position = positionAtCmPosition(state.doc, offset)
+
+    // First we try to get an occurrence from the occurrences API
+    const data = state.facet(codeGraphData)
+    if (data.length > 0) {
+        return scipOccurrenceAtPosition(data, position)
+    }
+
+    // Next we try to get an occurrence from syntax highlighting data.
+    const fromHighlighting = highlightingOccurrenceAtPosition(state, position)
+    if (fromHighlighting && isInteractiveOccurrence(fromHighlighting)) {
         return fromHighlighting
     }
+
     // If the syntax highlighting data is incomplete then we fallback to a
     // heursitic to infer the occurrence.
-    return inferOccurrenceAtPosition(state, offset)
+    return inferOccurrenceAtOffset(state, offset)
 }
 
 // Returns the occurrence at this position based on syntax highlighting data.
 // The highlighting data can come from Syntect (low-ish quality) or tree-sitter
 // (better quality). When we implement semantic highlighting in the future, the
 // highlighting data may come from precise indexers.
-function highlightingOccurrenceAtPosition(state: EditorState, offset: number): Occurrence | undefined {
-    const position = positionAtCmPosition(state.doc, offset)
+function highlightingOccurrenceAtPosition(state: EditorState, position: Position): Occurrence | undefined {
     const table = state.facet(syntaxHighlight)
     for (
         let index = table.lineIndex[position.line];
@@ -66,12 +75,33 @@ function highlightingOccurrenceAtPosition(state: EditorState, offset: number): O
     return undefined
 }
 
+// Returns the occurrence at this position based on data from the GraphQL occurrences API.
+function scipOccurrenceAtPosition(data: CodeGraphData[], position: Position): Occurrence | undefined {
+    for (const datum of data) {
+        // Binary search over the sorted, non-overlapping ranges.
+        const arr = datum.nonOverlappingOccurrences
+        let [low, high] = [0, arr.length]
+        while (low < high) {
+            const mid = Math.floor((low + high) / 2)
+            if (arr[mid].range.contains(position)) {
+                return arr[mid]
+            }
+            if (arr[mid].range.end.compare(position) < 0) {
+                low = mid + 1
+            } else {
+                high = mid
+            }
+        }
+    }
+    return undefined
+}
+
 // Returns the occurrence at this position based on CodeMirror's built-in
 // "wordAt" helper method.  This helper is a heuristic that works reasonably
 // well for languages with C/Java-like identifiers, but we may want to customize
 // the heurstic for other languages like Clojure where kebab-case identifiers
 // are common.
-function inferOccurrenceAtPosition(state: EditorState, offset: number): Occurrence | undefined {
+function inferOccurrenceAtOffset(state: EditorState, offset: number): Occurrence | undefined {
     const identifier = state.wordAt(offset)
     // We need to ignore words that end at the requested position to match the logic
     // we use to look up occurrences in SCIP data.

schema/schema.go

@@ -2543,6 +2543,8 @@ type SettingsExperimentalFeatures struct {
 	EnableLazyBlobSyntaxHighlighting *bool `json:"enableLazyBlobSyntaxHighlighting,omitempty"`
 	// EnableLazyFileResultSyntaxHighlighting description: Fetch un-highlighted file result contents to render immediately, decorate with syntax highlighting once loaded.
 	EnableLazyFileResultSyntaxHighlighting *bool `json:"enableLazyFileResultSyntaxHighlighting,omitempty"`
+	// EnablePreciseOccurrences description: Enable the new precise occurrences API, which can provide more accurate hovers for some languages.
+	EnablePreciseOccurrences bool `json:"enablePreciseOccurrences,omitempty"`
 	// EnableSearchFilePrefetch description: Pre-fetch plaintext file revisions from search results on hover/focus.
 	EnableSearchFilePrefetch *bool `json:"enableSearchFilePrefetch,omitempty"`
 	// EnableSidebarFilePrefetch description: Pre-fetch plaintext file revisions from sidebar on hover/focus.
@@ -2625,6 +2627,7 @@ func (v *SettingsExperimentalFeatures) UnmarshalJSON(data []byte) error {
 	delete(m, "codeMonitoringWebHooks")
 	delete(m, "enableLazyBlobSyntaxHighlighting")
 	delete(m, "enableLazyFileResultSyntaxHighlighting")
+	delete(m, "enablePreciseOccurrences")
 	delete(m, "enableSearchFilePrefetch")
 	delete(m, "enableSidebarFilePrefetch")
 	delete(m, "fuzzyFinder")

schema/settings.schema.json

@@ -227,6 +227,11 @@
           "description": "Whether to enable the 'keyword search' language improvement",
           "type": "boolean",
           "default": true
+        },
+        "enablePreciseOccurrences": {
+          "description": "Enable the new precise occurrences API, which can provide more accurate hovers for some languages.",
+          "type": "boolean",
+          "default": true
         }
       },
       "group": "Experimental"