artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 18:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

search frontend: basic regexp highlighting with smartQuery flag (#15644)

Browse Source
This commit is contained in:
Rijnard van Tonder 2020-11-11 15:25:20 -07:00 committed by GitHub
parent 5f5422a5b9
commit d37737a122
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 362 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
client/shared/src/search/parser/providers.ts
View File

@ -37,6 +37,7 @@ export function getProviders(
options: {
patternType: SearchPatternType
globbing: boolean
enableSmartQuery: boolean
interpretComments?: boolean
}
): SearchFieldProviders {
@ -55,10 +56,10 @@ export function getProviders(
tokens: {
getInitialState: () => SCANNER_STATE,
tokenize: line => {
const result = scanSearchQuery(line, options.interpretComments ?? false)
const result = scanSearchQuery(line, options.interpretComments ?? false, options.patternType)
if (result.type === 'success') {
return {
tokens: getMonacoTokens(result.term),
tokens: getMonacoTokens(result.term, options.enableSmartQuery),
endState: SCANNER_STATE,
}
}

9
client/shared/src/search/parser/scanner.test.ts
View File

@ -1,4 +1,5 @@
import { scanSearchQuery, scanBalancedPattern, PatternKind } from './scanner'
import { scanSearchQuery, scanBalancedPattern } from './scanner'
import { SearchPatternType } from '../../graphql-operations'
expect.addSnapshotSerializer({
serialize: value => JSON.stringify(value),
@ -157,7 +158,7 @@ describe('scanSearchQuery() for literal search', () => {
describe('scanSearchQuery() for regexp', () => {
test('interpret regexp pattern with match groups', () => {
expect(
scanSearchQuery('((sauce|graph)(\\s)?)is best(g*r*a*p*h*)', false, PatternKind.Regexp)
scanSearchQuery('((sauce|graph)(\\s)?)is best(g*r*a*p*h*)', false, SearchPatternType.regexp)
).toMatchInlineSnapshot(
'{"type":"success","term":[{"type":"pattern","range":{"start":0,"end":22},"kind":2,"value":"((sauce|graph)(\\\\s)?)is"},{"type":"whitespace","range":{"start":22,"end":23}},{"type":"pattern","range":{"start":23,"end":39},"kind":2,"value":"best(g*r*a*p*h*)"}]}'
)
@ -165,14 +166,14 @@ describe('scanSearchQuery() for regexp', () => {
test('interpret regexp pattern with match groups between keywords', () => {
expect(
scanSearchQuery('(((sauce|graph)\\s?) or (best)) and (gr|aph)', false, PatternKind.Regexp)
scanSearchQuery('(((sauce|graph)\\s?) or (best)) and (gr|aph)', false, SearchPatternType.regexp)
).toMatchInlineSnapshot(
'{"type":"success","term":[{"type":"openingParen","range":{"start":0,"end":1}},{"type":"pattern","range":{"start":1,"end":19},"kind":2,"value":"((sauce|graph)\\\\s?)"},{"type":"whitespace","range":{"start":19,"end":20}},{"type":"keyword","value":"or","range":{"start":20,"end":22},"kind":"or"},{"type":"whitespace","range":{"start":22,"end":23}},{"type":"pattern","range":{"start":23,"end":29},"kind":2,"value":"(best)"},{"type":"closingParen","range":{"start":29,"end":30}},{"type":"whitespace","range":{"start":30,"end":31}},{"type":"keyword","value":"and","range":{"start":31,"end":34},"kind":"and"},{"type":"whitespace","range":{"start":34,"end":35}},{"type":"pattern","range":{"start":35,"end":43},"kind":2,"value":"(gr|aph)"}]}'
)
})
test('interpret regexp slash quotes', () => {
expect(scanSearchQuery('r:a /a regexp \\ pattern/', false, PatternKind.Regexp)).toMatchInlineSnapshot(
expect(scanSearchQuery('r:a /a regexp \\ pattern/', false, SearchPatternType.regexp)).toMatchInlineSnapshot(
'{"type":"success","term":[{"type":"filter","range":{"start":0,"end":3},"filterType":{"type":"literal","value":"r","range":{"start":0,"end":1}},"filterValue":{"type":"literal","value":"a","range":{"start":2,"end":3}},"negated":false},{"type":"whitespace","range":{"start":3,"end":4}},{"type":"quoted","quotedValue":"a regexp \\\\ pattern","range":{"start":4,"end":24}}]}'
)
})

6
client/shared/src/search/parser/scanner.ts
View File

@ -1,5 +1,6 @@
import { IRange } from 'monaco-editor'
import { filterTypeKeysWithAliases } from '../interactive/util'
import { SearchPatternType } from '../../graphql-operations'
/**
* Defines common properties for tokens.
@ -547,8 +548,9 @@ const createScanner = (kind: PatternKind, interpretComments?: boolean): Scanner<
export const scanSearchQuery = (
query: string,
interpretComments?: boolean,
kind = PatternKind.Literal
searchPatternType = SearchPatternType.literal
): ScanResult<Token[]> => {
const scanner = createScanner(kind, interpretComments)
const patternKind = searchPatternType === SearchPatternType.regexp ? PatternKind.Regexp : PatternKind.Literal
const scanner = createScanner(patternKind, interpretComments)
return scanner(query, 0)
}

118
client/shared/src/search/parser/tokens.test.ts
View File

@ -1,5 +1,11 @@
import { getMonacoTokens } from './tokens'
import { scanSearchQuery, ScanSuccess, Token, ScanResult } from './scanner'
import { SearchPatternType } from '../../graphql-operations'
expect.addSnapshotSerializer({
serialize: value => JSON.stringify(value, null, 2),
test: () => true,
})
const toSuccess = (result: ScanResult<Token[]>): Token[] => (result as ScanSuccess<Token[]>).term
@ -87,4 +93,116 @@ describe('getMonacoTokens()', () => {
},
])
})
test('no decoration for literal', () => {
expect(getMonacoTokens(toSuccess(scanSearchQuery('(a\\sb)', false, SearchPatternType.literal)), true))
.toMatchInlineSnapshot(`
[
{
"startIndex": 0,
"scopes": "identifier"
}
]
`)
})
test('decorate regexp character set and group', () => {
expect(getMonacoTokens(toSuccess(scanSearchQuery('(a\\sb)', false, SearchPatternType.regexp)), true))
.toMatchInlineSnapshot(`
[
{
"startIndex": 0,
"scopes": "regexpMetaDelimited"
},
{
"startIndex": 1,
"scopes": "identifier"
},
{
"startIndex": 2,
"scopes": "regexpMetaCharacterSet"
},
{
"startIndex": 4,
"scopes": "identifier"
},
{
"startIndex": 5,
"scopes": "regexpMetaDelimited"
}
]
`)
})
test('decorate regexp assertion', () => {
expect(getMonacoTokens(toSuccess(scanSearchQuery('^oh\\.hai$', false, SearchPatternType.regexp)), true))
.toMatchInlineSnapshot(`
[
{
"startIndex": 0,
"scopes": "regexpMetaAssertion"
},
{
"startIndex": 1,
"scopes": "identifier"
},
{
"startIndex": 2,
"scopes": "identifier"
},
{
"startIndex": 3,
"scopes": "identifier"
},
{
"startIndex": 5,
"scopes": "identifier"
},
{
"startIndex": 6,
"scopes": "identifier"
},
{
"startIndex": 7,
"scopes": "identifier"
},
{
"startIndex": 8,
"scopes": "regexpMetaAssertion"
}
]
`)
})
test('decorate regexp quantifiers', () => {
expect(getMonacoTokens(toSuccess(scanSearchQuery('a*?(b)+', false, SearchPatternType.regexp)), true))
.toMatchInlineSnapshot(`
[
{
"startIndex": 0,
"scopes": "identifier"
},
{
"startIndex": 1,
"scopes": "regexpMetaQuantifier"
},
{
"startIndex": 3,
"scopes": "regexpMetaDelimited"
},
{
"startIndex": 4,
"scopes": "identifier"
},
{
"startIndex": 5,
"scopes": "regexpMetaDelimited"
},
{
"startIndex": 6,
"scopes": "regexpMetaQuantifier"
}
]
`)
})
})

199
client/shared/src/search/parser/tokens.ts
View File

@ -1,10 +1,192 @@
import * as Monaco from 'monaco-editor'
import { Token } from './scanner'
import { Token, Pattern, CharacterRange, PatternKind } from './scanner'
import { RegExpParser, visitRegExpAST } from 'regexpp'
import { Character, CharacterSet, CapturingGroup, Assertion, Quantifier } from 'regexpp/ast'
/**
* Returns the tokens in a scanned search query displayed in the Monaco query input.
*/
export function getMonacoTokens(tokens: Token[]): Monaco.languages.IToken[] {
export enum RegexpMetaKind {
Delimited = 'Delimited',
CharacterSet = 'CharacterSet',
Quantifier = 'Quantifier',
Assertion = 'Assertion',
}
export interface RegexpMeta {
type: 'regexpMeta'
range: CharacterRange
kind: RegexpMetaKind
value: string
}
export enum StructuralMetaKind {
Hole = 'Hole',
}
export interface StructuralMeta {
type: 'structuralMeta'
range: CharacterRange
kind: StructuralMetaKind
value: string
}
export type MetaToken = RegexpMeta | StructuralMeta
type DecoratedToken = Token | MetaToken
const mapRegexpMeta = (pattern: Pattern): DecoratedToken[] => {
const tokens: DecoratedToken[] = []
try {
const ast = new RegExpParser().parsePattern(pattern.value)
const offset = pattern.range.start
visitRegExpAST(ast, {
onAssertionEnter(node: Assertion) {
tokens.push({
type: 'regexpMeta',
range: { start: offset + node.start, end: offset + node.end },
value: node.raw,
kind: RegexpMetaKind.Assertion,
})
},
onCapturingGroupEnter(node: CapturingGroup) {
// Push the leading '('
tokens.push({
type: 'regexpMeta',
range: { start: offset + node.start, end: offset + node.start + 1 },
value: '(',
kind: RegexpMetaKind.Delimited,
})
// Push the trailing ')'
tokens.push({
type: 'regexpMeta',
range: { start: offset + node.end - 1, end: offset + node.end },
value: ')',
kind: RegexpMetaKind.Delimited,
})
},
onCharacterSetEnter(node: CharacterSet) {
tokens.push({
type: 'regexpMeta',
range: { start: offset + node.start, end: offset + node.end },
value: node.raw,
kind: RegexpMetaKind.CharacterSet,
})
},
onQuantifierEnter(node: Quantifier) {
// the lazy quantifier ? adds one
const lazyQuantifierOffset = node.greedy ? 0 : 1
const quantifier = node.raw[node.raw.length - lazyQuantifierOffset - 1]
if (quantifier === '+' || quantifier === '*' || quantifier === '?') {
tokens.push({
type: 'regexpMeta',
range: { start: offset + node.end - 1 - lazyQuantifierOffset, end: offset + node.end },
value: node.raw,
kind: RegexpMetaKind.Quantifier,
})
} else {
// regexpp provides no easy way to tell whether the quantifier is a range '{number, number}'.
// At this point we know it is none of +, *, or ?, so it is a ranged quantifer.
// We skip highlighting for now; it's trickier.
tokens.push({
type: 'pattern',
range: { start: offset + node.start, end: offset + node.end },
value: node.raw,
kind: PatternKind.Regexp,
})
}
},
onCharacterEnter(node: Character) {
tokens.push({
type: 'pattern',
range: { start: offset + node.start, end: offset + node.end },
value: node.raw,
kind: PatternKind.Regexp,
})
},
})
} catch {
tokens.push(pattern)
}
// The AST is not necessarily traversed in increasing range. We need
// to sort by increasing range because the ordering is significant to Monaco.
tokens.sort((left, right) => {
if (left.range.start < right.range.start) {
return -1
}
return 0
})
return tokens
}
const mapStructuralMeta = (pattern: Pattern): DecoratedToken[] => [pattern]
const decorateTokens = (tokens: Token[]): DecoratedToken[] => {
const decorated: DecoratedToken[] = []
for (const token of tokens) {
if (token.type === 'pattern') {
switch (token.kind) {
case PatternKind.Regexp:
decorated.push(...mapRegexpMeta(token))
break
case PatternKind.Structural:
decorated.push(...mapStructuralMeta(token))
break
default:
decorated.push(token)
}
continue
}
decorated.push(token)
}
return decorated
}
const fromDecoratedTokens = (tokens: DecoratedToken[]): Monaco.languages.IToken[] => {
const monacoTokens: Monaco.languages.IToken[] = []
for (const token of tokens) {
switch (token.type) {
case 'filter':
{
monacoTokens.push({
startIndex: token.filterType.range.start,
scopes: 'filterKeyword',
})
if (token.filterValue) {
monacoTokens.push({
startIndex: token.filterValue.range.start,
scopes: 'identifier',
})
}
}
break
case 'whitespace':
case 'keyword':
case 'comment':
monacoTokens.push({
startIndex: token.range.start,
scopes: token.type,
})
break
case 'regexpMeta':
case 'structuralMeta':
/** The scopes value is derived from the token type and its kind.
* E.g., regexpMetaDelimited dervies from {@link RegexpMeta} and {@link RegexpMetaKind}.
*/
monacoTokens.push({
startIndex: token.range.start,
scopes: `${token.type}${token.kind}`,
})
break
default:
monacoTokens.push({
startIndex: token.range.start,
scopes: 'identifier',
})
break
}
}
return monacoTokens
}
const fromTokens = (tokens: Token[]): Monaco.languages.IToken[] => {
const monacoTokens: Monaco.languages.IToken[] = []
for (const token of tokens) {
switch (token.type) {
@ -40,3 +222,10 @@ export function getMonacoTokens(tokens: Token[]): Monaco.languages.IToken[] {
}
return monacoTokens
}
/**
* Returns the tokens in a scanned search query displayed in the Monaco query input. If the experimental
* decorate flag is true, a list of {@link DecoratedToken} provides more contextual highlighting for patterns.
*/
export const getMonacoTokens = (tokens: Token[], decorate = false): Monaco.languages.IToken[] =>
decorate ? fromDecoratedTokens(decorateTokens(tokens)) : fromTokens(tokens)

10
client/web/src/components/MonacoEditor.tsx
View File

@ -35,6 +35,11 @@ monaco.editor.defineTheme(SOURCEGRAPH_DARK, {
{ token: 'filterKeyword', foreground: '#569cd6' },
{ token: 'keyword', foreground: '#da77f2' },
{ token: 'comment', foreground: '#ffa94d' },
// Regexp pattern highlighting
{ token: 'regexpMetaDelimited', foreground: '#ff6b6b' },
{ token: 'regexpMetaAssertion', foreground: '#ff6b6b' },
{ token: 'regexpMetaCharacterSet', foreground: '#3bc9db' },
{ token: 'regexpMetaQuantifier', foreground: '#3bc9db' },
],
})
@ -61,6 +66,11 @@ monaco.editor.defineTheme(SOURCEGRAPH_LIGHT, {
{ token: 'filterKeyword', foreground: '#268bd2' },
{ token: 'keyword', foreground: '#ae3ec9' },
{ token: 'comment', foreground: '#d9480f' },
// Regexp pattern highlighting
{ token: 'regexpMetaDelimited', foreground: '#c92a2a' },
{ token: 'regexpMetaAssertion', foreground: '#c92a2a' },
{ token: 'regexpMetaCharacterSet', foreground: '#1098ad' },
{ token: 'regexpMetaQuantifier', foreground: '#1098ad' },
],
})

1
client/web/src/nav/GlobalNavbar.story.tsx
View File

@ -38,6 +38,7 @@ const defaultProps = (
setVersionContext: () => undefined,
availableVersionContexts: [],
globbing: false,
enableSmartQuery: false,
patternType: SearchPatternType.literal,
setPatternType: () => undefined,
caseSensitive: false,

1
client/web/src/nav/GlobalNavbar.test.tsx
View File

@ -45,6 +45,7 @@ const PROPS: React.ComponentProps<typeof GlobalNavbar> = {
availableVersionContexts: [],
variant: 'default',
globbing: false,
enableSmartQuery: false,
showOnboardingTour: false,
branding: undefined,
}

3
client/web/src/nav/GlobalNavbar.tsx
View File

@ -62,6 +62,9 @@ interface Props
// Whether globbing is enabled for filters.
globbing: boolean
// Whether to additionally highlight or provide hovers for tokens, e.g., regexp character sets.
enableSmartQuery: boolean
/**
* Which variation of the global navbar to render.
*

1
client/web/src/nav/__snapshots__/GlobalNavbar.test.tsx.snap
View File

@ -21,6 +21,7 @@ exports[`GlobalNavbar default 1`] = `
authenticatedUser={null}
caseSensitive={false}
copyQueryButton={false}
enableSmartQuery={false}
extensionsController={Object {}}
filtersInQuery={Object {}}
globbing={false}

1
client/web/src/repogroups/RepogroupPage.story.tsx
View File

@ -96,6 +96,7 @@ const commonProps = subtypeOf<Partial<RepogroupPageProps>>()({
authenticatedUser: authUser,
repogroupMetadata: python2To3Metadata,
globbing: false,
enableSmartQuery: false,
showOnboardingTour: false,
showQueryBuilder: false,
})

3
client/web/src/repogroups/RepogroupPage.tsx
View File

@ -61,6 +61,9 @@ export interface RepogroupPageProps
/** Whether globbing is enabled for filters. */
globbing: boolean
// Whether to additionally highlight or provide hovers for tokens, e.g., regexp character sets.
enableSmartQuery: boolean
}
export const RepogroupPage: React.FunctionComponent<RepogroupPageProps> = (props: RepogroupPageProps) => {

1
client/web/src/search/SearchConsolePage.tsx
View File

@ -96,6 +96,7 @@ export const SearchConsolePage: React.FunctionComponent<SearchConsolePageProps>
const subscription = addSourcegraphSearchCodeIntelligence(monacoInstance, searchQuery, {
patternType,
globbing,
enableSmartQuery: true,
interpretComments: true,
})
return () => subscription.unsubscribe()

2
client/web/src/search/input/LazyMonacoQueryInput.test.tsx
View File

@ -29,6 +29,7 @@ describe('PlainQueryInput', () => {
copyQueryButton={false}
versionContext={undefined}
globbing={false}
enableSmartQuery={false}
showOnboardingTour={false}
/>
)
@ -57,6 +58,7 @@ describe('PlainQueryInput', () => {
copyQueryButton={false}
versionContext={undefined}
globbing={false}
enableSmartQuery={false}
showOnboardingTour={false}
/>
)

7
client/web/src/search/input/MonacoQueryInput.tsx
View File

@ -48,6 +48,9 @@ export interface MonacoQueryInputProps
// Whether globbing is enabled for filters.
globbing: boolean
// Whether to additionally highlight or provide hovers for tokens, e.g., regexp character sets.
enableSmartQuery: boolean
// Whether comments are parsed and highlighted
interpretComments?: boolean
}
@ -73,6 +76,7 @@ export function addSourcegraphSearchCodeIntelligence(
patternType: SearchPatternType
globbing: boolean
interpretComments?: boolean
enableSmartQuery: boolean
}
): Subscription {
const subscriptions = new Subscription()
@ -292,9 +296,10 @@ export class MonacoQueryInput extends React.PureComponent<MonacoQueryInputProps>
this.subscriptions.add(
this.componentUpdates
.pipe(
map(({ patternType, globbing, interpretComments }) => ({
map(({ patternType, globbing, enableSmartQuery, interpretComments }) => ({
patternType,
globbing,
enableSmartQuery,
interpretComments,
})),
distinctUntilChanged((a, b) => isEqual(a, b)),

1
client/web/src/search/input/SearchNavbarItem.tsx
View File

@ -27,6 +27,7 @@ interface Props
navbarSearchState: QueryState
onChange: (newValue: QueryState) => void
globbing: boolean
enableSmartQuery: boolean
}
/**

1
client/web/src/search/input/SearchPage.story.tsx
View File

@ -30,6 +30,7 @@ const defaultProps = (props: ThemeProps): SearchPageProps => ({
setVersionContext: () => undefined,
availableVersionContexts: [],
globbing: false,
enableSmartQuery: false,
patternType: SearchPatternType.literal,
setPatternType: () => undefined,
caseSensitive: false,

1
client/web/src/search/input/SearchPage.test.tsx
View File

@ -38,6 +38,7 @@ describe('SearchPage', () => {
setVersionContext: () => undefined,
availableVersionContexts: [],
globbing: false,
enableSmartQuery: false,
patternType: SearchPatternType.literal,
setPatternType: () => undefined,
caseSensitive: false,

3
client/web/src/search/input/SearchPage.tsx
View File

@ -66,6 +66,9 @@ export interface SearchPageProps
// Whether globbing is enabled for filters.
globbing: boolean
// Whether to additionally highlight or provide hovers for tokens, e.g., regexp character sets.
enableSmartQuery: boolean
}
/**

2
client/web/src/search/input/SearchPageInput.tsx
View File

@ -66,6 +66,8 @@ interface Props
availableVersionContexts: VersionContext[] | undefined
/** Whether globbing is enabled for filters. */
globbing: boolean
// Whether to additionally highlight or provide hovers for tokens, e.g., regexp character sets.
enableSmartQuery: boolean
/** Show the query builder link. */
showQueryBuilder: boolean
/** A query fragment to appear at the beginning of the input. */

1
package.json
View File

@ -305,6 +305,7 @@
"react-visibility-sensor": "^5.1.1",
"reactstrap": "^8.4.1",
"recharts": "^1.8.5",
"regexpp": "^3.1.0",
"rxjs": "^6.6.2",
"sanitize-html": "^1.26.0",
"semver": "^7.3.2",