mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 17:11:49 +00:00
codeintel: Remove old typescript services (#10566)
This commit is contained in:
Eric Fritz
GitHub
parent
0f46868cfd
commit
81845676da
@ -101,8 +101,3 @@ node_modules/
|
||||
|
||||
# Extensions
|
||||
/packages/sourcegraph-extension-api/dist
|
||||
|
||||
# Precise code intel
|
||||
./cmd/precise-code-intel/node_modules
|
||||
./cmd/precise-code-intel/out
|
||||
./cmd/precise-code-intel/test-data
|
||||
|
||||
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
@ -223,7 +223,6 @@ Dockerfile @sourcegraph/distribution
|
||||
/cmd/frontend/db/discussion* @slimsag
|
||||
|
||||
# Precise code intel
|
||||
/cmd/precise-code-intel/ @sourcegraph/code-intel
|
||||
/cmd/precise-code-intel-api-server/ @sourcegraph/code-intel
|
||||
/cmd/precise-code-intel-bundle-manager/ @sourcegraph/code-intel
|
||||
/cmd/precise-code-intel-worker/ @sourcegraph/code-intel
|
||||
|
||||
15
.github/workflows/lsif.yml
vendored
15
.github/workflows/lsif.yml
vendored
@ -30,21 +30,6 @@ jobs:
|
||||
working-directory: web/
|
||||
run: src lsif upload -github-token=${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
lsif-precise-code-intel:
|
||||
runs-on: ubuntu-latest
|
||||
container: sourcegraph/lsif-node
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cmd/precise-code-intel
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: Install dependencies
|
||||
run: yarn --ignore-engines --ignore-scripts
|
||||
- name: Generate LSIF data
|
||||
run: lsif-tsc -p .
|
||||
- name: Upload LSIF data
|
||||
run: src lsif upload -github-token=${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
lsif-shared:
|
||||
runs-on: ubuntu-latest
|
||||
container: sourcegraph/lsif-node
|
||||
|
||||
4
.vscode/settings.json
vendored
4
.vscode/settings.json
vendored
@ -64,10 +64,6 @@
|
||||
"directory": "shared",
|
||||
"changeProcessCWD": true,
|
||||
},
|
||||
{
|
||||
"directory": "cmd/precise-code-intel",
|
||||
"changeProcessCWD": true,
|
||||
},
|
||||
],
|
||||
"go.lintTool": "golangci-lint",
|
||||
"shellformat.flag": "-i 2 -ci",
|
||||
|
||||
7
.vscode/tasks.json
vendored
7
.vscode/tasks.json
vendored
@ -80,13 +80,6 @@
|
||||
"path": "web/",
|
||||
"problemMatcher": ["$eslint-stylish"],
|
||||
},
|
||||
{
|
||||
"label": "eslint:precise-code-intel",
|
||||
"type": "npm",
|
||||
"script": "eslint",
|
||||
"path": "cmd/precise-code-intel/",
|
||||
"problemMatcher": ["$eslint-stylish"],
|
||||
},
|
||||
{
|
||||
"label": "eslint:release",
|
||||
"type": "npm",
|
||||
|
||||
@ -619,361 +619,3 @@
|
||||
"yallist@3.0.3","ISC","https://github.com/isaacs/yallist"
|
||||
"yargs-parser@9.0.2","ISC","https://github.com/yargs/yargs-parser"
|
||||
"yargs@11.0.0","MIT","https://github.com/yargs/yargs"
|
||||
"@sindresorhus/is@2.1.0","MIT","https://github.com/sindresorhus/is"
|
||||
"@szmarczak/http-timer@4.0.5","MIT","https://github.com/szmarczak/http-timer"
|
||||
"@types/cacheable-request@6.0.1","MIT","https://github.com/DefinitelyTyped/DefinitelyTyped"
|
||||
"@types/http-cache-semantics@4.0.0","MIT","https://github.com/DefinitelyTyped/DefinitelyTyped"
|
||||
"@types/keyv@3.1.1","MIT","https://github.com/DefinitelyTyped/DefinitelyTyped"
|
||||
"@types/node@12.7.11","MIT","https://github.com/DefinitelyTyped/DefinitelyTyped"
|
||||
"@types/responselike@1.0.0","MIT","https://github.com/DefinitelyTyped/DefinitelyTyped"
|
||||
"@types/retry@0.12.0","MIT","https://github.com/DefinitelyTyped/DefinitelyTyped"
|
||||
"abbrev@1.1.1","ISC","https://github.com/isaacs/abbrev-js"
|
||||
"accepts@1.3.7","MIT","https://github.com/jshttp/accepts"
|
||||
"ajv@6.10.2","MIT","https://github.com/epoberezkin/ajv"
|
||||
"ansi-color@0.2.1","BSD*","https://github.com/loopj/commonjs-ansi-color"
|
||||
"ansi-regex@2.1.1","MIT","https://github.com/chalk/ansi-regex"
|
||||
"ansi-regex@4.1.0","MIT","https://github.com/chalk/ansi-regex"
|
||||
"ansi-styles@2.2.1","MIT","https://github.com/chalk/ansi-styles"
|
||||
"ansi-styles@3.2.1","MIT","https://github.com/chalk/ansi-styles"
|
||||
"any-promise@1.3.0","MIT","https://github.com/kevinbeaty/any-promise"
|
||||
"app-root-path@3.0.0","MIT","https://github.com/inxilpro/node-app-root-path"
|
||||
"append-type@1.0.2","MIT-0","https://github.com/shinnn/append-type"
|
||||
"aproba@1.2.0","ISC","https://github.com/iarna/aproba"
|
||||
"are-we-there-yet@1.1.5","ISC","https://github.com/iarna/are-we-there-yet"
|
||||
"argparse@1.0.10","MIT","https://github.com/nodeca/argparse"
|
||||
"array-flatten@1.1.1","MIT","https://github.com/blakeembrey/array-flatten"
|
||||
"array-to-sentence@1.1.0","MIT","https://github.com/shinnn/array-to-sentence"
|
||||
"asn1@0.2.4","MIT","https://github.com/joyent/node-asn1"
|
||||
"assert-plus@1.0.0","MIT","https://github.com/mcavage/node-assert-plus"
|
||||
"assert-valid-glob-opts@1.0.0","CC0-1.0","https://github.com/shinnn/assert-valid-glob-opts"
|
||||
"async-middleware@1.2.1","MIT","https://github.com/blakeembrey/async-middleware"
|
||||
"async-polling@0.2.1","MIT","https://github.com/cGuille/async-polling"
|
||||
"async@2.6.3","MIT","https://github.com/caolan/async"
|
||||
"asynckit@0.4.0","MIT","https://github.com/alexindigo/asynckit"
|
||||
"aws-sign2@0.7.0","Apache-2.0","https://github.com/mikeal/aws-sign"
|
||||
"aws4@1.8.0","MIT","https://github.com/mhart/aws4"
|
||||
"balanced-match@1.0.0","MIT","https://github.com/juliangruber/balanced-match"
|
||||
"base64-js@1.3.1","MIT","https://github.com/beatgammit/base64-js"
|
||||
"bcrypt-pbkdf@1.0.2","BSD-3-Clause","https://github.com/joyent/node-bcrypt-pbkdf"
|
||||
"bintrees@1.0.1","MIT*","https://github.com/vadimg/js_bintrees"
|
||||
"bloomfilter@0.0.18","BSD-3-Clause","https://github.com/jasondavies/bloomfilter.js"
|
||||
"body-parser@1.19.0","MIT","https://github.com/expressjs/body-parser"
|
||||
"brace-expansion@1.1.11","MIT","https://github.com/juliangruber/brace-expansion"
|
||||
"buffer-writer@2.0.0","MIT","https://github.com/brianc/node-buffer-writer"
|
||||
"buffer@5.4.3","MIT","https://github.com/feross/buffer"
|
||||
"bufrw@1.3.0","MIT","https://github.com/uber/bufrw"
|
||||
"bytes@3.1.0","MIT","https://github.com/visionmedia/bytes.js"
|
||||
"cacheable-lookup@2.0.0","MIT","https://github.com/szmarczak/cacheable-lookup"
|
||||
"cacheable-request@7.0.1","MIT","https://github.com/lukechilds/cacheable-request"
|
||||
"camelcase@5.3.1","MIT","https://github.com/sindresorhus/camelcase"
|
||||
"caseless@0.12.0","Apache-2.0","https://github.com/mikeal/caseless"
|
||||
"chalk@1.1.3","MIT","https://github.com/chalk/chalk"
|
||||
"chalk@2.4.2","MIT","https://github.com/chalk/chalk"
|
||||
"chownr@1.1.3","ISC","https://github.com/isaacs/chownr"
|
||||
"cli-highlight@2.1.1","ISC","https://github.com/felixfbecker/cli-highlight"
|
||||
"cliui@5.0.0","ISC","https://github.com/yargs/cliui"
|
||||
"clone-response@1.0.2","MIT","https://github.com/lukechilds/clone-response"
|
||||
"code-point-at@1.1.0","MIT","https://github.com/sindresorhus/code-point-at"
|
||||
"color-convert@1.9.3","MIT","https://github.com/Qix-/color-convert"
|
||||
"color-name@1.1.3","MIT","https://github.com/dfcreative/color-name"
|
||||
"color-name@1.1.4","MIT","https://github.com/colorjs/color-name"
|
||||
"color-string@1.5.3","MIT","https://github.com/Qix-/color-string"
|
||||
"color@3.0.0","MIT","https://github.com/Qix-/color"
|
||||
"colornames@1.1.1","MIT","https://github.com/timoxley/colornames"
|
||||
"colors@1.4.0","MIT","https://github.com/Marak/colors.js"
|
||||
"colorspace@1.1.2","MIT","https://github.com/3rd-Eden/colorspace"
|
||||
"combined-stream@1.0.8","MIT","https://github.com/felixge/node-combined-stream"
|
||||
"commander@2.20.3","MIT","https://github.com/tj/commander.js"
|
||||
"concat-map@0.0.1","MIT","https://github.com/substack/node-concat-map"
|
||||
"console-control-strings@1.1.0","ISC","https://github.com/iarna/console-control-strings"
|
||||
"content-disposition@0.5.3","MIT","https://github.com/jshttp/content-disposition"
|
||||
"content-type@1.0.4","MIT","https://github.com/jshttp/content-type"
|
||||
"cookie-signature@1.0.6","MIT","https://github.com/visionmedia/node-cookie-signature"
|
||||
"cookie@0.4.0","MIT","https://github.com/jshttp/cookie"
|
||||
"core-util-is@1.0.2","MIT","https://github.com/isaacs/core-util-is"
|
||||
"crc-32@1.2.0","Apache-2.0","https://github.com/SheetJS/js-crc32"
|
||||
"dashdash@1.14.1","MIT","https://github.com/trentm/node-dashdash"
|
||||
"debug@2.6.9","MIT","https://github.com/visionmedia/debug"
|
||||
"debug@3.2.6","MIT","https://github.com/visionmedia/debug"
|
||||
"debug@4.1.1","MIT","https://github.com/visionmedia/debug"
|
||||
"decamelize@1.2.0","MIT","https://github.com/sindresorhus/decamelize"
|
||||
"decompress-response@5.0.0","MIT","https://github.com/sindresorhus/decompress-response"
|
||||
"deep-extend@0.6.0","MIT","https://github.com/unclechu/node-deep-extend"
|
||||
"defer-to-connect@2.0.0","MIT","https://github.com/szmarczak/defer-to-connect"
|
||||
"define-properties@1.1.3","MIT","https://github.com/ljharb/define-properties"
|
||||
"delay@4.3.0","MIT","https://github.com/sindresorhus/delay"
|
||||
"delayed-stream@1.0.0","MIT","https://github.com/felixge/node-delayed-stream"
|
||||
"delegates@1.0.0","MIT","https://github.com/visionmedia/node-delegates"
|
||||
"depd@1.1.2","MIT","https://github.com/dougwilson/nodejs-depd"
|
||||
"destroy@1.0.4","MIT","https://github.com/stream-utils/destroy"
|
||||
"detect-libc@1.0.3","Apache-2.0","https://github.com/lovell/detect-libc"
|
||||
"diagnostics@1.1.1","MIT","https://github.com/bigpipe/diagnostics"
|
||||
"dotenv@6.2.0","BSD-2-Clause","https://github.com/motdotla/dotenv"
|
||||
"duplexer3@0.1.4","BSD-3-Clause","https://github.com/floatdrop/duplexer3"
|
||||
"ecc-jsbn@0.1.2","MIT","https://github.com/quartzjer/ecc-jsbn"
|
||||
"ee-first@1.1.1","MIT","https://github.com/jonathanong/ee-first"
|
||||
"emoji-regex@7.0.3","MIT","https://github.com/mathiasbynens/emoji-regex"
|
||||
"enabled@1.0.2","MIT","https://github.com/bigpipe/enabled"
|
||||
"encodeurl@1.0.2","MIT","https://github.com/pillarjs/encodeurl"
|
||||
"end-of-stream@1.4.4","MIT","https://github.com/mafintosh/end-of-stream"
|
||||
"env-variable@0.0.5","MIT","https://github.com/3rd-Eden/env-variable"
|
||||
"error@7.0.2","MIT","https://github.com/Raynos/error"
|
||||
"es-abstract@1.17.0","MIT","https://github.com/ljharb/es-abstract"
|
||||
"es-to-primitive@1.2.1","MIT","https://github.com/ljharb/es-to-primitive"
|
||||
"escape-html@1.0.3","MIT","https://github.com/component/escape-html"
|
||||
"escape-string-regexp@1.0.5","MIT","https://github.com/sindresorhus/escape-string-regexp"
|
||||
"esprima@4.0.1","BSD-2-Clause","https://github.com/jquery/esprima"
|
||||
"etag@1.8.1","MIT","https://github.com/jshttp/etag"
|
||||
"exit-on-epipe@1.0.1","Apache-2.0","https://github.com/SheetJS/node-exit-on-epipe"
|
||||
"express-opentracing@0.1.1","Apache-2.0","https://github.com/opentracing-contrib/javascript-express"
|
||||
"express-validator@6.4.0","MIT","https://github.com/express-validator/express-validator"
|
||||
"express-winston@4.0.3","MIT","https://github.com/bithavoc/express-winston"
|
||||
"express@4.17.1","MIT","https://github.com/expressjs/express"
|
||||
"extend@3.0.2","MIT","https://github.com/justmoon/node-extend"
|
||||
"extsprintf@1.3.0","MIT","https://github.com/davepacheco/node-extsprintf"
|
||||
"fast-deep-equal@2.0.1","MIT","https://github.com/epoberezkin/fast-deep-equal"
|
||||
"fast-json-stable-stringify@2.0.0","MIT","https://github.com/epoberezkin/fast-json-stable-stringify"
|
||||
"fast-safe-stringify@2.0.7","MIT","https://github.com/davidmarkclements/fast-safe-stringify"
|
||||
"fecha@2.3.3","MIT","git+https://taylorhakes@github.com/taylorhakes/fecha"
|
||||
"figlet@1.2.4","MIT","https://github.com/patorjk/figlet.js"
|
||||
"finalhandler@1.1.2","MIT","https://github.com/pillarjs/finalhandler"
|
||||
"find-up@3.0.0","MIT","https://github.com/sindresorhus/find-up"
|
||||
"forever-agent@0.6.1","Apache-2.0","https://github.com/mikeal/forever-agent"
|
||||
"form-data@2.3.3","MIT","https://github.com/form-data/form-data"
|
||||
"forwarded@0.1.2","MIT","https://github.com/jshttp/forwarded"
|
||||
"fresh@0.5.2","MIT","https://github.com/jshttp/fresh"
|
||||
"fs-minipass@1.2.7","ISC","https://github.com/npm/fs-minipass"
|
||||
"fs.realpath@1.0.0","ISC","https://github.com/isaacs/fs.realpath"
|
||||
"function-bind@1.1.1","MIT","https://github.com/Raynos/function-bind"
|
||||
"gauge@2.7.4","ISC","https://github.com/iarna/gauge"
|
||||
"get-caller-file@2.0.5","ISC","https://github.com/stefanpenner/get-caller-file"
|
||||
"get-stream@5.1.0","MIT","https://github.com/sindresorhus/get-stream"
|
||||
"getpass@0.1.7","MIT","https://github.com/arekinath/node-getpass"
|
||||
"glob-option-error@1.0.0","MIT","https://github.com/shinnn/glob-option-error"
|
||||
"glob@7.1.6","ISC","https://github.com/isaacs/node-glob"
|
||||
"got@10.7.0","MIT","https://github.com/sindresorhus/got"
|
||||
"graceful-fs@4.2.3","ISC","https://github.com/isaacs/node-graceful-fs"
|
||||
"har-schema@2.0.0","ISC","https://github.com/ahmadnassri/har-schema"
|
||||
"har-validator@5.1.3","MIT","https://github.com/ahmadnassri/node-har-validator"
|
||||
"has-ansi@2.0.0","MIT","https://github.com/sindresorhus/has-ansi"
|
||||
"has-flag@3.0.0","MIT","https://github.com/sindresorhus/has-flag"
|
||||
"has-symbols@1.0.1","MIT","https://github.com/ljharb/has-symbols"
|
||||
"has-unicode@2.0.1","ISC","https://github.com/iarna/has-unicode"
|
||||
"has@1.0.3","MIT","https://github.com/tarruda/has"
|
||||
"hexer@1.5.0","MIT","https://github.com/jcorbin/hexer"
|
||||
"highlight.js@9.15.10","BSD-3-Clause","https://github.com/highlightjs/highlight.js"
|
||||
"http-cache-semantics@4.0.3","BSD-2-Clause","https://github.com/kornelski/http-cache-semantics"
|
||||
"http-errors@1.7.2","MIT","https://github.com/jshttp/http-errors"
|
||||
"http-signature@1.2.0","MIT","https://github.com/joyent/node-http-signature"
|
||||
"iconv-lite@0.4.24","MIT","https://github.com/ashtuchkin/iconv-lite"
|
||||
"ieee754@1.1.13","BSD-3-Clause","https://github.com/feross/ieee754"
|
||||
"ignore-walk@3.0.2","ISC","https://github.com/isaacs/ignore-walk"
|
||||
"indexed-filter@1.0.3","ISC","https://github.com/shinnn/indexed-filter"
|
||||
"inflight@1.0.6","ISC","https://github.com/npm/inflight"
|
||||
"inherits@2.0.3","ISC","https://github.com/isaacs/inherits"
|
||||
"ini@1.3.5","ISC","https://github.com/isaacs/ini"
|
||||
"inspect-with-kind@1.0.5","ISC","https://github.com/shinnn/inspect-with-kind"
|
||||
"ipaddr.js@1.9.0","MIT","https://github.com/whitequark/ipaddr.js"
|
||||
"is-arrayish@0.3.2","MIT","https://github.com/qix-/node-is-arrayish"
|
||||
"is-callable@1.1.5","MIT","https://github.com/ljharb/is-callable"
|
||||
"is-date-object@1.0.1","MIT","https://github.com/ljharb/is-date-object"
|
||||
"is-fullwidth-code-point@1.0.0","MIT","https://github.com/sindresorhus/is-fullwidth-code-point"
|
||||
"is-fullwidth-code-point@2.0.0","MIT","https://github.com/sindresorhus/is-fullwidth-code-point"
|
||||
"is-plain-obj@1.1.0","MIT","https://github.com/sindresorhus/is-plain-obj"
|
||||
"is-regex@1.0.5","MIT","https://github.com/ljharb/is-regex"
|
||||
"is-stream@1.1.0","MIT","https://github.com/sindresorhus/is-stream"
|
||||
"is-symbol@1.0.2","MIT","https://github.com/ljharb/is-symbol"
|
||||
"is-typedarray@1.0.0","MIT","https://github.com/hughsk/is-typedarray"
|
||||
"isarray@1.0.0","MIT","https://github.com/juliangruber/isarray"
|
||||
"isstream@0.1.2","MIT","https://github.com/rvagg/isstream"
|
||||
"jaeger-client@3.17.2","Apache-2.0","https://github.com/jaegertracing/jaeger-client-node"
|
||||
"js-yaml@3.13.1","MIT","https://github.com/nodeca/js-yaml"
|
||||
"jsbn@0.1.1","MIT","https://github.com/andyperlitch/jsbn"
|
||||
"json-buffer@3.0.1","MIT","https://github.com/dominictarr/json-buffer"
|
||||
"json-schema-traverse@0.4.1","MIT","https://github.com/epoberezkin/json-schema-traverse"
|
||||
"json-schema@0.2.3","AFLv2.1,BSD","https://github.com/kriszyp/json-schema"
|
||||
"json-stringify-safe@5.0.1","ISC","https://github.com/isaacs/json-stringify-safe"
|
||||
"json5@2.1.1","MIT","https://github.com/json5/json5"
|
||||
"jsprim@1.4.1","MIT","https://github.com/joyent/node-jsprim"
|
||||
"keyv@4.0.0","MIT","https://github.com/lukechilds/keyv"
|
||||
"kind-of@6.0.2","MIT","https://github.com/jonschlinkert/kind-of"
|
||||
"kuler@1.0.1","MIT","https://github.com/3rd-Eden/kuler"
|
||||
"limiter@1.1.5","MIT","https://github.com/jhurliman/node-rate-limiter"
|
||||
"locate-path@3.0.0","MIT","https://github.com/sindresorhus/locate-path"
|
||||
"lodash@4.17.15","MIT","https://github.com/lodash/lodash"
|
||||
"logform@2.1.2","MIT","https://github.com/winstonjs/logform"
|
||||
"long@2.4.0","Apache-2.0","https://github.com/dcodeIO/Long.js"
|
||||
"lowercase-keys@2.0.0","MIT","https://github.com/sindresorhus/lowercase-keys"
|
||||
"lsif-protocol@0.4.3","MIT","https://github.com/Microsoft/lsif-typescript"
|
||||
"media-typer@0.3.0","MIT","https://github.com/jshttp/media-typer"
|
||||
"merge-descriptors@1.0.1","MIT","https://github.com/component/merge-descriptors"
|
||||
"methods@1.1.2","MIT","https://github.com/jshttp/methods"
|
||||
"mime-db@1.40.0","MIT","https://github.com/jshttp/mime-db"
|
||||
"mime-types@2.1.24","MIT","https://github.com/jshttp/mime-types"
|
||||
"mime@1.6.0","MIT","https://github.com/broofa/node-mime"
|
||||
"mimic-response@1.0.1","MIT","https://github.com/sindresorhus/mimic-response"
|
||||
"mimic-response@2.1.0","MIT","https://github.com/sindresorhus/mimic-response"
|
||||
"minimatch@3.0.4","ISC","https://github.com/isaacs/minimatch"
|
||||
"minimist@0.0.8","MIT","https://github.com/substack/minimist"
|
||||
"minimist@1.2.0","MIT","https://github.com/substack/minimist"
|
||||
"minipass@2.9.0","ISC","https://github.com/isaacs/minipass"
|
||||
"minizlib@1.3.3","MIT","https://github.com/isaacs/minizlib"
|
||||
"mkdirp@0.5.1","MIT","https://github.com/substack/node-mkdirp"
|
||||
"ms@2.0.0","MIT","https://github.com/zeit/ms"
|
||||
"ms@2.1.1","MIT","https://github.com/zeit/ms"
|
||||
"mz@2.7.0","MIT","https://github.com/normalize/mz"
|
||||
"nan@2.14.0","MIT","https://github.com/nodejs/nan"
|
||||
"needle@2.4.0","MIT","https://github.com/tomas/needle"
|
||||
"negotiator@0.6.2","MIT","https://github.com/jshttp/negotiator"
|
||||
"node-int64@0.4.0","MIT","https://github.com/broofa/node-int64"
|
||||
"node-pre-gyp@0.11.0","BSD-3-Clause","https://github.com/mapbox/node-pre-gyp"
|
||||
"nopt@4.0.1","ISC","https://github.com/npm/nopt"
|
||||
"normalize-url@4.5.0","MIT","https://github.com/sindresorhus/normalize-url"
|
||||
"npm-bundled@1.0.6","ISC","https://github.com/npm/npm-bundled"
|
||||
"npm-packlist@1.4.4","ISC","https://github.com/npm/npm-packlist"
|
||||
"npmlog@4.1.2","ISC","https://github.com/npm/npmlog"
|
||||
"number-is-nan@1.0.1","MIT","https://github.com/sindresorhus/number-is-nan"
|
||||
"oauth-sign@0.9.0","Apache-2.0","https://github.com/mikeal/oauth-sign"
|
||||
"object-assign@4.1.1","MIT","https://github.com/sindresorhus/object-assign"
|
||||
"object-inspect@1.7.0","MIT","https://github.com/substack/object-inspect"
|
||||
"object-keys@1.1.1","MIT","https://github.com/ljharb/object-keys"
|
||||
"object.assign@4.1.0","MIT","https://github.com/ljharb/object.assign"
|
||||
"object.getownpropertydescriptors@2.0.3","MIT","https://github.com/ljharb/object.getownpropertydescriptors"
|
||||
"on-finished@2.3.0","MIT","https://github.com/jshttp/on-finished"
|
||||
"once@1.4.0","ISC","https://github.com/isaacs/once"
|
||||
"one-time@0.0.4","MIT","https://github.com/unshiftio/one-time"
|
||||
"opentracing@0.13.0","MIT","https://github.com/opentracing/opentracing-javascript"
|
||||
"opentracing@0.14.4","Apache-2.0","https://github.com/opentracing/opentracing-javascript"
|
||||
"os-homedir@1.0.2","MIT","https://github.com/sindresorhus/os-homedir"
|
||||
"os-tmpdir@1.0.2","MIT","https://github.com/sindresorhus/os-tmpdir"
|
||||
"osenv@0.1.5","ISC","https://github.com/npm/osenv"
|
||||
"p-cancelable@2.0.0","MIT","https://github.com/sindresorhus/p-cancelable"
|
||||
"p-event@4.1.0","MIT","https://github.com/sindresorhus/p-event"
|
||||
"p-finally@1.0.0","MIT","https://github.com/sindresorhus/p-finally"
|
||||
"p-limit@2.2.2","MIT","https://github.com/sindresorhus/p-limit"
|
||||
"p-locate@3.0.0","MIT","https://github.com/sindresorhus/p-locate"
|
||||
"p-retry@4.2.0","MIT","https://github.com/sindresorhus/p-retry"
|
||||
"p-timeout@2.0.1","MIT","https://github.com/sindresorhus/p-timeout"
|
||||
"p-try@2.2.0","MIT","https://github.com/sindresorhus/p-try"
|
||||
"packet-reader@1.0.0","MIT","https://github.com/brianc/node-packet-reader"
|
||||
"parent-require@1.0.0","MIT","https://github.com/jaredhanson/node-parent-require"
|
||||
"parse5@4.0.0","MIT","https://github.com/inikulin/parse5"
|
||||
"parseurl@1.3.3","MIT","https://github.com/pillarjs/parseurl"
|
||||
"path-exists@3.0.0","MIT","https://github.com/sindresorhus/path-exists"
|
||||
"path-is-absolute@1.0.1","MIT","https://github.com/sindresorhus/path-is-absolute"
|
||||
"path-to-regexp@0.1.7","MIT","https://github.com/component/path-to-regexp"
|
||||
"performance-now@2.1.0","MIT","https://github.com/braveg1rl/performance-now"
|
||||
"pg-connection-string@0.1.3","MIT","https://github.com/iceddev/pg-connection-string"
|
||||
"pg-int8@1.0.1","ISC","https://github.com/charmander/pg-int8"
|
||||
"pg-packet-stream@1.1.0","MIT",""
|
||||
"pg-pool@2.0.10","MIT","https://github.com/brianc/node-pg-pool"
|
||||
"pg-types@2.2.0","MIT","https://github.com/brianc/node-pg-types"
|
||||
"pg@7.18.2","MIT","https://github.com/brianc/node-postgres"
|
||||
"pgpass@1.0.2","MIT","https://github.com/hoegaarden/pgpass"
|
||||
"postgres-array@2.0.0","MIT","https://github.com/bendrucker/postgres-array"
|
||||
"postgres-bytea@1.0.0","MIT","https://github.com/bendrucker/postgres-bytea"
|
||||
"postgres-date@1.0.4","MIT","https://github.com/bendrucker/postgres-date"
|
||||
"postgres-interval@1.2.0","MIT","https://github.com/bendrucker/postgres-interval"
|
||||
"printj@1.1.2","Apache-2.0","https://github.com/SheetJS/printj"
|
||||
"process-nextick-args@2.0.1","MIT","https://github.com/calvinmetcalf/process-nextick-args"
|
||||
"process@0.10.1","MIT*","https://github.com/shtylman/node-process"
|
||||
"prom-client@12.0.0","Apache-2.0","https://github.com/siimon/prom-client"
|
||||
"proxy-addr@2.0.5","MIT","https://github.com/jshttp/proxy-addr"
|
||||
"psl@1.7.0","MIT","https://github.com/lupomontero/psl"
|
||||
"pump@3.0.0","MIT","https://github.com/mafintosh/pump"
|
||||
"punycode@1.4.1","MIT","https://github.com/bestiejs/punycode.js"
|
||||
"punycode@2.1.1","MIT","https://github.com/bestiejs/punycode.js"
|
||||
"qs@6.5.2","BSD-3-Clause","https://github.com/ljharb/qs"
|
||||
"qs@6.7.0","BSD-3-Clause","https://github.com/ljharb/qs"
|
||||
"range-parser@1.2.1","MIT","https://github.com/jshttp/range-parser"
|
||||
"raw-body@2.4.0","MIT","https://github.com/stream-utils/raw-body"
|
||||
"rc@1.2.8","(BSD-2-Clause OR MIT OR Apache-2.0)","https://github.com/dominictarr/rc"
|
||||
"readable-stream@2.3.6","MIT","https://github.com/nodejs/readable-stream"
|
||||
"readable-stream@3.4.0","MIT","https://github.com/nodejs/readable-stream"
|
||||
"reflect-metadata@0.1.13","Apache-2.0","https://github.com/rbuckton/reflect-metadata"
|
||||
"relateurl@0.2.7","MIT","https://github.com/stevenvachon/relateurl"
|
||||
"request@2.88.0","Apache-2.0","https://github.com/request/request"
|
||||
"require-directory@2.1.1","MIT","https://github.com/troygoode/node-require-directory"
|
||||
"require-main-filename@2.0.0","ISC","https://github.com/yargs/require-main-filename"
|
||||
"responselike@2.0.0","MIT","https://github.com/lukechilds/responselike"
|
||||
"retry@0.12.0","MIT","https://github.com/tim-kos/node-retry"
|
||||
"rimraf@2.7.1","ISC","https://github.com/isaacs/rimraf"
|
||||
"rmfr@2.0.0","ISC","https://github.com/shinnn/rmfr"
|
||||
"safe-buffer@5.1.2","MIT","https://github.com/feross/safe-buffer"
|
||||
"safer-buffer@2.1.2","MIT","https://github.com/ChALkeR/safer-buffer"
|
||||
"sax@1.2.4","ISC","https://github.com/isaacs/sax-js"
|
||||
"semver@4.3.2","BSD*","https://github.com/npm/node-semver"
|
||||
"semver@5.7.1","ISC","https://github.com/npm/node-semver"
|
||||
"send@0.17.1","MIT","https://github.com/pillarjs/send"
|
||||
"serve-static@1.14.1","MIT","https://github.com/expressjs/serve-static"
|
||||
"set-blocking@2.0.0","ISC","https://github.com/yargs/set-blocking"
|
||||
"setprototypeof@1.1.1","ISC","https://github.com/wesleytodd/setprototypeof"
|
||||
"sha.js@2.4.11","(MIT AND BSD-3-Clause)","https://github.com/crypto-browserify/sha.js"
|
||||
"signal-exit@3.0.2","ISC","https://github.com/tapjs/signal-exit"
|
||||
"simple-swizzle@0.2.2","MIT","https://github.com/qix-/node-simple-swizzle"
|
||||
"split@1.0.1","MIT","https://github.com/dominictarr/split"
|
||||
"sprintf-js@1.0.3","BSD-3-Clause","https://github.com/alexei/sprintf.js"
|
||||
"sqlite3@4.1.1","BSD-3-Clause","https://github.com/mapbox/node-sqlite3"
|
||||
"sshpk@1.16.1","MIT","https://github.com/joyent/node-sshpk"
|
||||
"stack-trace@0.0.10","MIT","https://github.com/felixge/node-stack-trace"
|
||||
"statuses@1.5.0","MIT","https://github.com/jshttp/statuses"
|
||||
"stream-throttle@0.1.3","BSD-3-Clause","https://github.com/tjgq/node-stream-throttle"
|
||||
"string-template@0.2.1","MIT","https://github.com/Matt-Esch/string-template"
|
||||
"string-width@1.0.2","MIT","https://github.com/sindresorhus/string-width"
|
||||
"string-width@3.1.0","MIT","https://github.com/sindresorhus/string-width"
|
||||
"string.prototype.trimleft@2.1.1","MIT","https://github.com/es-shims/String.prototype.trimLeft"
|
||||
"string.prototype.trimright@2.1.1","MIT","https://github.com/es-shims/String.prototype.trimRight"
|
||||
"string_decoder@1.1.1","MIT","https://github.com/nodejs/string_decoder"
|
||||
"strip-ansi@3.0.1","MIT","https://github.com/chalk/strip-ansi"
|
||||
"strip-ansi@5.2.0","MIT","https://github.com/chalk/strip-ansi"
|
||||
"strip-json-comments@2.0.1","MIT","https://github.com/sindresorhus/strip-json-comments"
|
||||
"supports-color@2.0.0","MIT","https://github.com/chalk/supports-color"
|
||||
"supports-color@5.5.0","MIT","https://github.com/chalk/supports-color"
|
||||
"tar@4.4.13","ISC","https://github.com/npm/node-tar"
|
||||
"tdigest@0.1.1","MIT","https://github.com/welch/tdigest"
|
||||
"text-hex@1.0.0","MIT","https://github.com/3rd-Eden/text-hex"
|
||||
"thenify-all@1.6.0","MIT","https://github.com/thenables/thenify-all"
|
||||
"thenify@3.3.0","MIT","https://github.com/thenables/thenify"
|
||||
"thriftrw@3.11.3","MIT","https://github.com/thriftrw/thriftrw-node"
|
||||
"through@2.3.8","MIT","https://github.com/dominictarr/through"
|
||||
"to-readable-stream@2.1.0","MIT","https://github.com/sindresorhus/to-readable-stream"
|
||||
"toidentifier@1.0.0","MIT","https://github.com/component/toidentifier"
|
||||
"tough-cookie@2.4.3","BSD-3-Clause","https://github.com/salesforce/tough-cookie"
|
||||
"triple-beam@1.3.0","MIT","https://github.com/winstonjs/triple-beam"
|
||||
"tslib@1.10.0","Apache-2.0","https://github.com/Microsoft/tslib"
|
||||
"tunnel-agent@0.6.0","Apache-2.0","https://github.com/mikeal/tunnel-agent"
|
||||
"tweetnacl@0.14.5","Unlicense","https://github.com/dchest/tweetnacl-js"
|
||||
"type-fest@0.10.0","(MIT OR CC0-1.0)","https://github.com/sindresorhus/type-fest"
|
||||
"type-is@1.6.18","MIT","https://github.com/jshttp/type-is"
|
||||
"typeorm@0.2.24","MIT","https://github.com/typeorm/typeorm"
|
||||
"unpipe@1.0.0","MIT","https://github.com/stream-utils/unpipe"
|
||||
"uri-js@4.2.2","BSD-2-Clause","https://github.com/garycourt/uri-js"
|
||||
"util-deprecate@1.0.2","MIT","https://github.com/TooTallNate/util-deprecate"
|
||||
"util.promisify@1.0.0","MIT","https://github.com/ljharb/util.promisify"
|
||||
"utils-merge@1.0.1","MIT","https://github.com/jaredhanson/utils-merge"
|
||||
"uuid@3.4.0","MIT","https://github.com/uuidjs/uuid"
|
||||
"uuid@7.0.3","MIT","https://github.com/uuidjs/uuid"
|
||||
"validate-glob-opts@1.0.2","ISC","https://github.com/shinnn/validate-glob-opts"
|
||||
"validator@12.2.0","MIT","https://github.com/chriso/validator.js"
|
||||
"vary@1.1.2","MIT","https://github.com/jshttp/vary"
|
||||
"verror@1.10.0","MIT","https://github.com/davepacheco/node-verror"
|
||||
"vscode-jsonrpc@5.0.1","MIT","https://github.com/Microsoft/vscode-languageserver-node"
|
||||
"vscode-languageserver-protocol@3.15.3","MIT","https://github.com/Microsoft/vscode-languageserver-node"
|
||||
"vscode-languageserver-types@3.15.1","MIT","https://github.com/Microsoft/vscode-languageserver-node"
|
||||
"vscode-languageserver@6.1.1","MIT","https://github.com/Microsoft/vscode-languageserver-node"
|
||||
"which-module@2.0.0","ISC","https://github.com/nexdrew/which-module"
|
||||
"wide-align@1.1.3","ISC","https://github.com/iarna/wide-align"
|
||||
"winston-transport@4.3.0","MIT","https://github.com/winstonjs/winston-transport"
|
||||
"winston@3.2.1","MIT","https://github.com/winstonjs/winston"
|
||||
"wrap-ansi@5.1.0","MIT","https://github.com/chalk/wrap-ansi"
|
||||
"wrappy@1.0.2","ISC","https://github.com/npm/wrappy"
|
||||
"xml2js@0.4.22","MIT","https://github.com/Leonidas-from-XIV/node-xml2js"
|
||||
"xmlbuilder@11.0.1","MIT","https://github.com/oozcitak/xmlbuilder-js"
|
||||
"xorshift@0.2.1","MIT","https://github.com/AndreasMadsen/xorshift"
|
||||
"xtend@4.0.2","MIT","https://github.com/Raynos/xtend"
|
||||
"y18n@4.0.0","ISC","https://github.com/yargs/y18n"
|
||||
"yallist@3.1.1","ISC","https://github.com/isaacs/yallist"
|
||||
"yallist@4.0.0","ISC","https://github.com/isaacs/yallist"
|
||||
"yargonaut@1.1.4","Apache-2.0","https://github.com/nexdrew/yargonaut"
|
||||
"yargs-parser@13.1.1","ISC","https://github.com/yargs/yargs-parser"
|
||||
"yargs@13.3.0","MIT","https://github.com/yargs/yargs"
|
||||
|
||||
|
@ -34,8 +34,6 @@ module.exports = api => {
|
||||
],
|
||||
plugins: [
|
||||
'babel-plugin-lodash',
|
||||
// Required to support typeorm decorators in ./cmd/precise-code-intel
|
||||
['@babel/plugin-proposal-decorators', { legacy: true }],
|
||||
// Node 12 (released 2019 Apr 23) supports these natively, but there seem to be issues when used with TypeScript.
|
||||
['@babel/plugin-proposal-class-properties', { loose: true }],
|
||||
],
|
||||
|
||||
@ -1,15 +0,0 @@
|
||||
const baseConfig = require('../../.eslintrc')
|
||||
module.exports = {
|
||||
extends: '../../.eslintrc.js',
|
||||
parserOptions: {
|
||||
...baseConfig.parserOptions,
|
||||
project: 'tsconfig.json',
|
||||
},
|
||||
rules: {
|
||||
'no-console': ['error'],
|
||||
'import/no-cycle': ['error'],
|
||||
'no-return-await': ['error'],
|
||||
'no-shadow': ['error', { allow: ['ctx'] }],
|
||||
},
|
||||
overrides: baseConfig.overrides,
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
# Precise code intelligence system
|
||||
|
||||
This project is an early adopter of Microsoft's [LSIF](https://code.visualstudio.com/blogs/2019/02/19/lsif) standard. LSIF (Language Server Index Format) is a format used to store the results of language server queries that are computed ahead-of-time. We uses this format to provide jump-to-definition, find-reference, and hover docstring functionality.
|
||||
|
||||
## LSIF code intelligence
|
||||
|
||||
LSIF dumps are generated by running an LSIF indexer in a build or continuous integration environment. The dump is uploaded to a Sourcegraph instance via [Sourcegraph CLI](https://github.com/sourcegraph/src-cli). An LSIF API server, proxied by the frontend for auth, answers relevant LSP queries to provide fast and precise code intelligence.
|
||||
|
||||
## Architecture
|
||||
|
||||
This project is split into three parts, all currently written in TypeScript. These parts are deployable independently but can also run in the same docker container to simplify deployment with docker-compose.
|
||||
|
||||
- The [API server](./src/api-server/api.ts) receives LSIF uploads and answers LSP queries via HTTP.
|
||||
- The [bundle-manager](./src/bundle-manager/manager.ts) answers queries about a particular upload by looking at relevant SQLite databases on-disk.
|
||||
- The [worker](./src/worker/worker.ts) dequeues unconverted LSIF uploads from Postgres and converts them into SQLite databases that can be queried by the server.
|
||||
|
||||
The `api-server`, `bundle-manager`, and `worker` directories contain only instructions to build Docker images for the three entrypoints.
|
||||
|
||||
## Documentation
|
||||
|
||||
- Usage documentation is provided on [Sourcegraph.com](https://docs.sourcegraph.com/user/code_intelligence/lsif).
|
||||
- API endpoint documentation is provided in [api.md](./docs/api/api.md).
|
||||
- Database configuration and migrations are described in [database.md](./docs/database/database.md).
|
||||
- Data models are described in [datamodel.md](./docs/database/datamodel.md) and [datamodel.pg.md](./docs/database/datamodel.pg.md).
|
||||
@ -1,45 +0,0 @@
|
||||
FROM alpine:3.10@sha256:e4355b66995c96b4b468159fc5c7e3540fcef961189ca13fee877798649f531a AS precise-code-intel-builder
|
||||
|
||||
RUN apk add --no-cache \
|
||||
nodejs-current=12.4.0-r0 \
|
||||
nodejs-npm=10.19.0-r0
|
||||
|
||||
RUN npm install -g yarn@1.17.3
|
||||
|
||||
COPY precise-code-intel/package.json precise-code-intel/yarn.lock precise-code-intel/tsconfig.json /precise-code-intel/
|
||||
RUN yarn --cwd /precise-code-intel --frozen-lockfile
|
||||
COPY precise-code-intel/src /precise-code-intel/src
|
||||
RUN yarn --cwd /precise-code-intel run build
|
||||
# Remove all devDependencies
|
||||
RUN yarn --cwd /precise-code-intel --frozen-lockfile --production
|
||||
|
||||
FROM sourcegraph/alpine:3.10@sha256:4d05cd5669726fc38823e92320659a6d1ef7879e62268adec5df658a0bacf65c
|
||||
|
||||
ARG COMMIT_SHA="unknown"
|
||||
ARG DATE="unknown"
|
||||
ARG VERSION="unknown"
|
||||
|
||||
LABEL org.opencontainers.image.revision=${COMMIT_SHA}
|
||||
LABEL org.opencontainers.image.created=${DATE}
|
||||
LABEL org.opencontainers.image.version=${VERSION}
|
||||
LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}
|
||||
|
||||
# hadolint ignore=DL3018
|
||||
RUN apk update && apk add --no-cache \
|
||||
nodejs-current=12.4.0-r0 \
|
||||
tini
|
||||
|
||||
# Ensures that a directory with the correct permissions exist in the image. Without this, in Docker Compose
|
||||
# deployments the Docker daemon would first create the volume directory and it would be owned by `root` and
|
||||
# then one of the precise-code-intel processes would be unable to create the `/lsif-storage` because it
|
||||
# would be trying to do so in a directory owned by `root` as the user `sourcegraph`. And no, this is not
|
||||
# dumb, this is just Docker: https://github.com/docker/compose/issues/3270#issuecomment-363478501.
|
||||
USER root
|
||||
RUN mkdir -p /lsif-storage && chown -R sourcegraph:sourcegraph /lsif-storage
|
||||
USER sourcegraph
|
||||
|
||||
COPY --from=precise-code-intel-builder /precise-code-intel /precise-code-intel
|
||||
|
||||
EXPOSE 3186
|
||||
ENV LOG_LEVEL=debug
|
||||
ENTRYPOINT ["/sbin/tini", "--", "node", "/precise-code-intel/out/api-server/api.js"]
|
||||
@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../.."
|
||||
set -eux
|
||||
|
||||
OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
|
||||
cleanup() {
|
||||
rm -rf "$OUTPUT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Environment for building linux binaries
|
||||
export GO111MODULE=on
|
||||
export GOARCH=amd64
|
||||
export GOOS=linux
|
||||
export CGO_ENABLED=0
|
||||
|
||||
cp -a ./cmd/precise-code-intel "$OUTPUT"
|
||||
|
||||
echo "--- docker build"
|
||||
docker build -f cmd/precise-code-intel/api-server/Dockerfile -t "$IMAGE" "$OUTPUT" \
|
||||
--progress=plain \
|
||||
--build-arg COMMIT_SHA \
|
||||
--build-arg DATE \
|
||||
--build-arg VERSION
|
||||
@ -1,8 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
/** @type {import('@babel/core').TransformOptions} */
|
||||
const config = {
|
||||
extends: '../../babel.config.js',
|
||||
}
|
||||
|
||||
module.exports = config
|
||||
@ -1,46 +0,0 @@
|
||||
FROM alpine:3.10@sha256:e4355b66995c96b4b468159fc5c7e3540fcef961189ca13fee877798649f531a AS precise-code-intel-builder
|
||||
|
||||
RUN apk add --no-cache \
|
||||
nodejs-current=12.4.0-r0 \
|
||||
nodejs-npm=10.19.0-r0
|
||||
|
||||
RUN npm install -g yarn@1.17.3
|
||||
|
||||
COPY precise-code-intel/package.json precise-code-intel/yarn.lock precise-code-intel/tsconfig.json /precise-code-intel/
|
||||
RUN yarn --cwd /precise-code-intel --frozen-lockfile
|
||||
COPY precise-code-intel/src /precise-code-intel/src
|
||||
RUN yarn --cwd /precise-code-intel run build
|
||||
# Remove all devDependencies
|
||||
RUN yarn --cwd /precise-code-intel --frozen-lockfile --production
|
||||
|
||||
FROM sourcegraph/alpine:3.10@sha256:4d05cd5669726fc38823e92320659a6d1ef7879e62268adec5df658a0bacf65c
|
||||
|
||||
ARG COMMIT_SHA="unknown"
|
||||
ARG DATE="unknown"
|
||||
ARG VERSION="unknown"
|
||||
|
||||
LABEL org.opencontainers.image.revision=${COMMIT_SHA}
|
||||
LABEL org.opencontainers.image.created=${DATE}
|
||||
LABEL org.opencontainers.image.version=${VERSION}
|
||||
LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}
|
||||
|
||||
# hadolint ignore=DL3018
|
||||
RUN apk update && apk add --no-cache \
|
||||
nodejs-current=12.4.0-r0 \
|
||||
tini
|
||||
|
||||
# Ensures that a directory with the correct permissions exist in the image. Without this, in Docker Compose
|
||||
# deployments the Docker daemon would first create the volume directory and it would be owned by `root` and
|
||||
# then one of the precise-code-intel processes would be unable to create the `/lsif-storage` because it
|
||||
# would be trying to do so in a directory owned by `root` as the user `sourcegraph`. And no, this is not
|
||||
# dumb, this is just Docker: https://github.com/docker/compose/issues/3270#issuecomment-363478501.
|
||||
USER root
|
||||
RUN mkdir -p /lsif-storage && chown -R sourcegraph:sourcegraph /lsif-storage
|
||||
USER sourcegraph
|
||||
|
||||
COPY --from=precise-code-intel-builder /precise-code-intel /precise-code-intel
|
||||
|
||||
EXPOSE 3187
|
||||
VOLUME ["/lsif-storage"]
|
||||
ENV LOG_LEVEL=debug
|
||||
ENTRYPOINT ["/sbin/tini", "--", "node", "/precise-code-intel/out/bundle-manager/manager.js"]
|
||||
@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../.."
|
||||
set -eux
|
||||
|
||||
OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
|
||||
cleanup() {
|
||||
rm -rf "$OUTPUT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Environment for building linux binaries
|
||||
export GO111MODULE=on
|
||||
export GOARCH=amd64
|
||||
export GOOS=linux
|
||||
export CGO_ENABLED=0
|
||||
|
||||
cp -a ./cmd/precise-code-intel "$OUTPUT"
|
||||
|
||||
echo "--- docker build"
|
||||
docker build -f cmd/precise-code-intel/bundle-manager/Dockerfile -t "$IMAGE" "$OUTPUT" \
|
||||
--progress=plain \
|
||||
--build-arg COMMIT_SHA \
|
||||
--build-arg DATE \
|
||||
--build-arg VERSION
|
||||
@ -1,15 +0,0 @@
|
||||
# LSIF API server endpoints
|
||||
|
||||
The LSIF API server endpoints are documented as an [OpenAPI v3](https://swagger.io/docs/specification/about/) document [api.yaml](./api.yaml). This document can be viewed locally via docker by running the following command from this directory (or a parent directory if the host path supplied to `-v` changes accordingly).
|
||||
|
||||
```bash
|
||||
docker run \
|
||||
-e SWAGGER_JSON=/data/api.yaml \
|
||||
-p 8080:8080 \
|
||||
-v `pwd`:/data \
|
||||
swaggerapi/swagger-ui
|
||||
```
|
||||
|
||||
The OpenAPI document assumes that the LSIF API server is running locally on port 3186 in order to make sample requests.
|
||||
|
||||
This API should **not** be directly accessible outside of development environments. The endpoints of this API are not authenticated and relies on the Sourcegraph frontend to proxy requests via the HTTP or GraphQL server.
|
||||
@ -1,628 +0,0 @@
|
||||
openapi: 3.0.0
|
||||
info:
|
||||
title: LSIF API Server
|
||||
description: An internal Sourcegraph microservice that serves LSIF-powered code intelligence.
|
||||
version: 1.0.0
|
||||
contact:
|
||||
name: Eric Fritz
|
||||
email: eric@sourcegraph.com
|
||||
url: https://sourcegraph.com
|
||||
servers:
|
||||
- url: http://localhost:3186
|
||||
tags:
|
||||
- name: LSIF
|
||||
description: LSIF operations
|
||||
- name: Uploads
|
||||
description: Upload operations
|
||||
- name: Internal
|
||||
description: Internal operations
|
||||
paths:
|
||||
/upload:
|
||||
post:
|
||||
description: Upload LSIF data for a particular commit and directory. Exactly one file must be uploaded, and it is assumed to be the gzipped output of an LSIF indexer.
|
||||
tags:
|
||||
- LSIF
|
||||
requestBody:
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
parameters:
|
||||
- name: repositoryId
|
||||
in: query
|
||||
description: The repository identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: commit
|
||||
in: query
|
||||
description: The 40-character commit hash.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: root
|
||||
in: query
|
||||
description: The path to the directory associated with the upload, relative to the repository root.
|
||||
example: cmd/project1
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
- name: indexerName
|
||||
in: query
|
||||
description: The name of the indexer that generated the payload. This is required only if there is no tool info supplied in the payload's metadata vertex.
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: Processed (synchronously)
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/EnqueueResponse'
|
||||
'202':
|
||||
description: Accepted
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/EnqueueResponse'
|
||||
/exists:
|
||||
get:
|
||||
description: Determine if LSIF data exists for a file within a particular commit. This endpoint will return the LSIF upload for which definitions, references, and hover queries will use.
|
||||
tags:
|
||||
- LSIF
|
||||
parameters:
|
||||
- name: repositoryId
|
||||
in: query
|
||||
description: The repository identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: commit
|
||||
in: query
|
||||
description: The 40-character commit hash.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
- $ref: '#/components/schemas/Uploads'
|
||||
post:
|
||||
description: Determine if LSIF data exists for a file within a particular commit. This endpoint will return true if there is a nearby commit (direct ancestor or descendant) with LSIF data for the same file if the exact commit does not have available LSIF data.
|
||||
tags:
|
||||
- LSIF
|
||||
parameters:
|
||||
- name: repositoryId
|
||||
in: query
|
||||
description: The repository identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: commit
|
||||
in: query
|
||||
description: The 40-character commit hash.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: file
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: boolean
|
||||
/definitions:
|
||||
get:
|
||||
description: Get definitions for the symbol at a source position.
|
||||
tags:
|
||||
- LSIF
|
||||
parameters:
|
||||
- name: repositoryId
|
||||
in: query
|
||||
description: The repository identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: commit
|
||||
in: query
|
||||
description: The 40-character commit hash.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: line
|
||||
in: query
|
||||
description: The line index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: character
|
||||
in: query
|
||||
description: The character index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: uploadId
|
||||
in: query
|
||||
description: The identifier of the upload to load. If not supplied, the upload nearest to the given commit will be loaded.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Locations'
|
||||
'404':
|
||||
description: Not found
|
||||
/references:
|
||||
get:
|
||||
description: Get references for the symbol at a source position.
|
||||
tags:
|
||||
- LSIF
|
||||
parameters:
|
||||
- name: repositoryId
|
||||
in: query
|
||||
description: The repository identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: commit
|
||||
in: query
|
||||
description: The 40-character commit hash.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: line
|
||||
in: query
|
||||
description: The line index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: character
|
||||
in: query
|
||||
description: The character index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: uploadId
|
||||
in: query
|
||||
description: The identifier of the upload to load. If not supplied, the upload nearest to the given commit will be loaded.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: limit
|
||||
in: query
|
||||
description: The maximum number of locations to return in one page.
|
||||
required: false
|
||||
schema:
|
||||
type: number
|
||||
default: 10
|
||||
- name: cursor
|
||||
in: query
|
||||
description: The end cursor given in the response of a previous page.
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Locations'
|
||||
headers:
|
||||
Link:
|
||||
description: If there are more results, this header includes the URL of the next page with relation type *next*. See [RFC 5988](https://tools.ietf.org/html/rfc5988).
|
||||
schema:
|
||||
type: string
|
||||
'404':
|
||||
description: Not found
|
||||
/hover:
|
||||
get:
|
||||
description: Get hover data for the symbol at a source position.
|
||||
tags:
|
||||
- LSIF
|
||||
parameters:
|
||||
- name: repositoryId
|
||||
in: query
|
||||
description: The repository identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: commit
|
||||
in: query
|
||||
description: The 40-character commit hash.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: line
|
||||
in: query
|
||||
description: The line index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: character
|
||||
in: query
|
||||
description: The character index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: uploadId
|
||||
in: query
|
||||
description: The identifier of the upload to load. If not supplied, the upload nearest to the given commit will be loaded.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Hover'
|
||||
'404':
|
||||
description: Not found
|
||||
/uploads/repositories/{repositoryId}:
|
||||
get:
|
||||
description: Get LSIF uploads for a repository.
|
||||
tags:
|
||||
- Uploads
|
||||
parameters:
|
||||
- name: repositoryId
|
||||
in: query
|
||||
description: The repository identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: query
|
||||
in: query
|
||||
description: A search query applied over commit, root, failure reason, and failure stacktrace properties.
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
- name: state
|
||||
in: query
|
||||
description: The target upload state.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
- processing
|
||||
- errored
|
||||
- completed
|
||||
- queued
|
||||
- name: visibleAtTip
|
||||
in: query
|
||||
description: If true, only show uploads visible at tip.
|
||||
required: false
|
||||
schema:
|
||||
type: boolean
|
||||
- name: limit
|
||||
in: query
|
||||
description: The maximum number of uploads to return in one page.
|
||||
required: false
|
||||
schema:
|
||||
type: number
|
||||
default: 50
|
||||
- name: offset
|
||||
in: query
|
||||
description: The number of uploads seen on previous pages.
|
||||
required: false
|
||||
schema:
|
||||
type: number
|
||||
default: 0
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/PaginatedUploads'
|
||||
headers:
|
||||
Link:
|
||||
description: If there are more results, this header includes the URL of the next page with relation type *next*. See [RFC 5988](https://tools.ietf.org/html/rfc5988).
|
||||
schema:
|
||||
type: string
|
||||
/uploads/{id}:
|
||||
get:
|
||||
description: Get an LSIF upload by its identifier.
|
||||
tags:
|
||||
- Uploads
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
description: The upload identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Upload'
|
||||
'404':
|
||||
description: Not Found
|
||||
delete:
|
||||
description: Delete an LSIF upload by its identifier.
|
||||
tags:
|
||||
- Uploads
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
description: The upload identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: No Content
|
||||
'404':
|
||||
description: Not Found
|
||||
/states:
|
||||
get:
|
||||
description: Retrieve the state of a set of uploads by identifier.
|
||||
tags:
|
||||
- Internal
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
ids:
|
||||
description: The upload identifier list.
|
||||
type: array
|
||||
items:
|
||||
type: number
|
||||
additionalProperties: false
|
||||
required:
|
||||
- ids
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
const: 'map'
|
||||
values:
|
||||
description: A list of key/value pairs.
|
||||
type: array
|
||||
items:
|
||||
description: A key/value pair.
|
||||
type: array
|
||||
items:
|
||||
oneOf:
|
||||
- type: number
|
||||
- type: string
|
||||
additionalProperties: false
|
||||
required:
|
||||
- type
|
||||
- values
|
||||
/prune:
|
||||
post:
|
||||
description: Remove the oldest prunable dump.
|
||||
tags:
|
||||
- Internal
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
description: The identifier of the pruned dump.
|
||||
type: number
|
||||
additionalProperties: false
|
||||
required:
|
||||
- id
|
||||
nullable: true
|
||||
components:
|
||||
schemas:
|
||||
Position:
|
||||
type: object
|
||||
description: A cursor position in a source file.
|
||||
properties:
|
||||
line:
|
||||
type: number
|
||||
description: The (zero-index) line index.
|
||||
character:
|
||||
type: number
|
||||
description: The (zero-index) character index.
|
||||
required:
|
||||
- line
|
||||
- character
|
||||
additionalProperties: false
|
||||
Range:
|
||||
type: object
|
||||
description: A half-open range of positions in a source file.
|
||||
properties:
|
||||
start:
|
||||
$ref: '#/components/schemas/Position'
|
||||
end:
|
||||
$ref: '#/components/schemas/Position'
|
||||
required:
|
||||
- start
|
||||
- end
|
||||
additionalProperties: false
|
||||
Location:
|
||||
type: object
|
||||
description: A position in a file of a code base.
|
||||
properties:
|
||||
repositoryId:
|
||||
type: number
|
||||
description: The identifier of the repository in which the location occurs.
|
||||
commit:
|
||||
type: string
|
||||
description: The commit in which the location occurs.
|
||||
path:
|
||||
type: string
|
||||
description: The root-relative path to the file.
|
||||
range:
|
||||
$ref: '#/components/schemas/Range'
|
||||
required:
|
||||
- repositoryId
|
||||
- commit
|
||||
- path
|
||||
- range
|
||||
additionalProperties: false
|
||||
Locations:
|
||||
type: array
|
||||
description: A list of definition or reference locations.
|
||||
items:
|
||||
$ref: '#/components/schemas/Location'
|
||||
Hover:
|
||||
type: object
|
||||
description: The text associated with a position in a source file.
|
||||
properties:
|
||||
text:
|
||||
type: string
|
||||
description: The raw hover text.
|
||||
required:
|
||||
- text
|
||||
additionalProperties: false
|
||||
EnqueueResponse:
|
||||
type: object
|
||||
description: A payload indicating the enqueued upload.
|
||||
properties:
|
||||
id:
|
||||
type: number
|
||||
description: The upload identifier.
|
||||
required:
|
||||
- id
|
||||
additionalProperties: false
|
||||
Uploads:
|
||||
type: object
|
||||
description: A wrapper for a list of uploads.
|
||||
properties:
|
||||
uploads:
|
||||
type: array
|
||||
description: A list of uploads.
|
||||
items:
|
||||
$ref: '#/components/schemas/Upload'
|
||||
PaginatedUploads:
|
||||
type: object
|
||||
description: A paginated wrapper for a list of uploads.
|
||||
properties:
|
||||
uploads:
|
||||
type: array
|
||||
description: A list of uploads with a particular state.
|
||||
items:
|
||||
$ref: '#/components/schemas/Upload'
|
||||
totalCount:
|
||||
type: number
|
||||
description: The total number of uploads in this set of results.
|
||||
required:
|
||||
- uploads
|
||||
additionalProperties: false
|
||||
Upload:
|
||||
type: object
|
||||
description: An LSIF upload.
|
||||
properties:
|
||||
id:
|
||||
type: number
|
||||
description: A unique identifier.
|
||||
repositoryId:
|
||||
type: number
|
||||
description: The repository identifier argument given on upload.
|
||||
commit:
|
||||
type: string
|
||||
description: The commit argument given on upload.
|
||||
root:
|
||||
type: string
|
||||
description: The root argument given on upload.
|
||||
indexer:
|
||||
type: string
|
||||
description: The name of the indexer given on upload.
|
||||
filename:
|
||||
type: string
|
||||
description: The filename where the upload was stored before conversion.
|
||||
state:
|
||||
type: string
|
||||
description: The upload's current state.
|
||||
enum:
|
||||
- processing
|
||||
- errored
|
||||
- completed
|
||||
- queued
|
||||
failureSummary:
|
||||
type: string
|
||||
description: A brief description of why the upload conversion failed.
|
||||
failureStacktrace:
|
||||
type: string
|
||||
description: The stacktrace of the upload error.
|
||||
uploadedAt:
|
||||
type: string
|
||||
description: An RFC3339-formatted time that the upload was uploaded.
|
||||
startedAt:
|
||||
type: string
|
||||
description: An RFC3339-formatted time that the conversion started.
|
||||
nullable: true
|
||||
finishedAt:
|
||||
type: string
|
||||
description: An RFC3339-formatted time that the conversion completed or errored.
|
||||
nullable: true
|
||||
visibleAtTip:
|
||||
type: boolean
|
||||
description: Whether or not this upload can provide global reference code intelligence.
|
||||
placeInQueue:
|
||||
type: number
|
||||
description: The rank of this upload in the queue. The value of this field is null if the upload has been processed.
|
||||
nullable: true
|
||||
required:
|
||||
- id
|
||||
- repositoryId
|
||||
- commit
|
||||
- root
|
||||
- filename
|
||||
- state
|
||||
- failureSummary
|
||||
- failureStacktrace
|
||||
- uploadedAt
|
||||
- startedAt
|
||||
- finishedAt
|
||||
additionalProperties: false
|
||||
@ -1,462 +0,0 @@
|
||||
openapi: 3.0.0
|
||||
info:
|
||||
title: LSIF Bundle Manager
|
||||
description: An internal Sourcegraph microservice that serves LSIF-powered code intelligence for a single processed dump.
|
||||
version: 1.0.0
|
||||
contact:
|
||||
name: Eric Fritz
|
||||
email: eric@sourcegraph.com
|
||||
url: https://sourcegraph.com
|
||||
servers:
|
||||
- url: http://localhost:3187
|
||||
tags:
|
||||
- name: Uploads
|
||||
description: Upload operations
|
||||
- name: Query
|
||||
description: Query operations
|
||||
paths:
|
||||
/uploads/{id}:
|
||||
get:
|
||||
description: Retrieve raw LSIF content.
|
||||
tags:
|
||||
- Uploads
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The upload identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
post:
|
||||
description: Upload raw LSIF content.
|
||||
tags:
|
||||
- Uploads
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The upload identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
requestBody:
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
/dbs/{id}:
|
||||
post:
|
||||
description: Upload a processed LSIF database.
|
||||
tags:
|
||||
- Uploads
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
requestBody:
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema:
|
||||
type: string
|
||||
format: binary
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
/dbs/{id}/exists:
|
||||
get:
|
||||
description: Determine if a file path exists in the given database.
|
||||
tags:
|
||||
- Query
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ExistsResponse'
|
||||
/dbs/{id}/definitions:
|
||||
get:
|
||||
description: Retrieve a list of definition locations for a position in the given database.
|
||||
tags:
|
||||
- Query
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: line
|
||||
in: query
|
||||
description: The line index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: character
|
||||
in: query
|
||||
description: The character index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/DefinitionsResponse'
|
||||
/dbs/{id}/references:
|
||||
get:
|
||||
description: Retrieve a list of reference locations for a position in the given database.
|
||||
tags:
|
||||
- Query
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: line
|
||||
in: query
|
||||
description: The line index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: character
|
||||
in: query
|
||||
description: The character index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/ReferencesResponse'
|
||||
/dbs/{id}/hover:
|
||||
get:
|
||||
description: Retrieve hover data for a position in the given database.
|
||||
tags:
|
||||
- Query
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: line
|
||||
in: query
|
||||
description: The line index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: character
|
||||
in: query
|
||||
description: The character index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/HoverResponse'
|
||||
/dbs/{id}/monikersByPosition:
|
||||
get:
|
||||
description: Retrieve a list of monikers for a position in the given database.
|
||||
tags:
|
||||
- Query
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: line
|
||||
in: query
|
||||
description: The line index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: character
|
||||
in: query
|
||||
description: The character index (zero-indexed).
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/MonikersByPositionResponse'
|
||||
/dbs/{id}/monikerResults:
|
||||
get:
|
||||
description: Retrieve a list of locations associated with the given moniker in the given database.
|
||||
tags:
|
||||
- Query
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: modelType
|
||||
in: query
|
||||
description: The type of query.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
- definition
|
||||
- reference
|
||||
- name: scheme
|
||||
in: query
|
||||
description: The moniker scheme.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: identifier
|
||||
in: query
|
||||
description: The moniker identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: skip
|
||||
in: query
|
||||
description: The number of results to skip.
|
||||
required: false
|
||||
schema:
|
||||
type: number
|
||||
- name: take
|
||||
in: query
|
||||
description: The maximum number of results to return.
|
||||
required: false
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/MonikerResultsResponse'
|
||||
/dbs/{id}/packageInformation:
|
||||
get:
|
||||
description: Retrieve package information data by identifier.
|
||||
tags:
|
||||
- Query
|
||||
parameters:
|
||||
- name: id
|
||||
in: query
|
||||
description: The database identifier.
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
- name: path
|
||||
in: query
|
||||
description: The file path within the repository (relative to the repository root).
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: packageInformationId
|
||||
in: query
|
||||
description: The identifier of the target package information data.
|
||||
required: true
|
||||
schema:
|
||||
oneOf:
|
||||
- type: number
|
||||
- type: string
|
||||
responses:
|
||||
'200':
|
||||
description: OK
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/PackageInformationResponse'
|
||||
components:
|
||||
schemas:
|
||||
Position:
|
||||
type: object
|
||||
description: A cursor position in a source file.
|
||||
properties:
|
||||
line:
|
||||
type: number
|
||||
description: The (zero-index) line index.
|
||||
character:
|
||||
type: number
|
||||
description: The (zero-index) character index.
|
||||
required:
|
||||
- line
|
||||
- character
|
||||
additionalProperties: false
|
||||
Range:
|
||||
type: object
|
||||
description: A half-open range of positions in a source file.
|
||||
properties:
|
||||
start:
|
||||
$ref: '#/components/schemas/Position'
|
||||
end:
|
||||
$ref: '#/components/schemas/Position'
|
||||
required:
|
||||
- start
|
||||
- end
|
||||
additionalProperties: false
|
||||
Location:
|
||||
type: object
|
||||
description: A position in a file of a code base.
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
description: The root-relative path to the file.
|
||||
range:
|
||||
$ref: '#/components/schemas/Range'
|
||||
required:
|
||||
- path
|
||||
- range
|
||||
additionalProperties: false
|
||||
ExistsResponse:
|
||||
type: boolean
|
||||
DefinitionsResponse:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/Location'
|
||||
ReferencesResponse:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/Location'
|
||||
HoverResponse:
|
||||
type: object
|
||||
properties:
|
||||
text:
|
||||
type: string
|
||||
description: The hover text.
|
||||
range:
|
||||
$ref: '#/components/schemas/Range'
|
||||
description: The range that the hover text describes.
|
||||
additionalProperties: false
|
||||
required:
|
||||
- text
|
||||
- range
|
||||
nullable: true
|
||||
MonikersByPositionResponse:
|
||||
type: array
|
||||
description: A list of monikers grouped by matching ranges.
|
||||
items:
|
||||
type: array
|
||||
description: A list of monikers for a single range.
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
kind:
|
||||
type: string
|
||||
description: The kind of moniker.
|
||||
enum:
|
||||
- import
|
||||
- export
|
||||
- local
|
||||
scheme:
|
||||
type: string
|
||||
description: The moniker scheme.
|
||||
identifier:
|
||||
type: string
|
||||
description: The moniker identifier.
|
||||
packageInformationId:
|
||||
type: number
|
||||
description: The identifier of any associated package information.
|
||||
nullable: true
|
||||
additionalProperties: false
|
||||
required:
|
||||
- kind
|
||||
- scheme
|
||||
- identifier
|
||||
- packageInformationId
|
||||
MonikerResultsResponse:
|
||||
type: object
|
||||
properties:
|
||||
locations:
|
||||
type: array
|
||||
items:
|
||||
$ref: '#/components/schemas/Location'
|
||||
count:
|
||||
type: number
|
||||
description: The total number of matching locations for this moniker.
|
||||
additionalProperties: false
|
||||
required:
|
||||
- locations
|
||||
- count
|
||||
PackageInformationResponse:
|
||||
type: object
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
description: The name of the package.
|
||||
version:
|
||||
type: string
|
||||
description: The package version.
|
||||
nullable: true
|
||||
additionalProperties: false
|
||||
required:
|
||||
- name
|
||||
- version
|
||||
nullable: true
|
||||
@ -1,12 +0,0 @@
|
||||
# Configuration
|
||||
|
||||
The LSIF processes store most of their data in SQLite repositories on a shared disk that are written once by a worker on LSIF dump upload, and read many times by the APIs to answer LSIF/LSP queries. Cross-repository and commit graph data is stored in Postgres, as this database requires many concurrent writers (which is an unsafe operation for SQLite in a networked application). The LSIF processes retrieve PostgreSQL connection configuration from the frontend process on startup.
|
||||
|
||||
We rely on the Sourcegraph frontend to apply our DB migrations. These live in the `/migrations` folder. This means:
|
||||
|
||||
- The server, bundle manager, and worker wait for the frontend to apply the migration version it cares about before starting.
|
||||
- We (and more importantly, site admins) only have to care about a single set of DB schema migrations. This is the primary property we benefit from by doing this.
|
||||
|
||||
## Migrations
|
||||
|
||||
To add a new migration for the tables used by the LSIF processes, create a new migration in the frontend according to the instructions in [the migration documentation](../../../../migrations/README.md). Then, update the value of `MINIMUM_MIGRATION_VERSION` in [postgres.ts](../src/shared/database/postgres.ts) to be the timestamp from the generated filename.
|
||||
@ -1,234 +0,0 @@
|
||||
# LSIF data model
|
||||
|
||||
This document outlines the data model for a single LSIF dump. The definition of the database tables and the entities encoded within it can be found in [sqlite.ts](../src/shared/models/sqlite.ts).
|
||||
|
||||
In the following document, we collapse ranges to keep the document readable, where `a:b-c:d` is shorthand for the following:
|
||||
|
||||
```
|
||||
{
|
||||
"startLine": a,
|
||||
"startCharacter": b,
|
||||
"endLine": c,
|
||||
"endCharacter": d
|
||||
}
|
||||
```
|
||||
|
||||
This applies to JSON payloads, and a similar shorthand is used for the columns of the `definitions` and `references` tables.
|
||||
|
||||
## Running example
|
||||
|
||||
The following source files compose the package `sample`, which is used as the running example for this document.
|
||||
|
||||
**foo.ts**
|
||||
|
||||
```typescript
|
||||
export function foo(value: string): string {
|
||||
return value.substring(1, value.length - 1)
|
||||
}
|
||||
```
|
||||
|
||||
**bar.ts**
|
||||
|
||||
```typescript
|
||||
import { foo } from './foo'
|
||||
|
||||
export function bar(input: string): string {
|
||||
return foo(foo(input))
|
||||
}
|
||||
```
|
||||
|
||||
## Database tables
|
||||
|
||||
**`meta` table**
|
||||
|
||||
This table is populated with **exactly** one row containing the version of the LSIF input, the version of the software that converted it into a SQLite database, and the number used to determine in which result chunk a result identifier belongs (via hash and modulus over the number of chunks). Generally, this number will be the number of rows in the `resultChunks` table, but this number may be higher as we won't insert empty chunks (in the case that no identifier happened to hash to it).
|
||||
|
||||
The last value is used in order to achieve a consistent hash of identifiers that map to the correct result chunk row identifier. This will be explained in more detail later in this document.
|
||||
|
||||
| id | lsifVersion | sourcegraphVersion | numResultChunks |
|
||||
| --- | ----------- | ------------------ | --------------- |
|
||||
| 0 | 0.4.3 | 0.1.0 | 1 |
|
||||
|
||||
**`documents` table**
|
||||
|
||||
This table is populated with a gzipped JSON payload that represents the ranges as well as each range's definition, reference, and hover result identifiers. The table is indexed on the path of the document relative to the project root.
|
||||
|
||||
| path | data |
|
||||
| ------ | ---------------------------- |
|
||||
| foo.ts | _gzipped_ and _json-encoded_ |
|
||||
| bar.ts | _gzipped_ and _json-encoded_ |
|
||||
|
||||
Each payload has the following form. As the documents are large, we show only the decoded version for `foo.ts`.
|
||||
|
||||
**encoded `foo.ts` payload**
|
||||
|
||||
````json
|
||||
{
|
||||
"ranges": {
|
||||
"9": {
|
||||
"range": "0:0-0:0",
|
||||
"definitionResultId": "49",
|
||||
"referenceResultId": "52",
|
||||
"monikerIds": ["9007199254740990"]
|
||||
},
|
||||
"14": {
|
||||
"range": "0:16-0:19",
|
||||
"definitionResultId": "55",
|
||||
"referenceResultId": "58",
|
||||
"hoverResultId": "16",
|
||||
"monikerIds": ["9007199254740987"]
|
||||
},
|
||||
"21": {
|
||||
"range": "0:20-0:25",
|
||||
"definitionResultId": "61",
|
||||
"referenceResultId": "64",
|
||||
"hoverResultId": "23",
|
||||
"monikerIds": []
|
||||
},
|
||||
"25": {
|
||||
"range": "1:9-1:14",
|
||||
"definitionResultId": "61",
|
||||
"referenceResultId": "64",
|
||||
"hoverResultId": "23",
|
||||
"monikerIds": []
|
||||
},
|
||||
"36": {
|
||||
"range": "1:15-1:24",
|
||||
"definitionResultId": "144",
|
||||
"referenceResultId": "68",
|
||||
"hoverResultId": "34",
|
||||
"monikerIds": ["30"]
|
||||
},
|
||||
"38": {
|
||||
"range": "1:28-1:33",
|
||||
"definitionResultId": "61",
|
||||
"referenceResultId": "64",
|
||||
"hoverResultId": "23",
|
||||
"monikerIds": []
|
||||
},
|
||||
"47": {
|
||||
"range": "1:34-1:40",
|
||||
"definitionResultId": "148",
|
||||
"referenceResultId": "71",
|
||||
"hoverResultId": "45",
|
||||
"monikerIds": []
|
||||
}
|
||||
},
|
||||
"hoverResults": {
|
||||
"16": "```typescript\nfunction foo(value: string): string\n```",
|
||||
"23": "```typescript\n(parameter) value: string\n```",
|
||||
"34": "```typescript\n(method) String.substring(start: number, end?: number): string\n```\n\n---\n\nReturns the substring at the specified location within a String object.",
|
||||
"45": "```typescript\n(property) String.length: number\n```\n\n---\n\nReturns the length of a String object."
|
||||
},
|
||||
"monikers": {
|
||||
"9007199254740987": {
|
||||
"kind": "export",
|
||||
"scheme": "npm",
|
||||
"identifier": "sample:foo:foo",
|
||||
"packageInformationId": "9007199254740991"
|
||||
},
|
||||
"9007199254740990": {
|
||||
"kind": "export",
|
||||
"scheme": "npm",
|
||||
"identifier": "sample:foo:",
|
||||
"packageInformationId": "9007199254740991"
|
||||
}
|
||||
},
|
||||
"packageInformation": {
|
||||
"9007199254740991": {
|
||||
"name": "sample",
|
||||
"version": "0.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
````
|
||||
|
||||
The `ranges` field holds a map from range identifier range data including the extents within the source code and optional fields for a definition result, a reference result, and a hover result. Each range also has a possibly empty list of moniker ids. The hover result and moniker identifiers index into the `hoverResults` and `monikers` field of the document. The definition and reference result identifiers index into a result chunk payload, as described below.
|
||||
|
||||
**`resultChunks` table**
|
||||
|
||||
Originally, definition and reference results were stored inline in the document payload. However, this caused document payloads to be come massive in some circumstances (for instance, where the reference result of a frequently used symbol includes multiple ranges in every document of the project). In order to keep each row to a manageable and cacheable size, the definition and reference results were moved into a separate table. The size of each result chunk can then be controlled by varying _how many_ result chunks there are available in a database. It may also be worth noting here that hover result and monikers are best left inlined, as normalizing the former would require another SQL lookup on hover queries, and normalizing the latter would require a SQL lookup per moniker attached to a range; normalizing either does not have a large effect on the size of the document payload.
|
||||
|
||||
This table is populated with gzipped JSON payloads that contains a mapping from definition result or reference result identifiers to the set of ranges that compose that result. A definition or reference result may be referred to by many documents, which is why it is encoded separately. The table is indexed on the common hash of each definition and reference result id inserted in this chunk.
|
||||
|
||||
| id | data |
|
||||
| --- | ---------------------------- |
|
||||
| 0 | _gzipped_ and _json-encoded_ |
|
||||
|
||||
Each payload has the following form.
|
||||
|
||||
**encoded result chunk #0 payload**
|
||||
|
||||
```json
|
||||
{
|
||||
"documentPaths": {
|
||||
"4": "foo.ts",
|
||||
"80": "bar.ts"
|
||||
},
|
||||
"documentIdRangeIds": {
|
||||
"49": [{ "documentId": "4", "rangeId": "9" }],
|
||||
"55": [{ "documentId": "4", "rangeId": "4" }],
|
||||
"61": [{ "documentId": "4", "rangeId": "21" }],
|
||||
"71": [{ "documentId": "4", "rangeId": "47" }],
|
||||
"52": [
|
||||
{ "documentId": "4", "rangeId": "9" },
|
||||
{ "documentId": "80", "rangeId": "95" }
|
||||
],
|
||||
"58": [
|
||||
{ "documentId": "4", "rangeId": "14" },
|
||||
{ "documentId": "80", "rangeId": "91" },
|
||||
{ "documentId": "80", "rangeId": "111" },
|
||||
{ "documentId": "80", "rangeId": "113" }
|
||||
],
|
||||
"64": [
|
||||
{ "documentId": "4", "rangeId": "21" },
|
||||
{ "documentId": "4", "rangeId": "25" },
|
||||
{ "documentId": "4", "rangeId": "38" }
|
||||
],
|
||||
"68": [{ "documentId": "4", "rangeId": "36" }],
|
||||
"117": [{ "documentId": "80", "rangeId": "85" }],
|
||||
"120": [{ "documentId": "80", "rangeId": "85" }],
|
||||
"125": [{ "documentId": "80", "rangeId": "100" }],
|
||||
"128": [{ "documentId": "80", "rangeId": "100" }],
|
||||
"131": [{ "documentId": "80", "rangeId": "107" }],
|
||||
"134": [
|
||||
{ "documentId": "80", "rangeId": "107" },
|
||||
{ "documentId": "80", "rangeId": "115" }
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `documentIdRangeIds` field store a list of _pairs_ of document identifiers and range identifiers. To look up a range in this format, the `documentId` must be translated into a document path via the `documentPaths` field. This gives the primary key of the document containing the range in the `documents` table, and the range identifier can be looked up in the decoded payload.
|
||||
|
||||
To retrieve a definition or reference result by its identifier, we must first determine in which result chunk it is defined. This requires that we take the hash of the identifier (modulo the `numResultChunks` field of the `meta` table). This gives us the unique identifier into the `resultChunks` table. In the running example of this document, there is only one result chunk. Larger dumps will have a greater number of result chunks to keep the amount of data encoded in a single database row reasonable.
|
||||
|
||||
**definitions table**
|
||||
|
||||
This table is populated with the monikers of a range and that range's definition result. The table is indexed on the `(scheme, identifier)` pair to allow quick lookup by moniker.
|
||||
|
||||
| id | scheme | identifier | documentPath | range |
|
||||
| --- | ------ | -------------- | ------------ | ------------ |
|
||||
| 1 | npm | sample:foo: | foo.ts | 0:0 to 0:0 |
|
||||
| 2 | npm | sample:foo:foo | foo.ts | 0:16 to 0:19 |
|
||||
| 3 | npm | sample:bar: | bar.ts | 0:0 to 0:0 |
|
||||
| 4 | npm | sample:bar:bar | bar.ts | 2:16 to 2:19 |
|
||||
|
||||
The row with id `2` correlates the `npm` moniker for the `foo` function with the range where it is defined in `foo.ts`. Similarly, the row with id `4` correlates the exported `npm` moniker for the `bar` function with the range where it is defined in `bar.ts`.
|
||||
|
||||
**references table**
|
||||
|
||||
This table is populated with the monikers of a range and that range's reference result. The table is indexed on the `(scheme, identifier)` pair to allow quick lookup by moniker.
|
||||
|
||||
| id | scheme | identifier | documentPath | range |
|
||||
| --- | ------ | -------------- | ------------ | ------------ |
|
||||
| 1 | npm | sample:foo | foo.ts | 0:0 to 0:0 |
|
||||
| 2 | npm | sample:foo | bar.ts | 0:20 to 0:27 |
|
||||
| 3 | npm | sample:bar | bar.ts | 0:0 to 0:0 |
|
||||
| 4 | npm | sample:foo:foo | foo.ts | 0:16 to 0:19 |
|
||||
| 5 | npm | sample:foo:foo | bar.ts | 0:9 to 0:12 |
|
||||
| 6 | npm | sample:foo:foo | bar.ts | 3:9 to 3:12 |
|
||||
| 7 | npm | sample:foo:foo | bar.ts | 3:13 to 3:16 |
|
||||
| 8 | npm | sample:bar:bar | bar.ts | 2:16 to 2:19 |
|
||||
|
||||
The row with ids `4` through `7` correlate the `npm` moniker for the `foo` function with its references: the definition in `foo.ts`, its import in `bar.ts`, and its two uses in `bar.ts`, respectively.
|
||||
@ -1,68 +0,0 @@
|
||||
# Cross-repo data model
|
||||
|
||||
This document outlines the data model used to correlate multiple LSIF dumps. The definition of the cross-repo database tables can be found in [pg.ts](../src/shared/models/pg.ts).
|
||||
|
||||
In the following document, commits have been abbreviated to 7 characters for readability.
|
||||
|
||||
## Database tables
|
||||
|
||||
**`lsif_commits` table**
|
||||
|
||||
This table contains all commits known for a repository for which LSIF data has been uploaded. Each commit consists of one or more rows indicating their parent. If a commit has no parent, then the parentCommit field is an empty string.
|
||||
|
||||
| id | repository_id | commit | parent_commit |
|
||||
| --- | ------------- | --------- | ------------- |
|
||||
| 1 | 6 | `a360643` | `f4fb066` |
|
||||
| 2 | 6 | `f4fb066` | `4c8d9dc` |
|
||||
| 3 | 6 | `313082b` | `4c8d9dc` |
|
||||
| 4 | 6 | `4c8d9dc` | `d67b8de` |
|
||||
| 5 | 6 | `d67b8de` | `323e23f` |
|
||||
| 6 | 6 | `323e23f` | |
|
||||
|
||||
This table allows us to ues recursive CTEs to find ancestor and descendant commits with a particular property (as indicated by the existence of an entry in the `lsif_dumps` table) and enables closest commit functionality.
|
||||
|
||||
**`lsif_uploads` table**
|
||||
|
||||
This table contains an entry for each LSIF upload. An upload is inserted with the state `queued` and is processed asynchronously by a worker process. The `root` field indicates the directory for which this upload provides code intelligence. The `indexer` field indicates the tool that generated the input. The `visible_at_tip` field indicates whether this a (completed) upload that is closest to the tip of the default branch.
|
||||
|
||||
| id | repository_id | commit | root | indexer | state | visible_at_tip |
|
||||
| --- | ------------- | --------- | ---- | ------- | --------- | -------------- |
|
||||
| 1 | 6 | `a360643` | | lsif-go | completed | true |
|
||||
| 2 | 6 | `f4fb066` | | lsif-go | completed | false |
|
||||
| 3 | 6 | `4c8d9dc` | cmd | lsif-go | completed | true |
|
||||
| 4 | 6 | `323e23f` | | lsif-go | completed | false |
|
||||
|
||||
The view `lsif_dumps` selects all uploads with a state of `completed`.
|
||||
|
||||
Additional fields are not shown in the table above which do not affect code intelligence queries in a meaningful way.
|
||||
|
||||
- `filename`: The filename of the raw upload.
|
||||
- `uploaded_at`: The time the record was inserted.
|
||||
- `started_at`: The time the conversion was started.
|
||||
- `finished_at`: The time the conversion was finished.
|
||||
- `failure_summary`: The message of the error that occurred during conversion.
|
||||
- `failure_stacktrace`: The stacktrace of the error that occurred during conversion.
|
||||
- `tracing_context`: The tracing context from the `/upload` endpoint. Used to trace the entire span of work from the upload to the end of conversion.
|
||||
|
||||
**`lsif_packages` table**
|
||||
|
||||
This table links a package manager-specific identifier and version to the LSIF upload _provides_ the package. The scheme, name, and version values are correlated with a moniker and its package information from an LSIF dump.
|
||||
|
||||
| id | scheme | name | version | dump_id |
|
||||
| --- | ------ | ------ | ------- | ------- |
|
||||
| 1 | npm | sample | 0.1.0 | 6 |
|
||||
|
||||
This table enables cross-repository jump-to-definition. When a range has no definition result but does have an _import_ moniker, the scheme, name, and version of the moniker can be queried in this table to get the repository and commit of the package that should contain that moniker's definition.
|
||||
|
||||
**`lsif_references` table**
|
||||
|
||||
This table links an LSIF upload to the set of packages on which it depends. This table shares common columns with the `lsif_packages` table, which are documented above. In addition, this table also has a `filter` column, which encodes a [bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) populated with the set of identifiers that the commit imports from the dependent package.
|
||||
|
||||
| id | scheme | name | version | filter | dump_id |
|
||||
| --- | ------ | --------- | ------- | ---------------------------- | ------- |
|
||||
| 1 | npm | left-pad | 0.1.0 | _gzipped_ and _json-encoded_ | 6 |
|
||||
| 2 | npm | right-pad | 1.2.3 | _gzipped_ and _json-encoded_ | 6 |
|
||||
| 2 | npm | left-pad | 0.1.0 | _gzipped_ and _json-encoded_ | 7 |
|
||||
| 2 | npm | right-pad | 1.2.4 | _gzipped_ and _json-encoded_ | 7 |
|
||||
|
||||
This table enables global find-references. When finding all references of a definition that has an _export_ moniker, the set of repositories and commits that depend on the package of that moniker are queried. We want to open only the databases that import this particular symbol (not all projects depending on this package import the identifier under query). To do this, the bloom filter is deserialized and queried for the identifier under query. A positive response from a bloom filter indicates that the identifier may be present in the set; a negative response from the bloom filter indicates that the identifier is _definitely_ not in the set. We only open the set of databases for which the bloom filter query responds positively.
|
||||
@ -1,7 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
/** @type {jest.InitialOptions} */
|
||||
const config = require('../../jest.config.base')
|
||||
|
||||
/** @type {jest.InitialOptions} */
|
||||
module.exports = { ...config, setupFilesAfterEnv: ['./jest.setup.js'], displayName: 'lsif', rootDir: __dirname }
|
||||
@ -1,5 +0,0 @@
|
||||
// LSIF tests create and migrate Postgres databases, which can take more
|
||||
// time than the default test timeout. Increase it here for all tests in
|
||||
// this project.
|
||||
|
||||
jest.setTimeout(15000)
|
||||
@ -1,82 +0,0 @@
|
||||
{
|
||||
"private": true,
|
||||
"name": "precise-code-intel",
|
||||
"description": "Precise code intelligence services for Sourcegraph",
|
||||
"author": "Sourcegraph",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"build": "tsc -b .",
|
||||
"test": "jest",
|
||||
"eslint": "../../node_modules/.bin/eslint --cache 'src/**/*.ts?(x)'",
|
||||
"run:api-server": "tsc-watch --onSuccess \"node -r source-map-support/register out/api-server/api.js\" --noClear",
|
||||
"run:bundle-manager": "tsc-watch --onSuccess \"node -r source-map-support/register out/bundle-manager/manager.js\" --noClear",
|
||||
"run:worker": "tsc-watch --onSuccess \"node -r source-map-support/register out/worker/worker.js\" --noClear"
|
||||
},
|
||||
"dependencies": {
|
||||
"async-middleware": "^1.2.1",
|
||||
"async-polling": "^0.2.1",
|
||||
"bloomfilter": "^0.0.18",
|
||||
"body-parser": "^1.19.0",
|
||||
"crc-32": "^1.2.0",
|
||||
"delay": "^4.3.0",
|
||||
"express": "^4.17.1",
|
||||
"express-opentracing": "^0.1.1",
|
||||
"express-validator": "^6.4.0",
|
||||
"express-winston": "^4.0.3",
|
||||
"got": "^10.7.0",
|
||||
"jaeger-client": "^3.17.2",
|
||||
"json5": "^2.1.1",
|
||||
"lodash": "^4.17.15",
|
||||
"logform": "^2.1.2",
|
||||
"lsif-protocol": "0.4.3",
|
||||
"mz": "^2.7.0",
|
||||
"on-finished": "^2.3.0",
|
||||
"opentracing": "^0.14.4",
|
||||
"p-retry": "^4.2.0",
|
||||
"pg": "^7.18.2",
|
||||
"prom-client": "^12.0.0",
|
||||
"relateurl": "^0.2.7",
|
||||
"rmfr": "^2.0.0",
|
||||
"sqlite3": "^4.1.1",
|
||||
"stream-throttle": "^0.1.3",
|
||||
"triple-beam": "^1.3.0",
|
||||
"typeorm": "^0.2.24",
|
||||
"uuid": "^7.0.3",
|
||||
"vscode-languageserver": "^6.1.1",
|
||||
"winston": "^3.2.1",
|
||||
"yallist": "^4.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@sourcegraph/tsconfig": "^4.0.1",
|
||||
"@types/async-polling": "0.0.3",
|
||||
"@types/bloomfilter": "0.0.0",
|
||||
"@types/body-parser": "1.19.0",
|
||||
"@types/express": "4.17.4",
|
||||
"@types/express-winston": "3.0.4",
|
||||
"@types/got": "9.6.10",
|
||||
"@types/jaeger-client": "3.15.3",
|
||||
"@types/jest": "25.2.1",
|
||||
"@types/json5": "0.0.30",
|
||||
"@types/lodash": "4.14.150",
|
||||
"@types/logform": "1.2.0",
|
||||
"@types/mz": "2.7.0",
|
||||
"@types/on-finished": "2.3.1",
|
||||
"@types/relateurl": "0.2.28",
|
||||
"@types/rmfr": "2.0.0",
|
||||
"@types/sinon": "9.0.0",
|
||||
"@types/stream-throttle": "0.1.0",
|
||||
"@types/triple-beam": "1.3.0",
|
||||
"@types/uuid": "7.0.3",
|
||||
"@types/yallist": "3.0.1",
|
||||
"babel-jest": "^25.2.6",
|
||||
"copyfiles": "^2.2.0",
|
||||
"eslint-plugin-import": "^2.20.2",
|
||||
"jest": "^25.2.7",
|
||||
"nock": "^12.0.2",
|
||||
"sinon": "^9.0.1",
|
||||
"source-map-support": "^0.5.16",
|
||||
"tsc-watch": "^4.2.3",
|
||||
"typescript": "^3.7.2",
|
||||
"typescript-json-schema": "^0.42.0"
|
||||
}
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
declare module 'express-opentracing'
|
||||
|
||||
declare function middleware(options?: { tracer?: Tracer }): Handler
|
||||
@ -1,80 +0,0 @@
|
||||
import * as metrics from './metrics'
|
||||
import * as settings from './settings'
|
||||
import promClient from 'prom-client'
|
||||
import { Backend } from './backend/backend'
|
||||
import { createLogger } from '../shared/logging'
|
||||
import { createLsifRouter } from './routes/lsif'
|
||||
import { createPostgresConnection } from '../shared/database/postgres'
|
||||
import { createTracer } from '../shared/tracing'
|
||||
import { createUploadRouter } from './routes/uploads'
|
||||
import { ensureDirectory } from '../shared/paths'
|
||||
import { Logger } from 'winston'
|
||||
import { startTasks } from './tasks'
|
||||
import { UploadManager } from '../shared/store/uploads'
|
||||
import { waitForConfiguration } from '../shared/config/config'
|
||||
import { DumpManager } from '../shared/store/dumps'
|
||||
import { DependencyManager } from '../shared/store/dependencies'
|
||||
import { SRC_FRONTEND_INTERNAL } from '../shared/config/settings'
|
||||
import { startExpressApp } from '../shared/api/init'
|
||||
import { createInternalRouter } from './routes/internal'
|
||||
|
||||
/**
|
||||
* Runs the HTTP server that accepts LSIF dump uploads and responds to LSIF requests.
|
||||
*
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
async function main(logger: Logger): Promise<void> {
|
||||
// Collect process metrics
|
||||
promClient.collectDefaultMetrics({ prefix: 'lsif_' })
|
||||
|
||||
// Read configuration from frontend
|
||||
const fetchConfiguration = await waitForConfiguration(logger)
|
||||
|
||||
// Configure distributed tracing
|
||||
const tracer = createTracer('precise-code-intel-api-server', fetchConfiguration())
|
||||
|
||||
// Ensure storage roots exist
|
||||
await ensureDirectory(settings.STORAGE_ROOT)
|
||||
|
||||
// Create database connection and entity wrapper classes
|
||||
const connection = await createPostgresConnection(fetchConfiguration(), logger)
|
||||
const dumpManager = new DumpManager(connection)
|
||||
const uploadManager = new UploadManager(connection)
|
||||
const dependencyManager = new DependencyManager(connection)
|
||||
const backend = new Backend(dumpManager, dependencyManager, SRC_FRONTEND_INTERNAL)
|
||||
|
||||
// Start background tasks
|
||||
startTasks(connection, uploadManager, logger)
|
||||
|
||||
const routers = [
|
||||
createUploadRouter(dumpManager, uploadManager, logger),
|
||||
createLsifRouter(connection, backend, uploadManager, logger, tracer),
|
||||
createInternalRouter(dumpManager, uploadManager, logger),
|
||||
]
|
||||
|
||||
// Start server
|
||||
startExpressApp({ port: settings.HTTP_PORT, routers, logger, tracer, selectHistogram })
|
||||
}
|
||||
|
||||
function selectHistogram(route: string): promClient.Histogram<string> | undefined {
|
||||
switch (route) {
|
||||
case '/upload':
|
||||
return metrics.httpUploadDurationHistogram
|
||||
|
||||
case '/exists':
|
||||
case '/request':
|
||||
return metrics.httpQueryDurationHistogram
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Initialize logger
|
||||
const appLogger = createLogger('precise-code-intel-api-server')
|
||||
|
||||
// Run app!
|
||||
main(appLogger).catch(error => {
|
||||
appLogger.error('failed to start process', { error })
|
||||
appLogger.on('finish', () => process.exit(1))
|
||||
appLogger.end()
|
||||
})
|
||||
@ -1,633 +0,0 @@
|
||||
import * as sinon from 'sinon'
|
||||
import * as lsif from 'lsif-protocol'
|
||||
import * as pgModels from '../../shared/models/pg'
|
||||
import { Backend, sortMonikers } from './backend'
|
||||
import { DependencyManager } from '../../shared/store/dependencies'
|
||||
import { DumpManager } from '../../shared/store/dumps'
|
||||
import { Database } from './database'
|
||||
import { createCleanPostgresDatabase } from '../../shared/test-util'
|
||||
import { Connection } from 'typeorm'
|
||||
import { OrderedLocationSet, ResolvedInternalLocation } from './location'
|
||||
import { ReferencePaginationCursor } from './cursor'
|
||||
import { range } from 'lodash'
|
||||
|
||||
const zeroUpload: pgModels.LsifUpload = {
|
||||
id: 0,
|
||||
repositoryId: 0,
|
||||
commit: '',
|
||||
root: '',
|
||||
indexer: '',
|
||||
state: 'queued',
|
||||
numParts: 1,
|
||||
uploadedParts: [0],
|
||||
uploadedAt: new Date(),
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
failureSummary: null,
|
||||
failureStacktrace: null,
|
||||
visibleAtTip: false,
|
||||
}
|
||||
|
||||
const zeroDump: pgModels.LsifDump = {
|
||||
...zeroUpload,
|
||||
state: 'completed',
|
||||
processedAt: new Date(),
|
||||
}
|
||||
|
||||
const zeroPackage = {
|
||||
id: 0,
|
||||
scheme: '',
|
||||
name: '',
|
||||
version: '',
|
||||
dump: null,
|
||||
dump_id: 0,
|
||||
filter: Buffer.from(''),
|
||||
}
|
||||
|
||||
const monikersWithPackageInformation = [
|
||||
{ kind: lsif.MonikerKind.local, scheme: 'test', identifier: 'm1' },
|
||||
{ kind: lsif.MonikerKind.import, scheme: 'test', identifier: 'm2', packageInformationId: 71 },
|
||||
{ kind: lsif.MonikerKind.import, scheme: 'test', identifier: 'm3' },
|
||||
]
|
||||
|
||||
const makeRange = (i: number) => ({
|
||||
start: { line: i + 1, character: (i + 1) * 10 },
|
||||
end: { line: i + 1, character: (i + 1) * 10 + 5 },
|
||||
})
|
||||
|
||||
const createTestDatabase = (dbs: Map<pgModels.DumpId, Database>) => (dumpId: pgModels.DumpId) => {
|
||||
const db = dbs.get(dumpId)
|
||||
if (!db) {
|
||||
throw new Error(`Unexpected database construction (dumpId=${dumpId})`)
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
|
||||
describe('Backend', () => {
|
||||
let connection!: Connection
|
||||
let cleanup!: () => Promise<void>
|
||||
let dumpManager!: DumpManager
|
||||
let dependencyManager!: DependencyManager
|
||||
|
||||
beforeAll(async () => {
|
||||
;({ connection, cleanup } = await createCleanPostgresDatabase())
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
beforeEach(() => {
|
||||
dumpManager = new DumpManager(connection)
|
||||
dependencyManager = new DependencyManager(connection)
|
||||
})
|
||||
|
||||
describe('exists', () => {
|
||||
it('should return closest dumps with file', async () => {
|
||||
const database1 = new Database(1)
|
||||
const database2 = new Database(2)
|
||||
const database3 = new Database(3)
|
||||
const database4 = new Database(4)
|
||||
|
||||
// Commit graph traversal
|
||||
sinon.stub(dumpManager, 'findClosestDumps').resolves([
|
||||
{ ...zeroDump, id: 1 },
|
||||
{ ...zeroDump, id: 2 },
|
||||
{ ...zeroDump, id: 3 },
|
||||
{ ...zeroDump, id: 4 },
|
||||
])
|
||||
|
||||
// Path existence check
|
||||
const spy1 = sinon.stub(database1, 'exists').resolves(true)
|
||||
const spy2 = sinon.stub(database2, 'exists').resolves(false)
|
||||
const spy3 = sinon.stub(database3, 'exists').resolves(false)
|
||||
const spy4 = sinon.stub(database4, 'exists').resolves(true)
|
||||
|
||||
const dumps = await new Backend(
|
||||
dumpManager,
|
||||
dependencyManager,
|
||||
'',
|
||||
createTestDatabase(
|
||||
new Map([
|
||||
[1, database1],
|
||||
[2, database2],
|
||||
[3, database3],
|
||||
[4, database4],
|
||||
])
|
||||
)
|
||||
).exists(42, 'deadbeef', '/foo/bar/baz.ts')
|
||||
|
||||
expect(dumps).toEqual([
|
||||
{ ...zeroDump, id: 1 },
|
||||
{ ...zeroDump, id: 4 },
|
||||
])
|
||||
expect(spy1.args[0][0]).toEqual('/foo/bar/baz.ts')
|
||||
expect(spy2.args[0][0]).toEqual('/foo/bar/baz.ts')
|
||||
expect(spy3.args[0][0]).toEqual('/foo/bar/baz.ts')
|
||||
expect(spy4.args[0][0]).toEqual('/foo/bar/baz.ts')
|
||||
})
|
||||
})
|
||||
|
||||
describe('definitions', () => {
|
||||
it('should return definitions from database', async () => {
|
||||
const database1 = new Database(1)
|
||||
|
||||
// Loading source dump
|
||||
sinon.stub(dumpManager, 'getDumpById').resolves({ ...zeroDump, id: 1 })
|
||||
|
||||
// Resolving target dumps
|
||||
sinon.stub(dumpManager, 'getDumpsByIds').resolves(
|
||||
new Map([
|
||||
[1, { ...zeroDump, id: 1 }],
|
||||
[2, { ...zeroDump, id: 2 }],
|
||||
[3, { ...zeroDump, id: 3 }],
|
||||
[4, { ...zeroDump, id: 4 }],
|
||||
])
|
||||
)
|
||||
|
||||
// In-database definitions
|
||||
sinon.stub(database1, 'definitions').resolves([
|
||||
{ dumpId: 1, path: '1.ts', range: makeRange(1) },
|
||||
{ dumpId: 2, path: '2.ts', range: makeRange(2) },
|
||||
{ dumpId: 3, path: '3.ts', range: makeRange(3) },
|
||||
{ dumpId: 4, path: '4.ts', range: makeRange(4) },
|
||||
])
|
||||
|
||||
const locations = await new Backend(
|
||||
dumpManager,
|
||||
dependencyManager,
|
||||
'',
|
||||
createTestDatabase(new Map([[1, database1]]))
|
||||
).definitions(42, 'deadbeef', '/foo/bar/baz.ts', { line: 5, character: 10 }, 1)
|
||||
|
||||
expect(locations).toEqual([
|
||||
{ dump: { ...zeroDump, id: 1 }, path: '1.ts', range: makeRange(1) },
|
||||
{ dump: { ...zeroDump, id: 2 }, path: '2.ts', range: makeRange(2) },
|
||||
{ dump: { ...zeroDump, id: 3 }, path: '3.ts', range: makeRange(3) },
|
||||
{ dump: { ...zeroDump, id: 4 }, path: '4.ts', range: makeRange(4) },
|
||||
])
|
||||
})
|
||||
|
||||
it('should return definitions from local moniker search', async () => {
|
||||
const database1 = new Database(1)
|
||||
|
||||
// Loading source dump
|
||||
sinon.stub(dumpManager, 'getDumpById').resolves({ ...zeroDump, id: 1 })
|
||||
|
||||
// Resolving target dumps
|
||||
sinon.stub(dumpManager, 'getDumpsByIds').resolves(
|
||||
new Map([
|
||||
[1, { ...zeroDump, id: 1 }],
|
||||
[2, { ...zeroDump, id: 2 }],
|
||||
[3, { ...zeroDump, id: 3 }],
|
||||
[4, { ...zeroDump, id: 4 }],
|
||||
])
|
||||
)
|
||||
|
||||
// In-database definitions
|
||||
sinon.stub(database1, 'definitions').resolves([])
|
||||
|
||||
// Moniker resolution
|
||||
sinon.stub(database1, 'monikersByPosition').resolves([monikersWithPackageInformation])
|
||||
|
||||
// Moniker search
|
||||
sinon.stub(database1, 'monikerResults').resolves({
|
||||
locations: [
|
||||
{ dumpId: 1, path: '1.ts', range: makeRange(1) },
|
||||
{ dumpId: 2, path: '2.ts', range: makeRange(2) },
|
||||
{ dumpId: 3, path: '3.ts', range: makeRange(3) },
|
||||
{ dumpId: 4, path: '4.ts', range: makeRange(4) },
|
||||
],
|
||||
count: 4,
|
||||
})
|
||||
|
||||
const locations = await new Backend(
|
||||
dumpManager,
|
||||
dependencyManager,
|
||||
'',
|
||||
createTestDatabase(new Map([[1, database1]]))
|
||||
).definitions(42, 'deadbeef', '/foo/bar/baz.ts', { line: 5, character: 10 }, 1)
|
||||
|
||||
expect(locations).toEqual([
|
||||
{ dump: { ...zeroDump, id: 1 }, path: '1.ts', range: makeRange(1) },
|
||||
{ dump: { ...zeroDump, id: 2 }, path: '2.ts', range: makeRange(2) },
|
||||
{ dump: { ...zeroDump, id: 3 }, path: '3.ts', range: makeRange(3) },
|
||||
{ dump: { ...zeroDump, id: 4 }, path: '4.ts', range: makeRange(4) },
|
||||
])
|
||||
})
|
||||
|
||||
it('should return definitions from remote moniker search', async () => {
|
||||
const database1 = new Database(1)
|
||||
const database2 = new Database(2)
|
||||
|
||||
// Loading source dump
|
||||
sinon.stub(dumpManager, 'getDumpById').resolves({ ...zeroDump, id: 1 })
|
||||
|
||||
// Resolving target dumps
|
||||
sinon.stub(dumpManager, 'getDumpsByIds').resolves(
|
||||
new Map([
|
||||
[1, { ...zeroDump, id: 1 }],
|
||||
[2, { ...zeroDump, id: 2 }],
|
||||
[3, { ...zeroDump, id: 3 }],
|
||||
[4, { ...zeroDump, id: 4 }],
|
||||
])
|
||||
)
|
||||
|
||||
// In-database definitions
|
||||
sinon.stub(database1, 'definitions').resolves([])
|
||||
|
||||
// Moniker resolution
|
||||
sinon.stub(database1, 'monikersByPosition').resolves([monikersWithPackageInformation])
|
||||
|
||||
// Package resolution
|
||||
sinon.stub(database1, 'packageInformation').resolves({ name: 'pkg2', version: '0.0.1' })
|
||||
|
||||
// Package resolution
|
||||
sinon.stub(dependencyManager, 'getPackage').resolves({
|
||||
id: 71,
|
||||
scheme: 'test',
|
||||
name: 'pkg2',
|
||||
version: '0.0.1',
|
||||
dump: { ...zeroDump, id: 2 },
|
||||
dump_id: 2,
|
||||
})
|
||||
|
||||
// Moniker search (local database)
|
||||
sinon.stub(database1, 'monikerResults').resolves({ locations: [], count: 0 })
|
||||
|
||||
// Moniker search (remote database)
|
||||
sinon.stub(database2, 'monikerResults').resolves({
|
||||
locations: [
|
||||
{ dumpId: 1, path: '1.ts', range: makeRange(1) },
|
||||
{ dumpId: 2, path: '2.ts', range: makeRange(2) },
|
||||
{ dumpId: 3, path: '3.ts', range: makeRange(3) },
|
||||
{ dumpId: 4, path: '4.ts', range: makeRange(4) },
|
||||
],
|
||||
count: 4,
|
||||
})
|
||||
|
||||
const locations = await new Backend(
|
||||
dumpManager,
|
||||
dependencyManager,
|
||||
'',
|
||||
createTestDatabase(
|
||||
new Map([
|
||||
[1, database1],
|
||||
[2, database2],
|
||||
])
|
||||
)
|
||||
).definitions(42, 'deadbeef', '/foo/bar/baz.ts', { line: 5, character: 10 }, 1)
|
||||
|
||||
expect(locations).toEqual([
|
||||
{ dump: { ...zeroDump, id: 1 }, path: '1.ts', range: makeRange(1) },
|
||||
{ dump: { ...zeroDump, id: 2 }, path: '2.ts', range: makeRange(2) },
|
||||
{ dump: { ...zeroDump, id: 3 }, path: '3.ts', range: makeRange(3) },
|
||||
{ dump: { ...zeroDump, id: 4 }, path: '4.ts', range: makeRange(4) },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('references', () => {
|
||||
const queryAllReferences = async (
|
||||
backend: Backend,
|
||||
repositoryId: number,
|
||||
commit: string,
|
||||
path: string,
|
||||
position: lsif.lsp.Position,
|
||||
dumpId: number,
|
||||
limit: number,
|
||||
remoteDumpLimit?: number
|
||||
): Promise<{ locations: ResolvedInternalLocation[]; pageSizes: number[]; numPages: number }> => {
|
||||
let locations: ResolvedInternalLocation[] = []
|
||||
const pageSizes: number[] = []
|
||||
let cursor: ReferencePaginationCursor | undefined
|
||||
|
||||
while (true) {
|
||||
const result = await backend.references(
|
||||
repositoryId,
|
||||
commit,
|
||||
path,
|
||||
position,
|
||||
{ limit, cursor },
|
||||
remoteDumpLimit,
|
||||
dumpId
|
||||
)
|
||||
if (!result) {
|
||||
break
|
||||
}
|
||||
|
||||
locations = locations.concat(result.locations)
|
||||
pageSizes.push(result.locations.length)
|
||||
|
||||
if (!result.newCursor) {
|
||||
break
|
||||
}
|
||||
cursor = result.newCursor
|
||||
}
|
||||
|
||||
return { locations, pageSizes, numPages: pageSizes.length }
|
||||
}
|
||||
|
||||
const assertPagedReferences = async (
|
||||
numSameRepoDumps: number,
|
||||
numRemoteRepoDumps: number,
|
||||
locationsPerDump: number,
|
||||
pageLimit: number,
|
||||
remoteDumpLimit: number
|
||||
): Promise<void> => {
|
||||
const numDatabases = 2 + numSameRepoDumps + numRemoteRepoDumps
|
||||
const numLocations = numDatabases * locationsPerDump
|
||||
|
||||
const databases = range(0, numDatabases).map(i => new Database(i + 1))
|
||||
const dumps = range(0, numLocations).map(i => ({ ...zeroDump, id: i + 1 }))
|
||||
const locations = range(0, numLocations).map(i => ({
|
||||
dumpId: i + 1,
|
||||
path: `${i + 1}.ts`,
|
||||
range: makeRange(i),
|
||||
}))
|
||||
|
||||
const getChunk = (index: number) =>
|
||||
locations.slice(index * locationsPerDump, (index + 1) * locationsPerDump)
|
||||
|
||||
const sameRepoDumps = range(0, numSameRepoDumps).map(i => ({
|
||||
...zeroPackage,
|
||||
dump: dumps[i + 2],
|
||||
dump_id: i + 3,
|
||||
}))
|
||||
|
||||
const remoteRepoDumps = range(0, numRemoteRepoDumps).map(i => ({
|
||||
...zeroPackage,
|
||||
dump: dumps[i + numSameRepoDumps + 2],
|
||||
dump_id: i + numSameRepoDumps + 3,
|
||||
}))
|
||||
|
||||
const expectedLocations = locations.map((location, i) => ({
|
||||
dump: dumps[i],
|
||||
path: location.path,
|
||||
range: location.range,
|
||||
}))
|
||||
|
||||
const dumpMap = new Map(dumps.map(dump => [dump.id, dump]))
|
||||
const databaseMap = new Map(databases.map((db, i) => [i + 1, db]))
|
||||
const definitionPackage = {
|
||||
id: 71,
|
||||
scheme: 'test',
|
||||
name: 'pkg2',
|
||||
version: '0.0.1',
|
||||
dump: dumps[1],
|
||||
dump_id: 2,
|
||||
}
|
||||
|
||||
// Loading source dump
|
||||
sinon.stub(dumpManager, 'getDumpById').callsFake(id => {
|
||||
if (id <= dumps.length) {
|
||||
return Promise.resolve(dumps[id - 1])
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected getDumpById invocation (id=${id}`)
|
||||
})
|
||||
|
||||
// Resolving target dumps
|
||||
sinon.stub(dumpManager, 'getDumpsByIds').resolves(dumpMap)
|
||||
|
||||
// Package resolution
|
||||
sinon.stub(dependencyManager, 'getPackage').resolves(definitionPackage)
|
||||
|
||||
// Same-repo package references
|
||||
const sameRepoStub = sinon
|
||||
.stub(dependencyManager, 'getSameRepoRemotePackageReferences')
|
||||
.callsFake(({ limit, offset }) =>
|
||||
Promise.resolve({
|
||||
packageReferences: sameRepoDumps.slice(offset, offset + limit),
|
||||
totalCount: numSameRepoDumps,
|
||||
newOffset: offset + limit,
|
||||
})
|
||||
)
|
||||
|
||||
// Remote repo package references
|
||||
const remoteRepoStub = sinon
|
||||
.stub(dependencyManager, 'getPackageReferences')
|
||||
.callsFake(({ limit, offset }) =>
|
||||
Promise.resolve({
|
||||
packageReferences: remoteRepoDumps.slice(offset, offset + limit),
|
||||
totalCount: numRemoteRepoDumps,
|
||||
newOffset: offset + limit,
|
||||
})
|
||||
)
|
||||
|
||||
// Moniker resolution
|
||||
sinon.stub(databases[0], 'monikersByPosition').resolves([monikersWithPackageInformation])
|
||||
|
||||
// Package resolution
|
||||
sinon.stub(databases[0], 'packageInformation').resolves({ name: 'pkg2', version: '0.0.1' })
|
||||
|
||||
// Same dump results
|
||||
const referenceStub = sinon.stub(databases[0], 'references').resolves(new OrderedLocationSet(getChunk(0)))
|
||||
|
||||
const monikerStubs: sinon.SinonStub<
|
||||
Parameters<Database['monikerResults']>,
|
||||
ReturnType<Database['monikerResults']>
|
||||
>[] = []
|
||||
|
||||
// Local moniker results
|
||||
sinon.stub(databases[0], 'monikerResults').resolves({ locations: [], count: 0 })
|
||||
|
||||
// Remote dump results
|
||||
for (let i = 1; i < numDatabases; i++) {
|
||||
monikerStubs.push(
|
||||
sinon.stub(databases[i], 'monikerResults').callsFake((model, moniker, { skip = 0, take = 10 }) =>
|
||||
Promise.resolve({
|
||||
locations: getChunk(i).slice(skip, skip + take),
|
||||
count: locationsPerDump,
|
||||
})
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
// Read all reference pages
|
||||
const { locations: resolvedLocations, pageSizes } = await queryAllReferences(
|
||||
new Backend(dumpManager, dependencyManager, '', createTestDatabase(databaseMap)),
|
||||
42,
|
||||
'deadbeef',
|
||||
'/foo/bar/baz.ts',
|
||||
{ line: 5, character: 10 },
|
||||
1,
|
||||
pageLimit,
|
||||
remoteDumpLimit
|
||||
)
|
||||
|
||||
// Ensure we get all locations
|
||||
expect(resolvedLocations).toEqual(expectedLocations)
|
||||
|
||||
// Ensure all pages (except for last) are full
|
||||
const copy = Array.from(pageSizes)
|
||||
expect(copy.pop()).toBeLessThanOrEqual(pageLimit)
|
||||
expect(copy.every(v => v === pageLimit)).toBeTruthy()
|
||||
|
||||
// Ensure pagination limits are respected
|
||||
const expectedCalls = (results: number, limit: number) => Math.max(1, Math.ceil(results / limit))
|
||||
expect(sameRepoStub.callCount).toEqual(expectedCalls(numSameRepoDumps, remoteDumpLimit))
|
||||
expect(remoteRepoStub.callCount).toEqual(expectedCalls(numRemoteRepoDumps, remoteDumpLimit))
|
||||
expect(referenceStub.callCount).toEqual(expectedCalls(locationsPerDump, pageLimit))
|
||||
for (const stub of monikerStubs) {
|
||||
expect(stub.callCount).toEqual(expectedCalls(locationsPerDump, pageLimit))
|
||||
}
|
||||
}
|
||||
|
||||
it('should return references in source and definition dumps', () => assertPagedReferences(0, 0, 1, 10, 5))
|
||||
it('should return references in remote dumps', () => assertPagedReferences(1, 0, 1, 10, 5))
|
||||
it('should return references in remote repositories', () => assertPagedReferences(0, 1, 1, 10, 5))
|
||||
it('should page large results sets', () => assertPagedReferences(0, 0, 25, 10, 5))
|
||||
it('should page a large number of remote dumps', () => assertPagedReferences(25, 25, 25, 10, 5))
|
||||
it('should respect small page size', () => assertPagedReferences(25, 25, 25, 1, 5))
|
||||
it('should respect large page size', () => assertPagedReferences(25, 25, 25, 1000, 5))
|
||||
it('should respect small remote dumps page size', () => assertPagedReferences(25, 25, 25, 10, 1))
|
||||
it('should respect large remote dumps page size', () => assertPagedReferences(25, 25, 25, 10, 25))
|
||||
})
|
||||
|
||||
describe('hover', () => {
|
||||
it('should return hover content from database', async () => {
|
||||
const database1 = new Database(1)
|
||||
|
||||
// Loading source dump
|
||||
sinon.stub(dumpManager, 'getDumpById').resolves({ ...zeroDump, id: 1 })
|
||||
|
||||
// In-database hover
|
||||
sinon.stub(database1, 'hover').resolves({
|
||||
text: 'hover text',
|
||||
range: makeRange(1),
|
||||
})
|
||||
|
||||
const hover = await new Backend(
|
||||
dumpManager,
|
||||
dependencyManager,
|
||||
'',
|
||||
createTestDatabase(new Map([[1, database1]]))
|
||||
).hover(42, 'deadbeef', '/foo/bar/baz.ts', { line: 5, character: 10 }, 1)
|
||||
|
||||
expect(hover).toEqual({
|
||||
text: 'hover text',
|
||||
range: makeRange(1),
|
||||
})
|
||||
})
|
||||
|
||||
it('should return hover content from unique definition', async () => {
|
||||
const database1 = new Database(1)
|
||||
const database2 = new Database(2)
|
||||
|
||||
// Loading source dump
|
||||
sinon.stub(dumpManager, 'getDumpById').resolves({ ...zeroDump, id: 1 })
|
||||
|
||||
// Resolving target dumps
|
||||
sinon.stub(dumpManager, 'getDumpsByIds').resolves(new Map([[2, { ...zeroDump, id: 2 }]]))
|
||||
|
||||
// In-database hover
|
||||
sinon.stub(database1, 'hover').resolves(null)
|
||||
|
||||
// In-database definitions
|
||||
sinon.stub(database1, 'definitions').resolves([
|
||||
{
|
||||
dumpId: 2,
|
||||
path: '2.ts',
|
||||
range: makeRange(2),
|
||||
},
|
||||
])
|
||||
|
||||
// Remote-database hover
|
||||
sinon.stub(database2, 'hover').resolves({
|
||||
text: 'hover text',
|
||||
range: makeRange(1),
|
||||
})
|
||||
|
||||
const hover = await new Backend(
|
||||
dumpManager,
|
||||
dependencyManager,
|
||||
'',
|
||||
createTestDatabase(
|
||||
new Map([
|
||||
[1, database1],
|
||||
[2, database2],
|
||||
])
|
||||
)
|
||||
).hover(42, 'deadbeef', '/foo/bar/baz.ts', { line: 5, character: 10 }, 1)
|
||||
|
||||
expect(hover).toEqual({ text: 'hover text', range: makeRange(1) })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('sortMonikers', () => {
|
||||
it('should order monikers by kind', () => {
|
||||
const monikers = [
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'npm',
|
||||
identifier: 'foo',
|
||||
},
|
||||
{
|
||||
kind: lsif.MonikerKind.export,
|
||||
scheme: 'npm',
|
||||
identifier: 'bar',
|
||||
},
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'npm',
|
||||
identifier: 'baz',
|
||||
},
|
||||
{
|
||||
kind: lsif.MonikerKind.import,
|
||||
scheme: 'npm',
|
||||
identifier: 'bonk',
|
||||
},
|
||||
]
|
||||
|
||||
expect(sortMonikers(monikers)).toEqual([monikers[3], monikers[0], monikers[2], monikers[1]])
|
||||
})
|
||||
|
||||
it('should remove subsumed monikers', () => {
|
||||
const monikers = [
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'go',
|
||||
identifier: 'foo',
|
||||
},
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'tsc',
|
||||
identifier: 'bar',
|
||||
},
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'gomod',
|
||||
identifier: 'baz',
|
||||
},
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'npm',
|
||||
identifier: 'baz',
|
||||
},
|
||||
]
|
||||
|
||||
expect(sortMonikers(monikers)).toEqual([monikers[2], monikers[3]])
|
||||
})
|
||||
|
||||
it('should not remove subsumable (but non-subsumed) monikers', () => {
|
||||
const monikers = [
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'go',
|
||||
identifier: 'foo',
|
||||
},
|
||||
{
|
||||
kind: lsif.MonikerKind.local,
|
||||
scheme: 'tsc',
|
||||
identifier: 'bar',
|
||||
},
|
||||
]
|
||||
|
||||
expect(sortMonikers(monikers)).toEqual(monikers)
|
||||
})
|
||||
})
|
||||
@ -1,92 +0,0 @@
|
||||
import * as sqliteModels from '../../shared/models/sqlite'
|
||||
import * as lsp from 'vscode-languageserver-protocol'
|
||||
|
||||
/** Context describing the current request for paginated results. */
|
||||
export interface ReferencePaginationContext {
|
||||
/** The maximum number of locations to return on this page. */
|
||||
limit: number
|
||||
|
||||
/** Context describing the next page of results. */
|
||||
cursor?: ReferencePaginationCursor
|
||||
}
|
||||
|
||||
/** Context describing the next page of results. */
|
||||
export type ReferencePaginationCursor =
|
||||
| SameDumpReferenceCursor
|
||||
| DefinitionMonikersReferenceCursor
|
||||
| RemoteDumpReferenceCursor
|
||||
|
||||
/** A label that indicates which pagination phase is being expanded. */
|
||||
export type ReferencePaginationPhase = 'same-dump' | 'definition-monikers' | 'same-repo' | 'remote-repo'
|
||||
|
||||
/** Fields common to all reference pagination cursors. */
|
||||
interface ReferencePaginationCursorCommon {
|
||||
/** The identifier of the dump that contains the target range. */
|
||||
dumpId: number
|
||||
|
||||
/** The phase of the pagination. */
|
||||
phase: ReferencePaginationPhase
|
||||
}
|
||||
|
||||
/** Bookkeeping data for the reference results that come from the initial dump. */
|
||||
export interface SameDumpReferenceCursor extends ReferencePaginationCursorCommon {
|
||||
phase: 'same-dump'
|
||||
|
||||
/** The (database-relative) document path containing the symbol ranges. */
|
||||
path: string
|
||||
|
||||
/** The current hover position. */
|
||||
position: lsp.Position
|
||||
|
||||
/** A normalized list of monikers attached to the symbol ranges. */
|
||||
monikers: sqliteModels.MonikerData[]
|
||||
|
||||
/** The number of reference results to skip. */
|
||||
skipResults: number
|
||||
}
|
||||
|
||||
/** Bookkeeping data for the reference results that come from dumps defining a moniker. */
|
||||
export interface DefinitionMonikersReferenceCursor extends ReferencePaginationCursorCommon {
|
||||
phase: 'definition-monikers'
|
||||
|
||||
/** The (database-relative) document path containing the symbol ranges. */
|
||||
path: string
|
||||
|
||||
/** A normalized list of monikers attached to the symbol ranges. */
|
||||
monikers: sqliteModels.MonikerData[]
|
||||
|
||||
/** The number of location results to skip for the current moniker. */
|
||||
skipResults: number
|
||||
}
|
||||
|
||||
/** Bookkeeping data for the reference results that come from additional (remote) dumps. */
|
||||
export interface RemoteDumpReferenceCursor extends ReferencePaginationCursorCommon {
|
||||
phase: 'same-repo' | 'remote-repo'
|
||||
|
||||
/** The identifier of the moniker that has remote results. */
|
||||
identifier: string
|
||||
|
||||
/** The scheme of the moniker that has remote results. */
|
||||
scheme: string
|
||||
|
||||
/** The name of the package that has remote results. */
|
||||
name: string
|
||||
|
||||
/** The version of the package that has remote results. */
|
||||
version: string | null
|
||||
|
||||
/** The current batch of dumps to open. */
|
||||
dumpIds: number[]
|
||||
|
||||
/** The total count of candidate dumps that can be opened. */
|
||||
totalDumpsWhenBatching: number
|
||||
|
||||
/** The number of dumps we have already processed or bloom filtered. */
|
||||
skipDumpsWhenBatching: number
|
||||
|
||||
/** The number of dumps we have already completed in the current batch. */
|
||||
skipDumpsInBatch: number
|
||||
|
||||
/** The number of location results to skip for the current dump. */
|
||||
skipResultsInDump: number
|
||||
}
|
||||
@ -1,175 +0,0 @@
|
||||
import * as sqliteModels from '../../shared/models/sqlite'
|
||||
import * as lsp from 'vscode-languageserver-protocol'
|
||||
import * as pgModels from '../../shared/models/pg'
|
||||
import { TracingContext } from '../../shared/tracing'
|
||||
import { parseJSON } from '../../shared/encoding/json'
|
||||
import * as settings from '../settings'
|
||||
import got from 'got'
|
||||
import { InternalLocation, OrderedLocationSet } from './location'
|
||||
|
||||
/** A wrapper around operations related to a single SQLite dump. */
|
||||
export class Database {
|
||||
constructor(private dumpId: pgModels.DumpId) {}
|
||||
|
||||
/**
|
||||
* Determine if data exists for a particular document in this database.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public exists(path: string, ctx: TracingContext = {}): Promise<boolean> {
|
||||
return this.request('exists', new URLSearchParams({ path }), ctx)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of locations that define the symbol at the given position.
|
||||
*
|
||||
* @param path The path of the document to which the position belongs.
|
||||
* @param position The current hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async definitions(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<InternalLocation[]> {
|
||||
const locations = await this.request<{ path: string; range: lsp.Range }[]>(
|
||||
'definitions',
|
||||
new URLSearchParams({ path, line: String(position.line), character: String(position.character) }),
|
||||
ctx
|
||||
)
|
||||
|
||||
return locations.map(location => ({ ...location, dumpId: this.dumpId }))
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of unique locations that reference the symbol at the given position.
|
||||
*
|
||||
* @param path The path of the document to which the position belongs.
|
||||
* @param position The current hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async references(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<OrderedLocationSet> {
|
||||
const locations = await this.request<{ path: string; range: lsp.Range }[]>(
|
||||
'references',
|
||||
new URLSearchParams({ path, line: String(position.line), character: String(position.character) }),
|
||||
ctx
|
||||
)
|
||||
|
||||
return new OrderedLocationSet(locations.map(location => ({ ...location, dumpId: this.dumpId })))
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the hover content for the symbol at the given position.
|
||||
*
|
||||
* @param path The path of the document to which the position belongs.
|
||||
* @param position The current hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public hover(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<{ text: string; range: lsp.Range } | null> {
|
||||
return this.request(
|
||||
'hover',
|
||||
new URLSearchParams({ path, line: String(position.line), character: String(position.character) }),
|
||||
ctx
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all of the monikers attached to all ranges that contain the given position. The
|
||||
* resulting list is grouped by range. If multiple ranges contain this position, then the
|
||||
* list monikers for the inner-most ranges will occur before the outer-most ranges.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param position The user's hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public monikersByPosition(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<sqliteModels.MonikerData[][]> {
|
||||
return this.request(
|
||||
'monikersByPosition',
|
||||
new URLSearchParams({ path, line: String(position.line), character: String(position.character) }),
|
||||
ctx
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the definitions or references table of `db` for items that match the given moniker.
|
||||
* Convert each result into an `InternalLocation`. The `pathTransformer` function is invoked
|
||||
* on each result item to modify the resulting locations.
|
||||
*
|
||||
* @param model The constructor for the model type.
|
||||
* @param moniker The target moniker.
|
||||
* @param pagination A limit and offset to use for the query.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async monikerResults(
|
||||
model: typeof sqliteModels.DefinitionModel | typeof sqliteModels.ReferenceModel,
|
||||
moniker: Pick<sqliteModels.MonikerData, 'scheme' | 'identifier'>,
|
||||
pagination: { skip?: number; take?: number },
|
||||
ctx: TracingContext = {}
|
||||
): Promise<{ locations: InternalLocation[]; count: number }> {
|
||||
let p: {} | { skip: string } | { take: string } | { skip: string; take: string } = {}
|
||||
if (pagination.skip !== undefined) {
|
||||
p = { ...p, skip: String(pagination.skip) }
|
||||
}
|
||||
if (pagination.take !== undefined) {
|
||||
p = { ...p, take: String(pagination.take) }
|
||||
}
|
||||
|
||||
const { locations, count } = await this.request<{
|
||||
locations: { path: string; range: lsp.Range }[]
|
||||
count: number
|
||||
}>(
|
||||
'monikerResults',
|
||||
new URLSearchParams({
|
||||
modelType: model === sqliteModels.DefinitionModel ? 'definition' : 'reference',
|
||||
scheme: moniker.scheme,
|
||||
identifier: moniker.identifier,
|
||||
...p,
|
||||
}),
|
||||
ctx
|
||||
)
|
||||
|
||||
return { locations: locations.map(location => ({ ...location, dumpId: this.dumpId })), count }
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the package information data with the given identifier.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param packageInformationId The identifier of the package information data.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public packageInformation(
|
||||
path: string,
|
||||
packageInformationId: sqliteModels.PackageInformationId,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<sqliteModels.PackageInformationData | undefined> {
|
||||
return this.request(
|
||||
'packageInformation',
|
||||
new URLSearchParams({ path, packageInformationId: String(packageInformationId) }),
|
||||
ctx
|
||||
)
|
||||
}
|
||||
|
||||
//
|
||||
//
|
||||
|
||||
private async request<T>(method: string, searchParams: URLSearchParams, ctx: TracingContext): Promise<T> {
|
||||
const url = new URL(`/dbs/${this.dumpId}/${method}`, settings.PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL)
|
||||
url.search = searchParams.toString()
|
||||
const resp = await got.get(url.href)
|
||||
return parseJSON(resp.body)
|
||||
}
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
import * as lsp from 'vscode-languageserver-protocol'
|
||||
import * as pgModels from '../../shared/models/pg'
|
||||
import { OrderedSet } from '../../shared/datastructures/orderedset'
|
||||
|
||||
export interface InternalLocation {
|
||||
/** The identifier of the dump that contains the location. */
|
||||
dumpId: pgModels.DumpId
|
||||
/** The path relative to the dump root. */
|
||||
path: string
|
||||
range: lsp.Range
|
||||
}
|
||||
|
||||
export interface ResolvedInternalLocation {
|
||||
/** The dump that contains the location. */
|
||||
dump: pgModels.LsifDump
|
||||
/** The path relative to the dump root. */
|
||||
path: string
|
||||
range: lsp.Range
|
||||
}
|
||||
|
||||
/** A duplicate-free list of locations ordered by time of insertion. */
|
||||
export class OrderedLocationSet extends OrderedSet<InternalLocation> {
|
||||
/**
|
||||
* Create a new ordered locations set.
|
||||
*
|
||||
* @param values A set of values used to seed the set.
|
||||
*/
|
||||
constructor(values?: InternalLocation[]) {
|
||||
super(
|
||||
(value: InternalLocation): string =>
|
||||
[
|
||||
value.dumpId,
|
||||
value.path,
|
||||
value.range.start.line,
|
||||
value.range.start.character,
|
||||
value.range.end.line,
|
||||
value.range.end.character,
|
||||
].join(':'),
|
||||
values
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,40 +0,0 @@
|
||||
import promClient from 'prom-client'
|
||||
|
||||
//
|
||||
// HTTP Metrics
|
||||
|
||||
export const httpUploadDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_http_upload_request_duration_seconds',
|
||||
help: 'Total time spent on upload requests.',
|
||||
labelNames: ['code'],
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
export const httpQueryDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_http_query_request_duration_seconds',
|
||||
help: 'Total time spent on query requests.',
|
||||
labelNames: ['code'],
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
//
|
||||
// Database Metrics
|
||||
|
||||
export const databaseQueryDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_database_query_duration_seconds',
|
||||
help: 'Total time spent on database queries.',
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
export const databaseQueryErrorsCounter = new promClient.Counter({
|
||||
name: 'lsif_database_query_errors_total',
|
||||
help: 'The number of errors that occurred during a database query.',
|
||||
})
|
||||
|
||||
//
|
||||
// Unconverted Upload Metrics
|
||||
|
||||
export const unconvertedUploadSizeGauge = new promClient.Gauge({
|
||||
name: 'lsif_unconverted_upload_size',
|
||||
help: 'The current number of uploads that have are pending conversion.',
|
||||
})
|
||||
@ -1,95 +0,0 @@
|
||||
import express from 'express'
|
||||
import { wrap } from 'async-middleware'
|
||||
import { UploadManager } from '../../shared/store/uploads'
|
||||
import { DumpManager } from '../../shared/store/dumps'
|
||||
import { EntityManager } from 'typeorm'
|
||||
import { SRC_FRONTEND_INTERNAL } from '../../shared/config/settings'
|
||||
import { TracingContext, addTags } from '../../shared/tracing'
|
||||
import { Span } from 'opentracing'
|
||||
import { Logger } from 'winston'
|
||||
import { updateCommitsAndDumpsVisibleFromTip } from '../../shared/visibility'
|
||||
import { json } from 'body-parser'
|
||||
|
||||
/**
|
||||
* Create a router containing the endpoints used by the bundle manager.
|
||||
*
|
||||
* @param dumpManager The dumps manager instance.
|
||||
* @param uploadManager The uploads manager instance.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export function createInternalRouter(
|
||||
dumpManager: DumpManager,
|
||||
uploadManager: UploadManager,
|
||||
logger: Logger
|
||||
): express.Router {
|
||||
const router = express.Router()
|
||||
|
||||
/**
|
||||
* Create a tracing context from the request logger and tracing span
|
||||
* tagged with the given values.
|
||||
*
|
||||
* @param req The express request.
|
||||
* @param tags The tags to apply to the logger and span.
|
||||
*/
|
||||
const createTracingContext = (
|
||||
req: express.Request & { span?: Span },
|
||||
tags: { [K: string]: unknown }
|
||||
): TracingContext => addTags({ logger, span: req.span }, tags)
|
||||
|
||||
interface StatesBody {
|
||||
ids: number[]
|
||||
}
|
||||
|
||||
type StatesResponse = Map<number, string>
|
||||
|
||||
router.post(
|
||||
'/uploads',
|
||||
json(),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<StatesResponse>): Promise<void> => {
|
||||
const { ids }: StatesBody = req.body
|
||||
res.json(await dumpManager.getUploadStates(ids))
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
type PruneResponse = { id: number } | null
|
||||
|
||||
router.post(
|
||||
'/prune',
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<PruneResponse>): Promise<void> => {
|
||||
const ctx = createTracingContext(req, {})
|
||||
|
||||
const dump = await dumpManager.getOldestPrunableDump()
|
||||
if (!dump) {
|
||||
res.json(null)
|
||||
return
|
||||
}
|
||||
|
||||
logger.info('Pruning dump', {
|
||||
repository: dump.repositoryId,
|
||||
commit: dump.commit,
|
||||
root: dump.root,
|
||||
})
|
||||
|
||||
// This delete cascades to the packages and references tables as well
|
||||
await uploadManager.deleteUpload(
|
||||
dump.id,
|
||||
(entityManager: EntityManager, repositoryId: number): Promise<void> =>
|
||||
updateCommitsAndDumpsVisibleFromTip({
|
||||
entityManager,
|
||||
dumpManager,
|
||||
frontendUrl: SRC_FRONTEND_INTERNAL,
|
||||
repositoryId,
|
||||
ctx,
|
||||
})
|
||||
)
|
||||
|
||||
res.json({ id: dump.id })
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return router
|
||||
}
|
||||
@ -1,321 +0,0 @@
|
||||
import * as constants from '../../shared/constants'
|
||||
import * as fs from 'mz/fs'
|
||||
import * as lsp from 'vscode-languageserver-protocol'
|
||||
import * as nodepath from 'path'
|
||||
import * as settings from '../settings'
|
||||
import * as validation from '../../shared/api/middleware/validation'
|
||||
import express from 'express'
|
||||
import * as uuid from 'uuid'
|
||||
import { addTags, logAndTraceCall, TracingContext } from '../../shared/tracing'
|
||||
import { Backend } from '../backend/backend'
|
||||
import { encodeCursor } from '../../shared/api/pagination/cursor'
|
||||
import { Logger } from 'winston'
|
||||
import { nextLink } from '../../shared/api/pagination/link'
|
||||
import { pipeline as _pipeline } from 'stream'
|
||||
import { promisify } from 'util'
|
||||
import { Span, Tracer } from 'opentracing'
|
||||
import { wrap } from 'async-middleware'
|
||||
import { extractLimitOffset } from '../../shared/api/pagination/limit-offset'
|
||||
import { UploadManager } from '../../shared/store/uploads'
|
||||
import { readGzippedJsonElementsFromFile } from '../../shared/input'
|
||||
import * as lsif from 'lsif-protocol'
|
||||
import { ReferencePaginationCursor } from '../backend/cursor'
|
||||
import { LsifUpload } from '../../shared/models/pg'
|
||||
import got from 'got'
|
||||
import { Connection } from 'typeorm'
|
||||
|
||||
const pipeline = promisify(_pipeline)
|
||||
|
||||
/**
|
||||
* Create a router containing the LSIF upload and query endpoints.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param backend The backend instance.
|
||||
* @param uploadManager The uploads manager instance.
|
||||
* @param logger The logger instance.
|
||||
* @param tracer The tracer instance.
|
||||
*/
|
||||
export function createLsifRouter(
|
||||
connection: Connection,
|
||||
backend: Backend,
|
||||
uploadManager: UploadManager,
|
||||
logger: Logger,
|
||||
tracer: Tracer | undefined
|
||||
): express.Router {
|
||||
const router = express.Router()
|
||||
|
||||
// Used to validate commit hashes are 40 hex digits
|
||||
const commitPattern = /^[a-f0-9]{40}$/
|
||||
|
||||
/**
|
||||
* Ensure roots end with a slash, unless it refers to the top-level directory.
|
||||
*
|
||||
* @param root The input root.
|
||||
*/
|
||||
const sanitizeRoot = (root: string | undefined): string => {
|
||||
if (root === undefined || root === '/' || root === '') {
|
||||
return ''
|
||||
}
|
||||
|
||||
return root.endsWith('/') ? root : root + '/'
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a tracing context from the request logger and tracing span
|
||||
* tagged with the given values.
|
||||
*
|
||||
* @param req The express request.
|
||||
* @param tags The tags to apply to the logger and span.
|
||||
*/
|
||||
const createTracingContext = (
|
||||
req: express.Request & { span?: Span },
|
||||
tags: { [K: string]: unknown }
|
||||
): TracingContext => addTags({ logger, span: req.span }, tags)
|
||||
|
||||
interface UploadQueryArgs {
|
||||
repositoryId: number
|
||||
commit: string
|
||||
root?: string
|
||||
indexerName?: string
|
||||
}
|
||||
|
||||
interface UploadResponse {
|
||||
id: string
|
||||
}
|
||||
|
||||
router.post(
|
||||
'/upload',
|
||||
validation.validationMiddleware([
|
||||
validation.validateInt('repositoryId'),
|
||||
validation.validateNonEmptyString('commit').matches(commitPattern),
|
||||
validation.validateOptionalString('root'),
|
||||
validation.validateOptionalString('indexerName'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<UploadResponse>): Promise<void> => {
|
||||
const { repositoryId, commit, root: rootRaw, indexerName }: UploadQueryArgs = req.query
|
||||
|
||||
const root = sanitizeRoot(rootRaw)
|
||||
const ctx = createTracingContext(req, { repositoryId, commit, root })
|
||||
const filename = nodepath.join(settings.STORAGE_ROOT, uuid.v4())
|
||||
const output = fs.createWriteStream(filename)
|
||||
await logAndTraceCall(ctx, 'Receiving dump', () => pipeline(req, output))
|
||||
|
||||
try {
|
||||
const indexer = indexerName || (await findIndexer(filename))
|
||||
if (!indexer) {
|
||||
throw new Error('Could not find tool type in metadata vertex at the start of the dump.')
|
||||
}
|
||||
|
||||
const id = await connection.transaction(async entityManager => {
|
||||
// Add upload record
|
||||
const uploadId = await uploadManager.enqueue(
|
||||
{ repositoryId, commit, root, indexer },
|
||||
entityManager
|
||||
)
|
||||
|
||||
// Upload the payload file where it can be found by the worker
|
||||
await logAndTraceCall(ctx, 'Uploading payload to bundle manager', () =>
|
||||
pipeline(
|
||||
fs.createReadStream(filename),
|
||||
got.stream.post(
|
||||
new URL(`/uploads/${uploadId}`, settings.PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL).href
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return uploadId
|
||||
})
|
||||
|
||||
// Upload conversion will complete asynchronously, send an accepted response
|
||||
// with the upload id so that the client can continue to track the progress
|
||||
// asynchronously.
|
||||
res.status(202).send({ id: `${id}` })
|
||||
} finally {
|
||||
// Remove local file
|
||||
await fs.unlink(filename)
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface ExistsQueryArgs {
|
||||
repositoryId: number
|
||||
commit: string
|
||||
path: string
|
||||
}
|
||||
|
||||
interface ExistsResponse {
|
||||
uploads: LsifUpload[]
|
||||
}
|
||||
|
||||
router.get(
|
||||
'/exists',
|
||||
validation.validationMiddleware([
|
||||
validation.validateInt('repositoryId'),
|
||||
validation.validateNonEmptyString('commit').matches(commitPattern),
|
||||
validation.validateNonEmptyString('path'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<ExistsResponse>): Promise<void> => {
|
||||
const { repositoryId, commit, path }: ExistsQueryArgs = req.query
|
||||
const ctx = createTracingContext(req, { repositoryId, commit })
|
||||
const uploads = await backend.exists(repositoryId, commit, path, ctx)
|
||||
res.json({ uploads })
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface FilePositionArgs {
|
||||
repositoryId: number
|
||||
commit: string
|
||||
path: string
|
||||
line: number
|
||||
character: number
|
||||
uploadId: number
|
||||
}
|
||||
|
||||
interface LocationsResponse {
|
||||
locations: { repositoryId: number; commit: string; path: string; range: lsp.Range }[]
|
||||
}
|
||||
|
||||
router.get(
|
||||
'/definitions',
|
||||
validation.validationMiddleware([
|
||||
validation.validateInt('repositoryId'),
|
||||
validation.validateNonEmptyString('commit'),
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateInt('line'),
|
||||
validation.validateInt('character'),
|
||||
validation.validateInt('uploadId'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<LocationsResponse>): Promise<void> => {
|
||||
const { repositoryId, commit, path, line, character, uploadId }: FilePositionArgs = req.query
|
||||
const ctx = createTracingContext(req, { repositoryId, commit, path })
|
||||
|
||||
const locations = await backend.definitions(
|
||||
repositoryId,
|
||||
commit,
|
||||
path,
|
||||
{ line, character },
|
||||
uploadId,
|
||||
ctx
|
||||
)
|
||||
if (locations === undefined) {
|
||||
throw Object.assign(new Error('LSIF upload not found'), { status: 404 })
|
||||
}
|
||||
|
||||
res.send({
|
||||
locations: locations.map(l => ({
|
||||
repositoryId: l.dump.repositoryId,
|
||||
commit: l.dump.commit,
|
||||
path: l.path,
|
||||
range: l.range,
|
||||
})),
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface ReferencesQueryArgs extends FilePositionArgs {
|
||||
commit: string
|
||||
cursor: ReferencePaginationCursor | undefined
|
||||
}
|
||||
|
||||
router.get(
|
||||
'/references',
|
||||
validation.validationMiddleware([
|
||||
validation.validateInt('repositoryId'),
|
||||
validation.validateNonEmptyString('commit'),
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateInt('line'),
|
||||
validation.validateInt('character'),
|
||||
validation.validateInt('uploadId'),
|
||||
validation.validateLimit,
|
||||
validation.validateCursor<ReferencePaginationCursor>(),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<LocationsResponse>): Promise<void> => {
|
||||
const { repositoryId, commit, path, line, character, uploadId, cursor }: ReferencesQueryArgs = req.query
|
||||
const { limit } = extractLimitOffset(req.query, settings.DEFAULT_REFERENCES_PAGE_SIZE)
|
||||
const ctx = createTracingContext(req, { repositoryId, commit, path })
|
||||
|
||||
const result = await backend.references(
|
||||
repositoryId,
|
||||
commit,
|
||||
path,
|
||||
{ line, character },
|
||||
{ limit, cursor },
|
||||
constants.DEFAULT_REFERENCES_REMOTE_DUMP_LIMIT,
|
||||
uploadId,
|
||||
ctx
|
||||
)
|
||||
if (result === undefined) {
|
||||
throw Object.assign(new Error('LSIF upload not found'), { status: 404 })
|
||||
}
|
||||
|
||||
const { locations, newCursor } = result
|
||||
const encodedCursor = encodeCursor<ReferencePaginationCursor>(newCursor)
|
||||
if (encodedCursor) {
|
||||
res.set('Link', nextLink(req, { limit, cursor: encodedCursor }))
|
||||
}
|
||||
|
||||
res.json({
|
||||
locations: locations.map(l => ({
|
||||
repositoryId: l.dump.repositoryId,
|
||||
commit: l.dump.commit,
|
||||
path: l.path,
|
||||
range: l.range,
|
||||
})),
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
type HoverResponse = { text: string; range: lsp.Range } | null
|
||||
|
||||
router.get(
|
||||
'/hover',
|
||||
validation.validationMiddleware([
|
||||
validation.validateInt('repositoryId'),
|
||||
validation.validateNonEmptyString('commit'),
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateInt('line'),
|
||||
validation.validateInt('character'),
|
||||
validation.validateInt('uploadId'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<HoverResponse>): Promise<void> => {
|
||||
const { repositoryId, commit, path, line, character, uploadId }: FilePositionArgs = req.query
|
||||
const ctx = createTracingContext(req, { repositoryId, commit, path })
|
||||
|
||||
const result = await backend.hover(repositoryId, commit, path, { line, character }, uploadId, ctx)
|
||||
if (result === undefined) {
|
||||
throw Object.assign(new Error('LSIF upload not found'), { status: 404 })
|
||||
}
|
||||
|
||||
res.json(result)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
/**
|
||||
* Read and decode the first entry of the dump. If the entry exists, encodes a metadata vertex,
|
||||
* and contains a tool info name field, return the contents of that field; otherwise undefined.
|
||||
*
|
||||
* @param filename The filename to read.
|
||||
*/
|
||||
async function findIndexer(filename: string): Promise<string | undefined> {
|
||||
for await (const element of readGzippedJsonElementsFromFile(filename) as AsyncIterable<lsif.Vertex | lsif.Edge>) {
|
||||
if (element.type === lsif.ElementTypes.vertex && element.label === lsif.VertexLabels.metaData) {
|
||||
return element.toolInfo?.name
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
@ -1,133 +0,0 @@
|
||||
import * as pgModels from '../../shared/models/pg'
|
||||
import * as settings from '../settings'
|
||||
import * as validation from '../../shared/api/middleware/validation'
|
||||
import express from 'express'
|
||||
import { nextLink } from '../../shared/api/pagination/link'
|
||||
import { wrap } from 'async-middleware'
|
||||
import { extractLimitOffset } from '../../shared/api/pagination/limit-offset'
|
||||
import { UploadManager, LsifUploadWithPlaceInQueue } from '../../shared/store/uploads'
|
||||
import { DumpManager } from '../../shared/store/dumps'
|
||||
import { EntityManager } from 'typeorm'
|
||||
import { SRC_FRONTEND_INTERNAL } from '../../shared/config/settings'
|
||||
import { TracingContext, addTags } from '../../shared/tracing'
|
||||
import { Span } from 'opentracing'
|
||||
import { Logger } from 'winston'
|
||||
import { updateCommitsAndDumpsVisibleFromTip } from '../../shared/visibility'
|
||||
|
||||
/**
|
||||
* Create a router containing the upload endpoints.
|
||||
*
|
||||
* @param dumpManager The dumps manager instance.
|
||||
* @param uploadManager The uploads manager instance.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export function createUploadRouter(
|
||||
dumpManager: DumpManager,
|
||||
uploadManager: UploadManager,
|
||||
logger: Logger
|
||||
): express.Router {
|
||||
const router = express.Router()
|
||||
|
||||
/**
|
||||
* Create a tracing context from the request logger and tracing span
|
||||
* tagged with the given values.
|
||||
*
|
||||
* @param req The express request.
|
||||
* @param tags The tags to apply to the logger and span.
|
||||
*/
|
||||
const createTracingContext = (
|
||||
req: express.Request & { span?: Span },
|
||||
tags: { [K: string]: unknown }
|
||||
): TracingContext => addTags({ logger, span: req.span }, tags)
|
||||
|
||||
interface UploadsQueryArgs {
|
||||
query: string
|
||||
state?: pgModels.LsifUploadState
|
||||
visibleAtTip?: boolean
|
||||
}
|
||||
|
||||
type UploadResponse = LsifUploadWithPlaceInQueue
|
||||
|
||||
router.get(
|
||||
'/uploads/:id([0-9]+)',
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<UploadResponse>): Promise<void> => {
|
||||
const upload = await uploadManager.getUpload(parseInt(req.params.id, 10))
|
||||
if (upload) {
|
||||
res.send(upload)
|
||||
return
|
||||
}
|
||||
|
||||
throw Object.assign(new Error('Upload not found'), {
|
||||
status: 404,
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
router.delete(
|
||||
'/uploads/:id([0-9]+)',
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<never>): Promise<void> => {
|
||||
const id = parseInt(req.params.id, 10)
|
||||
const ctx = createTracingContext(req, { id })
|
||||
|
||||
const updateVisibility = (entityManager: EntityManager, repositoryId: number): Promise<void> =>
|
||||
updateCommitsAndDumpsVisibleFromTip({
|
||||
entityManager,
|
||||
dumpManager,
|
||||
frontendUrl: SRC_FRONTEND_INTERNAL,
|
||||
repositoryId,
|
||||
ctx,
|
||||
})
|
||||
|
||||
if (await uploadManager.deleteUpload(id, updateVisibility)) {
|
||||
res.status(204).send()
|
||||
return
|
||||
}
|
||||
|
||||
throw Object.assign(new Error('Upload not found'), {
|
||||
status: 404,
|
||||
})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface UploadsResponse {
|
||||
uploads: LsifUploadWithPlaceInQueue[]
|
||||
totalCount: number
|
||||
}
|
||||
|
||||
router.get(
|
||||
'/uploads/repository/:id([0-9]+)',
|
||||
validation.validationMiddleware([
|
||||
validation.validateQuery,
|
||||
validation.validateLsifUploadState,
|
||||
validation.validateOptionalBoolean('visibleAtTip'),
|
||||
validation.validateLimit,
|
||||
validation.validateOffset,
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<UploadsResponse>): Promise<void> => {
|
||||
const { query, state, visibleAtTip }: UploadsQueryArgs = req.query
|
||||
const { limit, offset } = extractLimitOffset(req.query, settings.DEFAULT_UPLOAD_PAGE_SIZE)
|
||||
const { uploads, totalCount } = await uploadManager.getUploads(
|
||||
parseInt(req.params.id, 10),
|
||||
state,
|
||||
query,
|
||||
!!visibleAtTip,
|
||||
limit,
|
||||
offset
|
||||
)
|
||||
|
||||
if (offset + uploads.length < totalCount) {
|
||||
res.set('Link', nextLink(req, { limit, offset: offset + uploads.length }))
|
||||
}
|
||||
|
||||
res.json({ uploads, totalCount })
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return router
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
import { readEnvInt } from '../shared/settings'
|
||||
|
||||
/** Which port to run the LSIF API server on. Defaults to 3186. */
|
||||
export const HTTP_PORT = readEnvInt('HTTP_PORT', 3186)
|
||||
|
||||
/** HTTP address for internal LSIF bundle manager server. */
|
||||
export const PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL =
|
||||
process.env.PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL || 'http://localhost:3187'
|
||||
|
||||
/** Where on the file system to temporarily store LSIF uploads. This need not be a persistent volume. */
|
||||
export const STORAGE_ROOT = process.env.LSIF_STORAGE_ROOT || 'lsif-storage'
|
||||
|
||||
/** The default number of results to return from the upload endpoints. */
|
||||
export const DEFAULT_UPLOAD_PAGE_SIZE = readEnvInt('DEFAULT_UPLOAD_PAGE_SIZE', 50)
|
||||
|
||||
/** The default number of results to return from the dumps endpoint. */
|
||||
export const DEFAULT_DUMP_PAGE_SIZE = readEnvInt('DEFAULT_DUMP_PAGE_SIZE', 50)
|
||||
|
||||
/** The default number of location results to return when performing a find-references operation. */
|
||||
export const DEFAULT_REFERENCES_PAGE_SIZE = readEnvInt('DEFAULT_REFERENCES_PAGE_SIZE', 100)
|
||||
|
||||
/** The interval (in seconds) to invoke the updateQueueSizeGaugeInterval task. */
|
||||
export const UPDATE_QUEUE_SIZE_GAUGE_INTERVAL = readEnvInt('UPDATE_QUEUE_SIZE_GAUGE_INTERVAL', 5)
|
||||
|
||||
/** The interval (in seconds) to run the resetStalledUploads task. */
|
||||
export const RESET_STALLED_UPLOADS_INTERVAL = readEnvInt('RESET_STALLED_UPLOADS_INTERVAL', 60)
|
||||
|
||||
/** The maximum age (in seconds) that an upload can be unlocked and in the `processing` state. */
|
||||
export const STALLED_UPLOAD_MAX_AGE = readEnvInt('STALLED_UPLOAD_MAX_AGE', 5)
|
||||
|
||||
/** The interval (in seconds) to invoke the cleanOldUploads task. */
|
||||
export const CLEAN_OLD_UPLOADS_INTERVAL = readEnvInt('CLEAN_OLD_UPLOADS_INTERVAL', 60 * 60 * 8) // 8 hours
|
||||
|
||||
/** The maximum age (in seconds) that an upload (completed or queued) will remain in Postgres. */
|
||||
export const UPLOAD_MAX_AGE = readEnvInt('UPLOAD_UPLOAD_AGE', 60 * 60 * 24 * 7) // 1 week
|
||||
@ -1,81 +0,0 @@
|
||||
import * as settings from './settings'
|
||||
import { Connection } from 'typeorm'
|
||||
import { Logger } from 'winston'
|
||||
import { UploadManager } from '../shared/store/uploads'
|
||||
import { ExclusivePeriodicTaskRunner } from '../shared/tasks'
|
||||
import * as metrics from './metrics'
|
||||
import { createSilentLogger } from '../shared/logging'
|
||||
import { TracingContext } from '../shared/tracing'
|
||||
|
||||
/**
|
||||
* Begin running cleanup tasks on a schedule in the background.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param uploadManager The uploads manager instance.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export function startTasks(connection: Connection, uploadManager: UploadManager, logger: Logger): void {
|
||||
const runner = new ExclusivePeriodicTaskRunner(connection, logger)
|
||||
|
||||
runner.register({
|
||||
name: 'Updating metrics',
|
||||
intervalMs: settings.UPDATE_QUEUE_SIZE_GAUGE_INTERVAL,
|
||||
task: () => updateQueueSizeGauge(uploadManager),
|
||||
silent: true,
|
||||
})
|
||||
|
||||
runner.register({
|
||||
name: 'Resetting stalled uploads',
|
||||
intervalMs: settings.RESET_STALLED_UPLOADS_INTERVAL,
|
||||
task: ({ ctx }) => resetStalledUploads(uploadManager, ctx),
|
||||
})
|
||||
|
||||
runner.register({
|
||||
name: 'Cleaning old uploads',
|
||||
intervalMs: settings.CLEAN_OLD_UPLOADS_INTERVAL,
|
||||
task: ({ ctx }) => cleanOldUploads(uploadManager, ctx),
|
||||
})
|
||||
|
||||
runner.run()
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the value of the unconverted uploads gauge.
|
||||
*
|
||||
* @param uploadManager The uploads manager instance.
|
||||
*/
|
||||
async function updateQueueSizeGauge(uploadManager: UploadManager): Promise<void> {
|
||||
metrics.unconvertedUploadSizeGauge.set(await uploadManager.getCount('queued'))
|
||||
}
|
||||
|
||||
/**
|
||||
* Move all unlocked uploads that have been in `processing` state for longer than
|
||||
* `STALLED_UPLOAD_MAX_AGE` back to the `queued` state.
|
||||
*
|
||||
* @param uploadManager The uploads manager instance.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
async function resetStalledUploads(
|
||||
uploadManager: UploadManager,
|
||||
{ logger = createSilentLogger() }: TracingContext
|
||||
): Promise<void> {
|
||||
for (const id of await uploadManager.resetStalled(settings.STALLED_UPLOAD_MAX_AGE)) {
|
||||
logger.debug('Reset stalled upload conversion', { id })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all upload data older than `UPLOAD_MAX_AGE`.
|
||||
*
|
||||
* @param uploadManager The uploads manager instance.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
async function cleanOldUploads(
|
||||
uploadManager: UploadManager,
|
||||
{ logger = createSilentLogger() }: TracingContext
|
||||
): Promise<void> {
|
||||
const count = await uploadManager.clean(settings.UPLOAD_MAX_AGE)
|
||||
if (count > 0) {
|
||||
logger.debug('Cleaned old uploads', { count })
|
||||
}
|
||||
}
|
||||
@ -1,238 +0,0 @@
|
||||
import * as sinon from 'sinon'
|
||||
import promClient from 'prom-client'
|
||||
import { createBarrierPromise, GenericCache } from './cache'
|
||||
|
||||
describe('GenericCache', () => {
|
||||
const testCacheSizeGauge = new promClient.Gauge({
|
||||
name: 'test_cache_size',
|
||||
help: 'test_cache_size',
|
||||
})
|
||||
|
||||
const testCacheEventsCounter = new promClient.Counter({
|
||||
name: 'test_cache_events_total',
|
||||
help: 'test_cache_events_total',
|
||||
labelNames: ['type'],
|
||||
})
|
||||
|
||||
const testMetrics = {
|
||||
sizeGauge: testCacheSizeGauge,
|
||||
eventsCounter: testCacheEventsCounter,
|
||||
}
|
||||
|
||||
it('should evict items based by reverse recency', async () => {
|
||||
const values = [
|
||||
'foo', // foo*
|
||||
'bar', // bar* foo
|
||||
'baz', // baz* bar foo
|
||||
'bonk', // bonk* baz bar foo
|
||||
'quux', // quux* bonk baz bar foo
|
||||
'bar', // bar quux bonk baz foo
|
||||
'foo', // foo bar quux bonk baz
|
||||
'honk', // honk* foo bar quux bonk
|
||||
'foo', // foo honk bar quux bonk
|
||||
'baz', // baz* foo honk bar quux
|
||||
]
|
||||
|
||||
// These are the cache values that need to be created, in-order
|
||||
const expectedInstantiations = ['foo', 'bar', 'baz', 'bonk', 'quux', 'honk', 'baz']
|
||||
|
||||
const factory = sinon.stub<string[], Promise<string>>()
|
||||
for (const [i, value] of expectedInstantiations.entries()) {
|
||||
// Log the value arg and resolve the cache data immediately
|
||||
factory.onCall(i).resolves(value)
|
||||
}
|
||||
|
||||
const cache = new GenericCache<string, string>(
|
||||
5,
|
||||
() => 1,
|
||||
() => {
|
||||
/* noop */
|
||||
},
|
||||
testMetrics
|
||||
)
|
||||
for (const value of values) {
|
||||
const returnValue = await cache.withValue(
|
||||
value,
|
||||
() => factory(value),
|
||||
v => Promise.resolve(v)
|
||||
)
|
||||
expect(returnValue).toBe(value)
|
||||
}
|
||||
|
||||
// Expect the args of the factory to equal the resolved values
|
||||
expect(factory.args).toEqual(expectedInstantiations.map(v => [v]))
|
||||
})
|
||||
|
||||
it('should asynchronously resolve cache values', async () => {
|
||||
const factory = sinon.stub<string[], Promise<string>>()
|
||||
const { wait, done } = createBarrierPromise()
|
||||
factory.returns(wait.then(() => 'bar'))
|
||||
|
||||
const cache = new GenericCache<string, string>(
|
||||
5,
|
||||
() => 1,
|
||||
() => {
|
||||
/* noop */
|
||||
},
|
||||
testMetrics
|
||||
)
|
||||
const p1 = cache.withValue('foo', factory, v => Promise.resolve(v))
|
||||
const p2 = cache.withValue('foo', factory, v => Promise.resolve(v))
|
||||
const p3 = cache.withValue('foo', factory, v => Promise.resolve(v))
|
||||
done()
|
||||
|
||||
expect(await Promise.all([p1, p2, p3])).toEqual(['bar', 'bar', 'bar'])
|
||||
expect(factory.callCount).toEqual(1)
|
||||
})
|
||||
|
||||
it('should call dispose function on eviction', async () => {
|
||||
const values = [
|
||||
'foo', // foo
|
||||
'bar', // bar foo
|
||||
'baz', // baz bar (drops foo)
|
||||
'foo', // foo baz (drops bar)
|
||||
]
|
||||
|
||||
const { wait, done } = createBarrierPromise()
|
||||
const disposer = sinon.spy(done)
|
||||
const cache = new GenericCache<string, string>(2, () => 1, disposer, testMetrics)
|
||||
|
||||
for (const value of values) {
|
||||
await cache.withValue(
|
||||
value,
|
||||
() => Promise.resolve(value),
|
||||
v => Promise.resolve(v)
|
||||
)
|
||||
}
|
||||
|
||||
await wait
|
||||
expect(disposer.args).toEqual([['foo'], ['bar']])
|
||||
})
|
||||
|
||||
it('should calculate size by resolved value', async () => {
|
||||
const values = [
|
||||
2, // 2, size = 2
|
||||
3, // 3 2, size = 5
|
||||
1, // 1 3, size = 4
|
||||
2, // 1 2, size = 3
|
||||
]
|
||||
|
||||
const expectedInstantiations = [2, 3, 1, 2]
|
||||
|
||||
const factory = sinon.stub<number[], Promise<number>>()
|
||||
for (const [i, value] of expectedInstantiations.entries()) {
|
||||
factory.onCall(i).resolves(value)
|
||||
}
|
||||
|
||||
const cache = new GenericCache<number, number>(
|
||||
5,
|
||||
v => v,
|
||||
() => {
|
||||
/* noop */
|
||||
},
|
||||
testMetrics
|
||||
)
|
||||
for (const value of values) {
|
||||
await cache.withValue(
|
||||
value,
|
||||
() => factory(value),
|
||||
v => Promise.resolve(v)
|
||||
)
|
||||
}
|
||||
|
||||
expect(factory.args).toEqual(expectedInstantiations.map(v => [v]))
|
||||
})
|
||||
|
||||
it('should not evict referenced cache entries', async () => {
|
||||
const { wait, done } = createBarrierPromise()
|
||||
const disposer = sinon.spy(done)
|
||||
const cache = new GenericCache<string, string>(5, () => 1, disposer, testMetrics)
|
||||
|
||||
const fooResolver = () => Promise.resolve('foo')
|
||||
const barResolver = () => Promise.resolve('bar')
|
||||
const bazResolver = () => Promise.resolve('baz')
|
||||
const bonkResolver = () => Promise.resolve('bonk')
|
||||
const quuxResolver = () => Promise.resolve('quux')
|
||||
const honkResolver = () => Promise.resolve('honk')
|
||||
const ronkResolver = () => Promise.resolve('ronk')
|
||||
|
||||
await cache.withValue('foo', fooResolver, async () => {
|
||||
await cache.withValue('bar', barResolver, async () => {
|
||||
await cache.withValue('baz', bazResolver, async () => {
|
||||
await cache.withValue('bonk', bonkResolver, async () => {
|
||||
await cache.withValue('quux', quuxResolver, async () => {
|
||||
// Sixth entry, but nothing to evict (all held)
|
||||
await cache.withValue('honk', honkResolver, () => Promise.resolve())
|
||||
|
||||
// Seventh entry, honk can now be removed as it's the least
|
||||
// recently used value that's not currently under a read lock.
|
||||
await cache.withValue('ronk', ronkResolver, () => Promise.resolve())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// Release and remove the least recently used
|
||||
|
||||
await cache.withValue(
|
||||
'honk',
|
||||
() => Promise.resolve('honk'),
|
||||
async () => {
|
||||
await wait
|
||||
expect(disposer.args).toEqual([['honk'], ['foo'], ['bar']])
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('should dispose busted keys', async () => {
|
||||
const { wait, done } = createBarrierPromise()
|
||||
const disposer = sinon.spy(done)
|
||||
const cache = new GenericCache<string, string>(5, () => 1, disposer, testMetrics)
|
||||
|
||||
const factory = sinon.stub<string[], Promise<string>>()
|
||||
factory.resolves('foo')
|
||||
|
||||
// Construct then bust a same key
|
||||
await cache.withValue('foo', factory, () => Promise.resolve())
|
||||
await cache.bustKey('foo')
|
||||
await wait
|
||||
|
||||
// Ensure value was disposed
|
||||
expect(disposer.args).toEqual([['foo']])
|
||||
|
||||
// Ensure entry was removed
|
||||
expect(cache.withValue('foo', factory, () => Promise.resolve()))
|
||||
expect(factory.args).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('should wait to dispose busted keys that are in use', async () => {
|
||||
const { wait: wait1, done: done1 } = createBarrierPromise()
|
||||
const { wait: wait2, done: done2 } = createBarrierPromise()
|
||||
|
||||
const resolver = () => Promise.resolve('foo')
|
||||
const disposer = sinon.spy(done1)
|
||||
const cache = new GenericCache<string, string>(5, () => 1, disposer, testMetrics)
|
||||
|
||||
// Create a cache entry for 'foo' that blocks on done2
|
||||
const p1 = cache.withValue('foo', resolver, () => wait2)
|
||||
|
||||
// Attempt to bust the cache key that's used in the blocking promise above
|
||||
const p2 = cache.bustKey('foo')
|
||||
|
||||
// Ensure that p1 and p2 are blocked on each other
|
||||
const timedResolver = new Promise(resolve => setTimeout(() => resolve('$'), 10))
|
||||
const winner = await Promise.race([p1, p2, timedResolver])
|
||||
expect(winner).toEqual('$')
|
||||
|
||||
// Ensure dispose hasn't been called
|
||||
expect(disposer.args).toHaveLength(0)
|
||||
|
||||
// Unblock p1
|
||||
done2()
|
||||
|
||||
// Show that all promises are unblocked and dispose was called
|
||||
await Promise.all([p1, p2, wait1])
|
||||
expect(disposer.args).toEqual([['foo']])
|
||||
})
|
||||
})
|
||||
@ -1,433 +0,0 @@
|
||||
import * as sqliteModels from '../../shared/models/sqlite'
|
||||
import * as metrics from '../metrics'
|
||||
import promClient from 'prom-client'
|
||||
import Yallist from 'yallist'
|
||||
import { Connection, EntityManager } from 'typeorm'
|
||||
import { createSqliteConnection } from '../../shared/database/sqlite'
|
||||
import { Logger } from 'winston'
|
||||
|
||||
/** A wrapper around a cache value promise. */
|
||||
interface CacheEntry<K, V> {
|
||||
/** The key that can retrieve this cache entry. */
|
||||
key: K
|
||||
|
||||
/** The promise that will resolve the cache value. */
|
||||
promise: Promise<V>
|
||||
|
||||
/**
|
||||
* The size of the promise value, once resolved. This value is
|
||||
* initially zero and is updated once an appropriate can be
|
||||
* determined from the result of `promise`.
|
||||
*/
|
||||
size: number
|
||||
|
||||
/**
|
||||
* The number of active withValue calls referencing this entry. If
|
||||
* this value is non-zero, it is not evict-able from the cache.
|
||||
*/
|
||||
readers: number
|
||||
|
||||
/**
|
||||
* A function reference that should be called, if present, when
|
||||
* the reader count for an entry goes to zero. This will unblock a
|
||||
* a promise created in `bustKey` to wait for all readers to finish
|
||||
* using the cache value.
|
||||
*/
|
||||
waiter: (() => void) | undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* A bag of prometheus metric objects that apply to a particular
|
||||
* instance of `GenericCache`.
|
||||
*/
|
||||
interface CacheMetrics {
|
||||
/**
|
||||
* A metric incremented on each cache insertion and decremented on
|
||||
* each cache eviction.
|
||||
*/
|
||||
sizeGauge: promClient.Gauge<string>
|
||||
|
||||
/**
|
||||
* A metric incremented on each cache hit, miss, and eviction. A `type`
|
||||
* label is applied to differentiate the events.
|
||||
*/
|
||||
eventsCounter: promClient.Counter<string>
|
||||
}
|
||||
|
||||
/**
|
||||
* A generic LRU cache. We use this instead of the `lru-cache` package
|
||||
* available in NPM so that we can handle async payloads in a more
|
||||
* first-class way as well as shedding some of the cruft around evictions.
|
||||
* We need to ensure database handles are closed when they are no longer
|
||||
* accessible, and we also do not want to evict any database handle while
|
||||
* it is actively being used.
|
||||
*/
|
||||
export class GenericCache<K, V> {
|
||||
/** A map from from keys to nodes in `lruList`. */
|
||||
private cache = new Map<K, Yallist.Node<CacheEntry<K, V>>>()
|
||||
|
||||
/** A linked list of cache entires ordered by last-touch. */
|
||||
private lruList = new Yallist<CacheEntry<K, V>>()
|
||||
|
||||
/** The additive size of the items currently in the cache. */
|
||||
private size = 0
|
||||
|
||||
/**
|
||||
* Create a new `GenericCache` with the given maximum (soft) size for
|
||||
* all items in the cache, a function that determine the size of a
|
||||
* cache item from its resolved value, and a function that is called
|
||||
* when an item falls out of the cache.
|
||||
*
|
||||
* @param max The maximum size of the cache before an eviction.
|
||||
* @param sizeFunction A function that determines the size of a cache item.
|
||||
* @param disposeFunction A function that disposes of evicted cache items.
|
||||
* @param cacheMetrics The bag of metrics to use for this instance of the cache.
|
||||
*/
|
||||
constructor(
|
||||
private max: number,
|
||||
private sizeFunction: (value: V) => number,
|
||||
private disposeFunction: (value: V) => Promise<void> | void,
|
||||
private cacheMetrics: CacheMetrics
|
||||
) {}
|
||||
|
||||
/** Remove all values from the cache. */
|
||||
public async flush(): Promise<void> {
|
||||
await Promise.all(Array.from(this.cache.keys()).map(key => this.bustKey(key)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if `key` exists in the cache. If it does not, create a value
|
||||
* from `factory`. Once the cache value resolves, invoke `callback` and
|
||||
* return its value. This method acts as a lock around the cache entry
|
||||
* so that it may not be removed while the factory or callback functions
|
||||
* are running.
|
||||
*
|
||||
* @param key The cache key.
|
||||
* @param factory The function used to create a new value.
|
||||
* @param callback The function to invoke with the resolved cache value.
|
||||
*/
|
||||
public async withValue<T>(key: K, factory: () => Promise<V>, callback: (value: V) => Promise<T>): Promise<T> {
|
||||
// Find or create the entry
|
||||
const entry = await this.getEntry(key, factory)
|
||||
|
||||
try {
|
||||
// Re-resolve the promise. If this is already resolved it's a fast
|
||||
// no-op. Otherwise, we got a cache entry that was under-construction
|
||||
// and will resolve shortly.
|
||||
|
||||
return await callback(await entry.promise)
|
||||
} finally {
|
||||
// Unlock the cache entry
|
||||
entry.readers--
|
||||
|
||||
// If we were the last reader and there's a bustKey call waiting on
|
||||
// us to finish, inform it that we're done using it. Bust away!
|
||||
|
||||
if (entry.readers === 0 && entry.waiter !== undefined) {
|
||||
entry.waiter()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a key from the cache. This blocks until all current readers
|
||||
* of the cached value have completed, then calls the dispose function.
|
||||
*
|
||||
* Do NOT call this function while holding the same: you will deadlock.
|
||||
*
|
||||
* @param key The cache key.
|
||||
*/
|
||||
public async bustKey(key: K): Promise<void> {
|
||||
const node = this.cache.get(key)
|
||||
if (!node) {
|
||||
return
|
||||
}
|
||||
|
||||
const {
|
||||
value: { promise, size, readers },
|
||||
} = node
|
||||
|
||||
// Immediately remove from cache so that another reader cannot get
|
||||
// ahold of the value, and so that another bust attempt cannot call
|
||||
// dispose twice on the same value.
|
||||
|
||||
this.removeNode(node, size)
|
||||
|
||||
// Wait for the value to resolve. We do this first in case the value
|
||||
// was still under construction. This simplifies the rest of the logic
|
||||
// below, as readers can never be negative once the promise value has
|
||||
// resolved.
|
||||
|
||||
const value = await promise
|
||||
|
||||
if (readers > 0) {
|
||||
// There's someone holding the cache value. Create a barrier promise
|
||||
// and stash the function that can unlock it. When the reader count
|
||||
// for an entry is decremented, the waiter function, if present, is
|
||||
// invoked. This basically forms a condition variable.
|
||||
|
||||
const { wait, done } = createBarrierPromise()
|
||||
node.value.waiter = done
|
||||
await wait
|
||||
}
|
||||
|
||||
// We have the resolved value, removed from the cache, which is no longer
|
||||
// used by any reader. It's safe to dispose now.
|
||||
await this.disposeFunction(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if `key` exists in the cache. If it does not, create a value
|
||||
* from `factory` and add it to the cache. In either case, update the
|
||||
* cache entry's place in `lruCache` and return the entry. If a new
|
||||
* value was created, then it may trigger a cache eviction once its
|
||||
* value resolves.
|
||||
*
|
||||
* @param key The cache key.
|
||||
* @param factory The function used to create a new value.
|
||||
*/
|
||||
private async getEntry(key: K, factory: () => Promise<V>): Promise<CacheEntry<K, V>> {
|
||||
const node = this.cache.get(key)
|
||||
if (node) {
|
||||
// Move to head of list
|
||||
this.lruList.unshiftNode(node)
|
||||
|
||||
// Log cache event
|
||||
this.cacheMetrics.eventsCounter.labels('hit').inc()
|
||||
|
||||
// Ensure entry is locked before returning
|
||||
const entry = node.value
|
||||
entry.readers++
|
||||
return entry
|
||||
}
|
||||
|
||||
// Log cache event
|
||||
this.cacheMetrics.eventsCounter.labels('miss').inc()
|
||||
|
||||
// Create promise and the entry that wraps it. We don't know the effective
|
||||
// size of the value until the promise resolves, so we put zero. We have a
|
||||
// reader count of 1, in order to lock the entry until after the user that
|
||||
// requested the entry is done using it. We don't want to block here while
|
||||
// waiting for the promise value to resolve, otherwise a second request for
|
||||
// the same key will create a duplicate cache entry.
|
||||
|
||||
const promise = factory()
|
||||
const newEntry = { key, promise, size: 0, readers: 1, waiter: undefined }
|
||||
|
||||
// Add to head of list
|
||||
this.lruList.unshift(newEntry)
|
||||
|
||||
// Grab the head of the list we just pushed and store it
|
||||
// in the map. We need the node that the unshift method
|
||||
// creates so we can unlink it in constant time.
|
||||
const head = this.lruList.head
|
||||
if (head) {
|
||||
this.cache.set(key, head)
|
||||
}
|
||||
|
||||
// Now that another call to getEntry will find the cache entry
|
||||
// and early-out, we can block here and wait to resolve the
|
||||
// value, then update the entry and cache sizes.
|
||||
|
||||
const value = await promise
|
||||
await this.resolved(newEntry, value)
|
||||
return newEntry
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the size of the resolved value and update the size of the
|
||||
* entry as well as `size`. While the total cache size exceeds `max`,
|
||||
* try to evict the least recently used cache entries that do not have
|
||||
* a non-zero `readers` count.
|
||||
*
|
||||
* @param entry The cache entry.
|
||||
* @param value The cache entry's resolved value.
|
||||
*/
|
||||
private async resolved(entry: CacheEntry<K, V>, value: V): Promise<void> {
|
||||
entry.size = this.sizeFunction(value)
|
||||
this.size += entry.size
|
||||
this.cacheMetrics.sizeGauge.set(this.size)
|
||||
|
||||
let node = this.lruList.tail
|
||||
while (this.size > this.max && node) {
|
||||
const {
|
||||
prev,
|
||||
value: { promise, size, readers },
|
||||
} = node
|
||||
|
||||
if (readers === 0) {
|
||||
// If readers > 0, then it may be actively used by another
|
||||
// part of the code that hit a portion of their critical
|
||||
// section that returned control to the event loop. We don't
|
||||
// want to mess with those if we can help it.
|
||||
|
||||
this.removeNode(node, size)
|
||||
await this.disposeFunction(await promise)
|
||||
|
||||
// Log cache event
|
||||
this.cacheMetrics.eventsCounter.labels('eviction').inc()
|
||||
} else {
|
||||
// Log cache event
|
||||
this.cacheMetrics.eventsCounter.labels('locked-eviction').inc()
|
||||
}
|
||||
|
||||
node = prev
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the given node from the list and update the cache size.
|
||||
*
|
||||
* @param node The node to remove.
|
||||
* @param size The size of the promise value.
|
||||
*/
|
||||
private removeNode(node: Yallist.Node<CacheEntry<K, V>>, size: number): void {
|
||||
this.size -= size
|
||||
this.cacheMetrics.sizeGauge.set(this.size)
|
||||
this.lruList.removeNode(node)
|
||||
this.cache.delete(node.value.key)
|
||||
}
|
||||
}
|
||||
|
||||
/** A cache of SQLite database connections indexed by database filenames. */
|
||||
export class ConnectionCache extends GenericCache<string, Connection> {
|
||||
/**
|
||||
* Create a new `ConnectionCache` with the given maximum (soft) size for
|
||||
* all items in the cache.
|
||||
*/
|
||||
constructor(max: number) {
|
||||
super(
|
||||
max,
|
||||
// Each handle is roughly the same size.
|
||||
() => 1,
|
||||
// Close the underlying file handle on cache eviction.
|
||||
connection => connection.close(),
|
||||
{
|
||||
sizeGauge: metrics.connectionCacheSizeGauge,
|
||||
eventsCounter: metrics.connectionCacheEventsCounter,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke `callback` with a SQLite connection object obtained from the
|
||||
* cache or created on cache miss. This connection is guaranteed not to
|
||||
* be disposed by cache eviction while the callback is active.
|
||||
*
|
||||
* @param database The database filename.
|
||||
* @param entities The set of entities to create on a new connection.
|
||||
* @param logger The logger instance.
|
||||
* @param callback The function invoke with the SQLite connection.
|
||||
*/
|
||||
public withConnection<T>(
|
||||
database: string,
|
||||
// Decorators are not possible type check
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
entities: Function[],
|
||||
logger: Logger,
|
||||
callback: (connection: Connection) => Promise<T>
|
||||
): Promise<T> {
|
||||
return this.withValue(database, () => createSqliteConnection(database, entities, logger), callback)
|
||||
}
|
||||
|
||||
/**
|
||||
* Like `withConnection`, but will open a transaction on the connection
|
||||
* before invoking the callback.
|
||||
*
|
||||
* @param database The database filename.
|
||||
* @param entities The set of entities to create on a new connection.
|
||||
* @param logger The logger instance.
|
||||
* @param callback The function invoke with a SQLite transaction connection.
|
||||
*/
|
||||
public withTransactionalEntityManager<T>(
|
||||
database: string,
|
||||
// Decorators are not possible type check
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
entities: Function[],
|
||||
logger: Logger,
|
||||
callback: (entityManager: EntityManager) => Promise<T>
|
||||
): Promise<T> {
|
||||
return this.withConnection(database, entities, logger, connection => connection.transaction(callback))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A wrapper around a cache value that retains its encoded size. In order to keep
|
||||
* the in-memory limit of these decoded items, we use this value as the cache entry
|
||||
* size. This assumes that the size of the encoded text is a good proxy for the size
|
||||
* of the in-memory representation.
|
||||
*/
|
||||
export interface EncodedJsonCacheValue<T> {
|
||||
/** The size of the encoded value. */
|
||||
size: number
|
||||
|
||||
/** The decoded value. */
|
||||
data: T
|
||||
}
|
||||
|
||||
/** A cache of decoded values encoded as JSON and gzipped in a SQLite database. */
|
||||
class EncodedJsonCache<K, V> extends GenericCache<K, EncodedJsonCacheValue<V>> {
|
||||
/**
|
||||
* Create a new `EncodedJsonCache` with the given maximum (soft) size for
|
||||
* all items in the cache.
|
||||
*
|
||||
* @param max The maximum size of the cache before an eviction.
|
||||
* @param cacheMetrics The bag of metrics to use for this instance of the cache.
|
||||
*/
|
||||
constructor(max: number, cacheMetrics: CacheMetrics) {
|
||||
super(
|
||||
max,
|
||||
v => v.size,
|
||||
// Let GC handle the cleanup of the object on cache eviction.
|
||||
() => {
|
||||
/* noop */
|
||||
},
|
||||
cacheMetrics
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A cache of deserialized `DocumentData` values indexed by a string containing
|
||||
* the database path and the path of the document.
|
||||
*/
|
||||
export class DocumentCache extends EncodedJsonCache<string, sqliteModels.DocumentData> {
|
||||
/**
|
||||
* Create a new `DocumentCache` with the given maximum (soft) size for
|
||||
* all items in the cache.
|
||||
*
|
||||
* @param max The maximum size of the cache before an eviction.
|
||||
*/
|
||||
constructor(max: number) {
|
||||
super(max, {
|
||||
sizeGauge: metrics.documentCacheSizeGauge,
|
||||
eventsCounter: metrics.documentCacheEventsCounter,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A cache of deserialized `ResultChunkData` values indexed by a string containing
|
||||
* the database path and the chunk index.
|
||||
*/
|
||||
export class ResultChunkCache extends EncodedJsonCache<string, sqliteModels.ResultChunkData> {
|
||||
/**
|
||||
* Create a new `ResultChunkCache` with the given maximum (soft) size for
|
||||
* all items in the cache.
|
||||
*
|
||||
* @param max The maximum size of the cache before an eviction.
|
||||
*/
|
||||
constructor(max: number) {
|
||||
super(max, {
|
||||
sizeGauge: metrics.resultChunkCacheSizeGauge,
|
||||
eventsCounter: metrics.resultChunkCacheEventsCounter,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/** Return a promise and a function pair. The promise resolves once the function is called. */
|
||||
export function createBarrierPromise(): { wait: Promise<void>; done: () => void } {
|
||||
let done!: () => void
|
||||
const wait = new Promise<void>(resolve => (done = resolve))
|
||||
return { wait, done }
|
||||
}
|
||||
@ -1,378 +0,0 @@
|
||||
import * as sqliteModels from '../../shared/models/sqlite'
|
||||
import { comparePosition, findRanges, mapRangesToInternalLocations, Database } from './database'
|
||||
import * as fs from 'mz/fs'
|
||||
import * as nodepath from 'path'
|
||||
import { convertLsif } from '../../worker/conversion/importer'
|
||||
import { PathExistenceChecker } from '../../worker/conversion/existence'
|
||||
import rmfr from 'rmfr'
|
||||
import * as uuid from 'uuid'
|
||||
|
||||
describe('Database', () => {
|
||||
let storageRoot!: string
|
||||
let database!: Database
|
||||
|
||||
const makeDatabase = async (filename: string): Promise<Database> => {
|
||||
// Create a filesystem read stream for the given test file. This will cover
|
||||
// the cases where `yarn test` is run from the root or from the lsif directory.
|
||||
const sourceFile = nodepath.join(
|
||||
(await fs.exists('cmd')) ? 'cmd/precise-code-intel' : '',
|
||||
'test-data',
|
||||
filename
|
||||
)
|
||||
const databaseFile = nodepath.join(storageRoot, uuid.v4())
|
||||
|
||||
await convertLsif({
|
||||
path: sourceFile,
|
||||
root: '',
|
||||
database: databaseFile,
|
||||
pathExistenceChecker: new PathExistenceChecker({
|
||||
repositoryId: 42,
|
||||
commit: 'ad3507cbeb18d1ed2b8a0f6354dea88a101197f3',
|
||||
root: '',
|
||||
}),
|
||||
})
|
||||
|
||||
return new Database(1, databaseFile)
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
storageRoot = await fs.mkdtemp('test-', { encoding: 'utf8' })
|
||||
database = await makeDatabase('lsif-go@ad3507cb.lsif.gz')
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (storageRoot) {
|
||||
await rmfr(storageRoot)
|
||||
}
|
||||
})
|
||||
|
||||
describe('exists', () => {
|
||||
it('should check document path', async () => {
|
||||
expect(await database.exists('cmd/lsif-go/main.go')).toEqual(true)
|
||||
expect(await database.exists('internal/index/indexer.go')).toEqual(true)
|
||||
expect(await database.exists('missing.go')).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('definitions', () => {
|
||||
it('should correlate definitions', async () => {
|
||||
// `\ts, err := indexer.Index()` -> `\t Index() (*Stats, error)`
|
||||
// ^^^^^ ^^^^^
|
||||
|
||||
expect(await database.definitions('cmd/lsif-go/main.go', { line: 110, character: 22 })).toEqual([
|
||||
{
|
||||
path: 'internal/index/indexer.go',
|
||||
range: { start: { line: 20, character: 1 }, end: { line: 20, character: 6 } },
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('references', () => {
|
||||
it('should correlate references', async () => {
|
||||
// `func (w *Writer) EmitRange(start, end Pos) (string, error) {`
|
||||
// ^^^^^^^^^
|
||||
//
|
||||
// -> `\t\trangeID, err := i.w.EmitRange(lspRange(ipos, ident.Name, isQuotedPkgName))`
|
||||
// ^^^^^^^^^
|
||||
//
|
||||
// -> `\t\t\trangeID, err = i.w.EmitRange(lspRange(ipos, ident.Name, false))`
|
||||
// ^^^^^^^^^
|
||||
|
||||
expect((await database.references('protocol/writer.go', { line: 85, character: 20 })).values).toEqual([
|
||||
{
|
||||
path: 'protocol/writer.go',
|
||||
range: { start: { line: 85, character: 17 }, end: { line: 85, character: 26 } },
|
||||
},
|
||||
{
|
||||
path: 'internal/index/indexer.go',
|
||||
range: { start: { line: 529, character: 22 }, end: { line: 529, character: 31 } },
|
||||
},
|
||||
{
|
||||
path: 'internal/index/indexer.go',
|
||||
range: { start: { line: 380, character: 22 }, end: { line: 380, character: 31 } },
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('hover', () => {
|
||||
it('should correlate hover text', async () => {
|
||||
// `\tcontents, err := findContents(pkgs, p, f, obj)`
|
||||
// ^^^^^^^^^^^^
|
||||
|
||||
const ticks = '```'
|
||||
const docstring = 'findContents returns contents used as hover info for given object.'
|
||||
const signature =
|
||||
'func findContents(pkgs []*Package, p *Package, f *File, obj Object) ([]MarkedString, error)'
|
||||
|
||||
expect(await database.hover('internal/index/indexer.go', { line: 628, character: 20 })).toEqual({
|
||||
text: `${ticks}go\n${signature}\n${ticks}\n\n---\n\n${docstring}`,
|
||||
range: { start: { line: 628, character: 18 }, end: { line: 628, character: 30 } },
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('monikersByPosition', () => {
|
||||
it('should return correct range and document with monikers', async () => {
|
||||
// `func NewMetaData(id, root string, info ToolInfo) *MetaData {`
|
||||
// ^^^^^^^^^^^
|
||||
|
||||
const monikers = await database.monikersByPosition('protocol/protocol.go', {
|
||||
line: 92,
|
||||
character: 10,
|
||||
})
|
||||
|
||||
expect(monikers).toHaveLength(1)
|
||||
expect(monikers[0]).toHaveLength(1)
|
||||
expect(monikers[0][0]?.kind).toEqual('export')
|
||||
expect(monikers[0][0]?.scheme).toEqual('gomod')
|
||||
expect(monikers[0][0]?.identifier).toEqual('github.com/sourcegraph/lsif-go/protocol:NewMetaData')
|
||||
})
|
||||
})
|
||||
|
||||
describe('monikerResults', () => {
|
||||
const edgeLocations = [
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 600, character: 1 }, end: { line: 600, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 644, character: 1 }, end: { line: 644, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 507, character: 1 }, end: { line: 507, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 553, character: 1 }, end: { line: 553, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 462, character: 1 }, end: { line: 462, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 484, character: 1 }, end: { line: 484, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 410, character: 5 }, end: { line: 410, character: 9 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 622, character: 1 }, end: { line: 622, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 440, character: 1 }, end: { line: 440, character: 5 } },
|
||||
},
|
||||
{
|
||||
path: 'protocol/protocol.go',
|
||||
range: { start: { line: 530, character: 1 }, end: { line: 530, character: 5 } },
|
||||
},
|
||||
]
|
||||
|
||||
it('should query definitions table', async () => {
|
||||
const { locations, count } = await database.monikerResults(
|
||||
sqliteModels.DefinitionModel,
|
||||
{
|
||||
scheme: 'gomod',
|
||||
identifier: 'github.com/sourcegraph/lsif-go/protocol:Edge',
|
||||
},
|
||||
{}
|
||||
)
|
||||
|
||||
expect(locations).toEqual(edgeLocations)
|
||||
expect(count).toEqual(10)
|
||||
})
|
||||
|
||||
it('should respect pagination', async () => {
|
||||
const { locations, count } = await database.monikerResults(
|
||||
sqliteModels.DefinitionModel,
|
||||
{
|
||||
scheme: 'gomod',
|
||||
identifier: 'github.com/sourcegraph/lsif-go/protocol:Edge',
|
||||
},
|
||||
{ skip: 3, take: 4 }
|
||||
)
|
||||
|
||||
expect(locations).toEqual(edgeLocations.slice(3, 7))
|
||||
expect(count).toEqual(10)
|
||||
})
|
||||
|
||||
it('should query references table', async () => {
|
||||
const { locations, count } = await database.monikerResults(
|
||||
sqliteModels.ReferenceModel,
|
||||
{
|
||||
scheme: 'gomod',
|
||||
identifier: 'github.com/slimsag/godocmd:ToMarkdown',
|
||||
},
|
||||
{}
|
||||
)
|
||||
|
||||
expect(locations).toEqual([
|
||||
{
|
||||
path: 'internal/index/helper.go',
|
||||
range: { start: { line: 78, character: 6 }, end: { line: 78, character: 16 } },
|
||||
},
|
||||
])
|
||||
expect(count).toEqual(1)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('findRanges', () => {
|
||||
it('should return ranges containing position', () => {
|
||||
const range1 = {
|
||||
startLine: 0,
|
||||
startCharacter: 3,
|
||||
endLine: 0,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range2 = {
|
||||
startLine: 1,
|
||||
startCharacter: 3,
|
||||
endLine: 1,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range3 = {
|
||||
startLine: 2,
|
||||
startCharacter: 3,
|
||||
endLine: 2,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range4 = {
|
||||
startLine: 3,
|
||||
startCharacter: 3,
|
||||
endLine: 3,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range5 = {
|
||||
startLine: 4,
|
||||
startCharacter: 3,
|
||||
endLine: 4,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
|
||||
expect(findRanges([range1, range2, range3, range4, range5], { line: 0, character: 4 })).toEqual([range1])
|
||||
expect(findRanges([range1, range2, range3, range4, range5], { line: 1, character: 4 })).toEqual([range2])
|
||||
expect(findRanges([range1, range2, range3, range4, range5], { line: 2, character: 4 })).toEqual([range3])
|
||||
expect(findRanges([range1, range2, range3, range4, range5], { line: 3, character: 4 })).toEqual([range4])
|
||||
expect(findRanges([range1, range2, range3, range4, range5], { line: 4, character: 4 })).toEqual([range5])
|
||||
})
|
||||
|
||||
it('should order inner-most ranges first', () => {
|
||||
const range1 = {
|
||||
startLine: 0,
|
||||
startCharacter: 3,
|
||||
endLine: 4,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range2 = {
|
||||
startLine: 1,
|
||||
startCharacter: 3,
|
||||
endLine: 3,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range3 = {
|
||||
startLine: 2,
|
||||
startCharacter: 3,
|
||||
endLine: 2,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range4 = {
|
||||
startLine: 5,
|
||||
startCharacter: 3,
|
||||
endLine: 5,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
const range5 = {
|
||||
startLine: 6,
|
||||
startCharacter: 3,
|
||||
endLine: 6,
|
||||
endCharacter: 5,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
|
||||
expect(findRanges([range1, range2, range3, range4, range5], { line: 2, character: 4 })).toEqual([
|
||||
range3,
|
||||
range2,
|
||||
range1,
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('comparePosition', () => {
|
||||
it('should return the relative order to a range', () => {
|
||||
const range = {
|
||||
startLine: 5,
|
||||
startCharacter: 11,
|
||||
endLine: 5,
|
||||
endCharacter: 13,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
}
|
||||
|
||||
expect(comparePosition(range, { line: 5, character: 11 })).toEqual(0)
|
||||
expect(comparePosition(range, { line: 5, character: 12 })).toEqual(0)
|
||||
expect(comparePosition(range, { line: 5, character: 13 })).toEqual(0)
|
||||
expect(comparePosition(range, { line: 4, character: 12 })).toEqual(+1)
|
||||
expect(comparePosition(range, { line: 5, character: 10 })).toEqual(+1)
|
||||
expect(comparePosition(range, { line: 5, character: 14 })).toEqual(-1)
|
||||
expect(comparePosition(range, { line: 6, character: 12 })).toEqual(-1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('mapRangesToInternalLocations', () => {
|
||||
it('should map ranges to locations', () => {
|
||||
const ranges = new Map<sqliteModels.RangeId, sqliteModels.RangeData>()
|
||||
ranges.set(1, {
|
||||
startLine: 1,
|
||||
startCharacter: 1,
|
||||
endLine: 1,
|
||||
endCharacter: 2,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
})
|
||||
ranges.set(2, {
|
||||
startLine: 3,
|
||||
startCharacter: 1,
|
||||
endLine: 3,
|
||||
endCharacter: 2,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
})
|
||||
ranges.set(4, {
|
||||
startLine: 2,
|
||||
startCharacter: 1,
|
||||
endLine: 2,
|
||||
endCharacter: 2,
|
||||
monikerIds: new Set<sqliteModels.MonikerId>(),
|
||||
})
|
||||
|
||||
const path = 'src/position.ts'
|
||||
const locations = mapRangesToInternalLocations(ranges, path, new Set([1, 2, 4]))
|
||||
expect(locations).toContainEqual({
|
||||
path,
|
||||
range: { start: { line: 1, character: 1 }, end: { line: 1, character: 2 } },
|
||||
})
|
||||
expect(locations).toContainEqual({
|
||||
path,
|
||||
range: { start: { line: 3, character: 1 }, end: { line: 3, character: 2 } },
|
||||
})
|
||||
expect(locations).toContainEqual({
|
||||
path,
|
||||
range: { start: { line: 2, character: 1 }, end: { line: 2, character: 2 } },
|
||||
})
|
||||
expect(locations).toHaveLength(3)
|
||||
})
|
||||
})
|
||||
@ -1,588 +0,0 @@
|
||||
import * as cache from './cache'
|
||||
import * as sqliteModels from '../../shared/models/sqlite'
|
||||
import * as lsp from 'vscode-languageserver-protocol'
|
||||
import * as metrics from '../metrics'
|
||||
import * as pgModels from '../../shared/models/pg'
|
||||
import { Connection } from 'typeorm'
|
||||
import { DefaultMap } from '../../shared/datastructures/default-map'
|
||||
import { gunzipJSON } from '../../shared/encoding/json'
|
||||
import { hashKey } from '../../shared/models/hash'
|
||||
import { instrument } from '../../shared/metrics'
|
||||
import { logSpan, TracingContext, logAndTraceCall, addTags } from '../../shared/tracing'
|
||||
import { mustGet } from '../../shared/maps'
|
||||
import { Logger } from 'winston'
|
||||
import { createSilentLogger } from '../../shared/logging'
|
||||
import { InternalLocation, OrderedLocationSet } from './location'
|
||||
import * as settings from '../settings'
|
||||
|
||||
/** The maximum number of results in a logSpan value. */
|
||||
const MAX_SPAN_ARRAY_LENGTH = 20
|
||||
|
||||
/** A wrapper around operations related to a single SQLite dump. */
|
||||
export class Database {
|
||||
/**
|
||||
* A static map of database paths to the `numResultChunks` value of their
|
||||
* metadata row. This map is populated lazily as the values are needed.
|
||||
*/
|
||||
private static numResultChunks = new Map<string, number>()
|
||||
private static connectionCache = new cache.ConnectionCache(settings.CONNECTION_CACHE_CAPACITY)
|
||||
private static documentCache = new cache.DocumentCache(settings.DOCUMENT_CACHE_CAPACITY)
|
||||
private static resultChunkCache = new cache.ResultChunkCache(settings.RESULT_CHUNK_CACHE_CAPACITY)
|
||||
|
||||
/**
|
||||
* Create a new `Database` with the given dump record, and the SQLite file
|
||||
* on disk that contains data for a particular repository and commit.
|
||||
*
|
||||
* @param dumpId The identifier of the dump for which this database answers queries.
|
||||
* @param databasePath The path to the database file.
|
||||
*/
|
||||
constructor(private dumpId: pgModels.DumpId, private databasePath: string) {}
|
||||
|
||||
/**
|
||||
* Determine if data exists for a particular document in this database.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public exists(path: string, ctx: TracingContext = {}): Promise<boolean> {
|
||||
return this.logAndTraceCall(
|
||||
ctx,
|
||||
'Checking if path exists',
|
||||
async () => (await this.getDocumentByPath(path)) !== undefined
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of locations that define the symbol at the given position.
|
||||
*
|
||||
* @param path The path of the document to which the position belongs.
|
||||
* @param position The current hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async definitions(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<InternalLocation[]> {
|
||||
return this.logAndTraceCall(ctx, 'Fetching definitions', async ctx => {
|
||||
const { document, ranges } = await this.getRangeByPosition(path, position, ctx)
|
||||
if (!document || ranges.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
for (const range of ranges) {
|
||||
if (!range.definitionResultId) {
|
||||
continue
|
||||
}
|
||||
|
||||
const definitionResults = await this.getResultById(range.definitionResultId)
|
||||
this.logSpan(ctx, 'definition_results', {
|
||||
definitionResultId: range.definitionResultId,
|
||||
definitionResults: definitionResults.slice(0, MAX_SPAN_ARRAY_LENGTH),
|
||||
numDefinitionResults: definitionResults.length,
|
||||
})
|
||||
|
||||
return this.convertRangesToInternalLocations(path, document, definitionResults)
|
||||
}
|
||||
|
||||
return []
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of unique locations that reference the symbol at the given position.
|
||||
*
|
||||
* @param path The path of the document to which the position belongs.
|
||||
* @param position The current hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async references(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<OrderedLocationSet> {
|
||||
return this.logAndTraceCall(ctx, 'Fetching references', async ctx => {
|
||||
const { document, ranges } = await this.getRangeByPosition(path, position, ctx)
|
||||
if (!document || ranges.length === 0) {
|
||||
return new OrderedLocationSet()
|
||||
}
|
||||
|
||||
const locationSet = new OrderedLocationSet()
|
||||
for (const range of ranges) {
|
||||
if (range.referenceResultId) {
|
||||
const referenceResults = await this.getResultById(range.referenceResultId)
|
||||
this.logSpan(ctx, 'reference_results', {
|
||||
referenceResultId: range.referenceResultId,
|
||||
referenceResults: referenceResults.slice(0, MAX_SPAN_ARRAY_LENGTH),
|
||||
numReferenceResults: referenceResults.length,
|
||||
})
|
||||
|
||||
if (referenceResults.length > 0) {
|
||||
for (const location of await this.convertRangesToInternalLocations(
|
||||
path,
|
||||
document,
|
||||
referenceResults
|
||||
)) {
|
||||
locationSet.push(location)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return locationSet
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the hover content for the symbol at the given position.
|
||||
*
|
||||
* @param path The path of the document to which the position belongs.
|
||||
* @param position The current hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async hover(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<{ text: string; range: lsp.Range } | null> {
|
||||
return this.logAndTraceCall(ctx, 'Fetching hover', async ctx => {
|
||||
const { document, ranges } = await this.getRangeByPosition(path, position, ctx)
|
||||
if (!document || ranges.length === 0) {
|
||||
return null
|
||||
}
|
||||
|
||||
for (const range of ranges) {
|
||||
if (!range.hoverResultId) {
|
||||
continue
|
||||
}
|
||||
|
||||
this.logSpan(ctx, 'hover_result', { hoverResultId: range.hoverResultId })
|
||||
|
||||
// Extract text
|
||||
const text = mustGet(document.hoverResults, range.hoverResultId, 'hoverResult')
|
||||
|
||||
// Return first defined hover result for the inner-most range. This response
|
||||
// includes the entire range so that the highlighted portion in the UI can be
|
||||
// accurate (rather than approximated by the tokenizer).
|
||||
return { text, range: createRange(range) }
|
||||
}
|
||||
|
||||
return null
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Return all of the monikers attached to all ranges that contain the given position. The
|
||||
* resulting list is grouped by range. If multiple ranges contain this position, then the
|
||||
* list monikers for the inner-most ranges will occur before the outer-most ranges.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param position The user's hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async monikersByPosition(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<sqliteModels.MonikerData[][]> {
|
||||
const { document, ranges } = await this.getRangeByPosition(path, position, ctx)
|
||||
if (!document) {
|
||||
return []
|
||||
}
|
||||
|
||||
return ranges.map(range =>
|
||||
Array.from(range.monikerIds).map(monikerId => mustGet(document.monikers, monikerId, 'moniker'))
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Query the definitions or references table of `db` for items that match the given moniker.
|
||||
* Convert each result into an `InternalLocation`. The `pathTransformer` function is invoked
|
||||
* on each result item to modify the resulting locations.
|
||||
*
|
||||
* @param model The constructor for the model type.
|
||||
* @param moniker The target moniker.
|
||||
* @param pagination A limit and offset to use for the query.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public monikerResults(
|
||||
model: typeof sqliteModels.DefinitionModel | typeof sqliteModels.ReferenceModel,
|
||||
moniker: Pick<sqliteModels.MonikerData, 'scheme' | 'identifier'>,
|
||||
pagination: { skip?: number; take?: number },
|
||||
ctx: TracingContext = {}
|
||||
): Promise<{ locations: InternalLocation[]; count: number }> {
|
||||
return this.logAndTraceCall(ctx, 'Fetching moniker results', async ctx => {
|
||||
const [results, count] = await this.withConnection(
|
||||
connection =>
|
||||
connection
|
||||
.getRepository<sqliteModels.DefinitionModel | sqliteModels.ReferenceModel>(model)
|
||||
.findAndCount({
|
||||
where: {
|
||||
scheme: moniker.scheme,
|
||||
identifier: moniker.identifier,
|
||||
},
|
||||
...pagination,
|
||||
}),
|
||||
ctx.logger
|
||||
)
|
||||
|
||||
this.logSpan(ctx, 'symbol_results', {
|
||||
moniker,
|
||||
results: results.slice(0, MAX_SPAN_ARRAY_LENGTH),
|
||||
numResults: results.length,
|
||||
})
|
||||
|
||||
const locations = results.map(result => ({
|
||||
path: result.documentPath,
|
||||
range: createRange(result),
|
||||
}))
|
||||
|
||||
return { locations, count }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the package information data with the given identifier.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param packageInformationId The identifier of the package information data.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
public async packageInformation(
|
||||
path: string,
|
||||
packageInformationId: string,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<sqliteModels.PackageInformationData | undefined> {
|
||||
const document = await this.getDocumentByPath(path, ctx)
|
||||
if (!document) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return (
|
||||
// TODO - normalize ids before we serialize them in the database
|
||||
document.packageInformation.get(parseInt(packageInformationId, 10)) ||
|
||||
document.packageInformation.get(packageInformationId)
|
||||
)
|
||||
}
|
||||
|
||||
//
|
||||
// Helper Functions
|
||||
|
||||
/**
|
||||
* Return a parsed document that describes the given path. The result of this
|
||||
* method is cached across all database instances. If the document is not found
|
||||
* it returns undefined; other errors will throw.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
private async getDocumentByPath(
|
||||
path: string,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<sqliteModels.DocumentData | undefined> {
|
||||
const factory = async (): Promise<cache.EncodedJsonCacheValue<sqliteModels.DocumentData>> => {
|
||||
const document = await this.withConnection(
|
||||
connection => connection.getRepository(sqliteModels.DocumentModel).findOneOrFail(path),
|
||||
ctx.logger
|
||||
)
|
||||
|
||||
return {
|
||||
size: document.data.length,
|
||||
data: await gunzipJSON<sqliteModels.DocumentData>(document.data),
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return await Database.documentCache.withValue(`${this.databasePath}::${path}`, factory, document =>
|
||||
Promise.resolve(document.data)
|
||||
)
|
||||
} catch (error) {
|
||||
if (error.name === 'EntityNotFound') {
|
||||
return undefined
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a parsed document that describes the given path as well as the ranges
|
||||
* from that document that contains the given position. If multiple ranges are
|
||||
* returned, then the inner-most ranges will occur before the outer-most ranges.
|
||||
*
|
||||
* @param path The path of the document.
|
||||
* @param position The user's hover position.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
private getRangeByPosition(
|
||||
path: string,
|
||||
position: lsp.Position,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<{ document: sqliteModels.DocumentData | undefined; ranges: sqliteModels.RangeData[] }> {
|
||||
return this.logAndTraceCall(ctx, 'Fetching range by position', async ctx => {
|
||||
const document = await this.getDocumentByPath(path)
|
||||
if (!document) {
|
||||
return { document: undefined, ranges: [] }
|
||||
}
|
||||
|
||||
const ranges = findRanges(document.ranges.values(), position)
|
||||
this.logSpan(ctx, 'matching_ranges', { ranges: cleanRanges(ranges) })
|
||||
return { document, ranges }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a set of range-document pairs (from a definition or reference query) into
|
||||
* a set of `InternalLocation` object. Each pair holds the range identifier as well as
|
||||
* the document path. For document paths matching the loaded document, find the range
|
||||
* data locally. For all other paths, find the document in this database and find the
|
||||
* range in that document.
|
||||
*
|
||||
* @param path The path of the document for this query.
|
||||
* @param document The document object for this query.
|
||||
* @param resultData A list of range ids and the document they belong to.
|
||||
*/
|
||||
private async convertRangesToInternalLocations(
|
||||
path: string,
|
||||
document: sqliteModels.DocumentData,
|
||||
resultData: sqliteModels.DocumentPathRangeId[]
|
||||
): Promise<InternalLocation[]> {
|
||||
// Group by document path so we only have to load each document once
|
||||
const groupedResults = new DefaultMap<string, Set<sqliteModels.RangeId>>(() => new Set())
|
||||
|
||||
for (const { documentPath, rangeId } of resultData) {
|
||||
groupedResults.getOrDefault(documentPath).add(rangeId)
|
||||
}
|
||||
|
||||
let results: InternalLocation[] = []
|
||||
for (const [documentPath, rangeIds] of groupedResults) {
|
||||
if (documentPath === path) {
|
||||
// If the document path is this document, convert the locations directly
|
||||
results = results.concat(mapRangesToInternalLocations(document.ranges, path, rangeIds))
|
||||
continue
|
||||
}
|
||||
|
||||
// Otherwise, we need to get the correct document
|
||||
const sibling = await this.getDocumentByPath(documentPath)
|
||||
if (!sibling) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Then finally convert the locations in the sibling document
|
||||
results = results.concat(mapRangesToInternalLocations(sibling.ranges, documentPath, rangeIds))
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a list of ranges with document ids into a list of ranges with
|
||||
* document paths by looking into the result chunks table and parsing the
|
||||
* data associated with the given identifier.
|
||||
*
|
||||
* @param id The identifier of the definition or reference result.
|
||||
*/
|
||||
private async getResultById(
|
||||
id: sqliteModels.DefinitionReferenceResultId
|
||||
): Promise<sqliteModels.DocumentPathRangeId[]> {
|
||||
const { documentPaths, documentIdRangeIds } = await this.getResultChunkByResultId(id)
|
||||
const ranges = mustGet(documentIdRangeIds, id, 'documentIdRangeId')
|
||||
|
||||
return ranges.map(range => ({
|
||||
documentPath: mustGet(documentPaths, range.documentId, 'documentPath'),
|
||||
rangeId: range.rangeId,
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a parsed result chunk that contains the given identifier.
|
||||
*
|
||||
* @param id An identifier contained in the result chunk.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
private async getResultChunkByResultId(
|
||||
id: sqliteModels.DefinitionReferenceResultId,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<sqliteModels.ResultChunkData> {
|
||||
// Find the result chunk index this id belongs to
|
||||
const index = hashKey(id, await this.getNumResultChunks())
|
||||
|
||||
const factory = async (): Promise<cache.EncodedJsonCacheValue<sqliteModels.ResultChunkData>> => {
|
||||
const resultChunk = await this.withConnection(
|
||||
connection => connection.getRepository(sqliteModels.ResultChunkModel).findOneOrFail(index),
|
||||
ctx.logger
|
||||
)
|
||||
|
||||
return {
|
||||
size: resultChunk.data.length,
|
||||
data: await gunzipJSON<sqliteModels.ResultChunkData>(resultChunk.data),
|
||||
}
|
||||
}
|
||||
|
||||
return Database.resultChunkCache.withValue(`${this.databasePath}::${index}`, factory, resultChunk =>
|
||||
Promise.resolve(resultChunk.data)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the `numResultChunks` value from this database's metadata row.
|
||||
*
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
private async getNumResultChunks(ctx: TracingContext = {}): Promise<number> {
|
||||
const numResultChunks = Database.numResultChunks.get(this.databasePath)
|
||||
if (numResultChunks !== undefined) {
|
||||
return numResultChunks
|
||||
}
|
||||
|
||||
// Not in the shared map, need to query it
|
||||
const meta = await this.withConnection(
|
||||
connection => connection.getRepository(sqliteModels.MetaModel).findOneOrFail(1),
|
||||
ctx.logger
|
||||
)
|
||||
Database.numResultChunks.set(this.databasePath, meta.numResultChunks)
|
||||
return meta.numResultChunks
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke `callback` with a SQLite connection object obtained from the
|
||||
* cache or created on cache miss.
|
||||
*
|
||||
* @param callback The function invoke with the SQLite connection.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
private withConnection<T>(
|
||||
callback: (connection: Connection) => Promise<T>,
|
||||
logger: Logger = createSilentLogger()
|
||||
): Promise<T> {
|
||||
return Database.connectionCache.withConnection(this.databasePath, sqliteModels.entities, logger, connection =>
|
||||
instrument(metrics.databaseQueryDurationHistogram, metrics.databaseQueryErrorsCounter, () =>
|
||||
callback(connection)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Log and trace the execution of a function.
|
||||
*
|
||||
* @param ctx The tracing context.
|
||||
* @param name The name of the span and text of the log message.
|
||||
* @param f The function to invoke.
|
||||
*/
|
||||
private logAndTraceCall<T>(ctx: TracingContext, name: string, f: (ctx: TracingContext) => Promise<T>): Promise<T> {
|
||||
return logAndTraceCall(addTags(ctx, { dbID: this.dumpId }), name, f)
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs an event to the span of the tracing context, if its defined.
|
||||
*
|
||||
* @param ctx The tracing context.
|
||||
* @param event The name of the event.
|
||||
* @param pairs The values to log.
|
||||
*/
|
||||
private logSpan(ctx: TracingContext, event: string, pairs: { [name: string]: unknown }): void {
|
||||
logSpan(ctx, event, { ...pairs, dbID: this.dumpId })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the set of ranges that contain the given position. If multiple ranges
|
||||
* are returned, then the inner-most ranges will occur before the outer-most
|
||||
* ranges.
|
||||
*
|
||||
* @param ranges The set of possible ranges.
|
||||
* @param position The user's hover position.
|
||||
*/
|
||||
export function findRanges(ranges: Iterable<sqliteModels.RangeData>, position: lsp.Position): sqliteModels.RangeData[] {
|
||||
const filtered = []
|
||||
for (const range of ranges) {
|
||||
if (comparePosition(range, position) === 0) {
|
||||
filtered.push(range)
|
||||
}
|
||||
}
|
||||
|
||||
return filtered.sort((a, b) => {
|
||||
if (comparePosition(a, { line: b.startLine, character: b.startCharacter }) === 0) {
|
||||
return +1
|
||||
}
|
||||
|
||||
return -1
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a position against a range. Returns 0 if the position occurs
|
||||
* within the range (inclusive bounds), -1 if the position occurs after
|
||||
* it, and +1 if the position occurs before it.
|
||||
*
|
||||
* @param range The range.
|
||||
* @param position The position.
|
||||
*/
|
||||
export function comparePosition(range: sqliteModels.RangeData, position: lsp.Position): number {
|
||||
if (position.line < range.startLine) {
|
||||
return +1
|
||||
}
|
||||
|
||||
if (position.line > range.endLine) {
|
||||
return -1
|
||||
}
|
||||
|
||||
if (position.line === range.startLine && position.character < range.startCharacter) {
|
||||
return +1
|
||||
}
|
||||
|
||||
if (position.line === range.endLine && position.character > range.endCharacter) {
|
||||
return -1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct an LSP range from a flat range.
|
||||
*
|
||||
* @param result The start/end line/character of the range.
|
||||
*/
|
||||
function createRange(result: {
|
||||
startLine: number
|
||||
startCharacter: number
|
||||
endLine: number
|
||||
endCharacter: number
|
||||
}): lsp.Range {
|
||||
return lsp.Range.create(result.startLine, result.startCharacter, result.endLine, result.endCharacter)
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the given range identifiers into an `InternalLocation` objects.
|
||||
*
|
||||
* @param ranges The map of ranges of the document.
|
||||
* @param uri The location URI.
|
||||
* @param ids The set of range identifiers for each resulting location.
|
||||
*/
|
||||
export function mapRangesToInternalLocations(
|
||||
ranges: Map<sqliteModels.RangeId, sqliteModels.RangeData>,
|
||||
uri: string,
|
||||
ids: Set<sqliteModels.RangeId>
|
||||
): InternalLocation[] {
|
||||
const locations = []
|
||||
for (const id of ids) {
|
||||
locations.push({
|
||||
path: uri,
|
||||
range: createRange(mustGet(ranges, id, 'range')),
|
||||
})
|
||||
}
|
||||
|
||||
return locations
|
||||
}
|
||||
|
||||
/**
|
||||
* Format ranges to be serialized in opentracing logs.
|
||||
*
|
||||
* @param ranges The ranges to clean.
|
||||
*/
|
||||
function cleanRanges(
|
||||
ranges: sqliteModels.RangeData[]
|
||||
): (Omit<sqliteModels.RangeData, 'monikerIds'> & { monikerIds: sqliteModels.MonikerId[] })[] {
|
||||
// We need to array-ize sets otherwise we get a "0 key" object
|
||||
return ranges.map(r => ({ ...r, monikerIds: Array.from(r.monikerIds) }))
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
import * as lsp from 'vscode-languageserver-protocol'
|
||||
import { OrderedSet } from '../../shared/datastructures/orderedset'
|
||||
|
||||
export interface InternalLocation {
|
||||
/** The path relative to the dump root. */
|
||||
path: string
|
||||
range: lsp.Range
|
||||
}
|
||||
|
||||
/** A duplicate-free list of locations ordered by time of insertion. */
|
||||
export class OrderedLocationSet extends OrderedSet<InternalLocation> {
|
||||
/**
|
||||
* Create a new ordered locations set.
|
||||
*
|
||||
* @param values A set of values used to seed the set.
|
||||
*/
|
||||
constructor(values?: InternalLocation[]) {
|
||||
super(
|
||||
(value: InternalLocation): string =>
|
||||
[
|
||||
value.path,
|
||||
value.range.start.line,
|
||||
value.range.start.character,
|
||||
value.range.end.line,
|
||||
value.range.end.character,
|
||||
].join(':'),
|
||||
values
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
import * as constants from '../shared/constants'
|
||||
import * as path from 'path'
|
||||
import * as settings from './settings'
|
||||
import * as metrics from './metrics'
|
||||
import promClient from 'prom-client'
|
||||
import { createLogger } from '../shared/logging'
|
||||
import { ensureDirectory } from '../shared/paths'
|
||||
import { Logger } from 'winston'
|
||||
import { startExpressApp } from '../shared/api/init'
|
||||
import { createDatabaseRouter } from './routes/database'
|
||||
import { createUploadRouter } from './routes/uploads'
|
||||
import { startTasks } from './tasks'
|
||||
import { createPostgresConnection } from '../shared/database/postgres'
|
||||
import { waitForConfiguration } from '../shared/config/config'
|
||||
|
||||
/**
|
||||
* Runs the HTTP server that stores and queries individual SQLite files.
|
||||
*
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
async function main(logger: Logger): Promise<void> {
|
||||
// Collect process metrics
|
||||
promClient.collectDefaultMetrics({ prefix: 'lsif_' })
|
||||
|
||||
// Read configuration from frontend
|
||||
const fetchConfiguration = await waitForConfiguration(logger)
|
||||
|
||||
// Update cache capacities on startup
|
||||
metrics.connectionCacheCapacityGauge.set(settings.CONNECTION_CACHE_CAPACITY)
|
||||
metrics.documentCacheCapacityGauge.set(settings.DOCUMENT_CACHE_CAPACITY)
|
||||
metrics.resultChunkCacheCapacityGauge.set(settings.RESULT_CHUNK_CACHE_CAPACITY)
|
||||
|
||||
// Ensure storage roots exist
|
||||
await ensureDirectory(settings.STORAGE_ROOT)
|
||||
await ensureDirectory(path.join(settings.STORAGE_ROOT, constants.DBS_DIR))
|
||||
await ensureDirectory(path.join(settings.STORAGE_ROOT, constants.UPLOADS_DIR))
|
||||
|
||||
// Create database connection
|
||||
const connection = await createPostgresConnection(fetchConfiguration(), logger)
|
||||
|
||||
// Start background tasks
|
||||
startTasks(connection, logger)
|
||||
|
||||
const routers = [createDatabaseRouter(logger), createUploadRouter(logger)]
|
||||
|
||||
// Start server
|
||||
startExpressApp({ port: settings.HTTP_PORT, routers, logger })
|
||||
}
|
||||
|
||||
// Initialize logger
|
||||
const appLogger = createLogger('precise-code-intel-bundle-manager')
|
||||
|
||||
// Launch!
|
||||
main(appLogger).catch(error => {
|
||||
appLogger.error('Failed to start process', { error })
|
||||
appLogger.on('finish', () => process.exit(1))
|
||||
appLogger.end()
|
||||
})
|
||||
@ -1,83 +0,0 @@
|
||||
import promClient from 'prom-client'
|
||||
|
||||
//
|
||||
// HTTP Metrics
|
||||
|
||||
export const httpUploadDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_http_upload_request_duration_seconds',
|
||||
help: 'Total time spent on upload requests.',
|
||||
labelNames: ['code'],
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
export const httpQueryDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_http_query_request_duration_seconds',
|
||||
help: 'Total time spent on query requests.',
|
||||
labelNames: ['code'],
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
//
|
||||
// Database Metrics
|
||||
|
||||
export const databaseQueryDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_database_query_duration_seconds',
|
||||
help: 'Total time spent on database queries.',
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
export const databaseQueryErrorsCounter = new promClient.Counter({
|
||||
name: 'lsif_database_query_errors_total',
|
||||
help: 'The number of errors that occurred during a database query.',
|
||||
})
|
||||
|
||||
//
|
||||
// Cache Metrics
|
||||
|
||||
export const connectionCacheCapacityGauge = new promClient.Gauge({
|
||||
name: 'lsif_connection_cache_capacity',
|
||||
help: 'The maximum number of open SQLite handles.',
|
||||
})
|
||||
|
||||
export const connectionCacheSizeGauge = new promClient.Gauge({
|
||||
name: 'lsif_connection_cache_size',
|
||||
help: 'The current number of open SQLite handles.',
|
||||
})
|
||||
|
||||
export const connectionCacheEventsCounter = new promClient.Counter({
|
||||
name: 'lsif_connection_cache_events_total',
|
||||
help: 'The number of connection cache hits, misses, and evictions.',
|
||||
labelNames: ['type'],
|
||||
})
|
||||
|
||||
export const documentCacheCapacityGauge = new promClient.Gauge({
|
||||
name: 'lsif_document_cache_capacity',
|
||||
help: 'The maximum number of documents loaded in memory.',
|
||||
})
|
||||
|
||||
export const documentCacheSizeGauge = new promClient.Gauge({
|
||||
name: 'lsif_document_cache_size',
|
||||
help: 'The current number of documents loaded in memory.',
|
||||
})
|
||||
|
||||
export const documentCacheEventsCounter = new promClient.Counter({
|
||||
name: 'lsif_document_cache_events_total',
|
||||
help: 'The number of document cache hits, misses, and evictions.',
|
||||
labelNames: ['type'],
|
||||
})
|
||||
|
||||
export const resultChunkCacheCapacityGauge = new promClient.Gauge({
|
||||
name: 'lsif_results_chunk_cache_capacity',
|
||||
help: 'The maximum number of result chunks loaded in memory.',
|
||||
})
|
||||
|
||||
export const resultChunkCacheSizeGauge = new promClient.Gauge({
|
||||
name: 'lsif_results_chunk_cache_size',
|
||||
help: 'The current number of result chunks loaded in memory.',
|
||||
})
|
||||
|
||||
export const resultChunkCacheEventsCounter = new promClient.Counter({
|
||||
name: 'lsif_results_chunk_cache_events_total',
|
||||
help: 'The number of result chunk cache hits, misses, and evictions.',
|
||||
labelNames: ['type'],
|
||||
})
|
||||
@ -1,227 +0,0 @@
|
||||
import * as settings from '../settings'
|
||||
import express from 'express'
|
||||
import { addTags, TracingContext } from '../../shared/tracing'
|
||||
import { Logger } from 'winston'
|
||||
import { pipeline as _pipeline } from 'stream'
|
||||
import { Span } from 'opentracing'
|
||||
import { wrap } from 'async-middleware'
|
||||
import { Database } from '../backend/database'
|
||||
import * as sqliteModels from '../../shared/models/sqlite'
|
||||
import { InternalLocation } from '../backend/location'
|
||||
import { dbFilename } from '../../shared/paths'
|
||||
import * as lsp from 'vscode-languageserver-protocol'
|
||||
import * as validation from '../../shared/api/middleware/validation'
|
||||
|
||||
/**
|
||||
* Create a router containing the SQLite query endpoints.
|
||||
*
|
||||
* For now, each public method of Database (see sif/src/bundle-manager/backend/database.ts) is
|
||||
* exposed at `/<database-id>/<method>`. This interface is likely to change soon.
|
||||
*
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export function createDatabaseRouter(logger: Logger): express.Router {
|
||||
const router = express.Router()
|
||||
|
||||
/**
|
||||
* Create a tracing context from the request logger and tracing span
|
||||
* tagged with the given values.
|
||||
*
|
||||
* @param req The express request.
|
||||
* @param tags The tags to apply to the logger and span.
|
||||
*/
|
||||
const createTracingContext = (
|
||||
req: express.Request & { span?: Span },
|
||||
tags: { [K: string]: unknown }
|
||||
): TracingContext => addTags({ logger, span: req.span }, tags)
|
||||
|
||||
const withDatabase = async <T>(
|
||||
req: express.Request,
|
||||
res: express.Response<T>,
|
||||
handler: (database: Database, ctx?: TracingContext) => Promise<T>
|
||||
): Promise<void> => {
|
||||
const id = parseInt(req.params.id, 10)
|
||||
const ctx = createTracingContext(req, { id })
|
||||
const database = new Database(id, dbFilename(settings.STORAGE_ROOT, id))
|
||||
|
||||
const payload = await handler(database, ctx)
|
||||
res.json(payload)
|
||||
}
|
||||
|
||||
interface ExistsQueryArgs {
|
||||
path: string
|
||||
}
|
||||
|
||||
type ExistsResponse = boolean
|
||||
|
||||
router.get(
|
||||
'/dbs/:id([0-9]+)/exists',
|
||||
validation.validationMiddleware([validation.validateNonEmptyString('path')]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<ExistsResponse>): Promise<void> => {
|
||||
const { path }: ExistsQueryArgs = req.query
|
||||
await withDatabase(req, res, (database, ctx) => database.exists(path, ctx))
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface DefinitionsQueryArgs {
|
||||
path: string
|
||||
line: number
|
||||
character: number
|
||||
}
|
||||
|
||||
type DefinitionsResponse = InternalLocation[]
|
||||
|
||||
router.get(
|
||||
'/dbs/:id([0-9]+)/definitions',
|
||||
validation.validationMiddleware([
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateInt('line'),
|
||||
validation.validateInt('character'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<DefinitionsResponse>): Promise<void> => {
|
||||
const { path, line, character }: DefinitionsQueryArgs = req.query
|
||||
await withDatabase(req, res, (database, ctx) => database.definitions(path, { line, character }, ctx))
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface ReferencesQueryArgs {
|
||||
path: string
|
||||
line: number
|
||||
character: number
|
||||
}
|
||||
|
||||
type ReferencesResponse = InternalLocation[]
|
||||
|
||||
router.get(
|
||||
'/dbs/:id([0-9]+)/references',
|
||||
validation.validationMiddleware([
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateInt('line'),
|
||||
validation.validateInt('character'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<ReferencesResponse>): Promise<void> => {
|
||||
const { path, line, character }: ReferencesQueryArgs = req.query
|
||||
await withDatabase(
|
||||
req,
|
||||
res,
|
||||
async (database, ctx) => (await database.references(path, { line, character }, ctx)).values
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface HoverQueryArgs {
|
||||
path: string
|
||||
line: number
|
||||
character: number
|
||||
}
|
||||
|
||||
type HoverResponse = { text: string; range: lsp.Range } | null
|
||||
|
||||
router.get(
|
||||
'/dbs/:id([0-9]+)/hover',
|
||||
validation.validationMiddleware([
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateInt('line'),
|
||||
validation.validateInt('character'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<HoverResponse>): Promise<void> => {
|
||||
const { path, line, character }: HoverQueryArgs = req.query
|
||||
await withDatabase(req, res, (database, ctx) => database.hover(path, { line, character }, ctx))
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface MonikersByPositionQueryArgs {
|
||||
path: string
|
||||
line: number
|
||||
character: number
|
||||
}
|
||||
|
||||
type MonikersByPositionResponse = sqliteModels.MonikerData[][]
|
||||
|
||||
router.get(
|
||||
'/dbs/:id([0-9]+)/monikersByPosition',
|
||||
validation.validationMiddleware([
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateInt('line'),
|
||||
validation.validateInt('character'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<MonikersByPositionResponse>): Promise<void> => {
|
||||
const { path, line, character }: MonikersByPositionQueryArgs = req.query
|
||||
await withDatabase(req, res, (database, ctx) =>
|
||||
database.monikersByPosition(path, { line, character }, ctx)
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface MonikerResultsQueryArgs {
|
||||
modelType: string
|
||||
scheme: string
|
||||
identifier: string
|
||||
skip?: number
|
||||
take?: number
|
||||
}
|
||||
|
||||
interface MonikerResultsResponse {
|
||||
locations: { path: string; range: lsp.Range }[]
|
||||
count: number
|
||||
}
|
||||
|
||||
router.get(
|
||||
'/dbs/:id([0-9]+)/monikerResults',
|
||||
validation.validationMiddleware([
|
||||
validation.validateNonEmptyString('modelType'),
|
||||
validation.validateNonEmptyString('scheme'),
|
||||
validation.validateNonEmptyString('identifier'),
|
||||
validation.validateOptionalInt('skip'),
|
||||
validation.validateOptionalInt('take'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<MonikerResultsResponse>): Promise<void> => {
|
||||
const { modelType, scheme, identifier, skip, take }: MonikerResultsQueryArgs = req.query
|
||||
await withDatabase(req, res, (database, ctx) =>
|
||||
database.monikerResults(
|
||||
modelType === 'definition' ? sqliteModels.DefinitionModel : sqliteModels.ReferenceModel,
|
||||
{ scheme, identifier },
|
||||
{ skip, take },
|
||||
ctx
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
interface PackageInformationQueryArgs {
|
||||
path: string
|
||||
packageInformationId: string
|
||||
}
|
||||
|
||||
type PackageInformationResponse = sqliteModels.PackageInformationData | undefined
|
||||
|
||||
router.get(
|
||||
'/dbs/:id([0-9]+)/packageInformation',
|
||||
validation.validationMiddleware([
|
||||
validation.validateNonEmptyString('path'),
|
||||
validation.validateNonEmptyString('packageInformationId'),
|
||||
]),
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<PackageInformationResponse>): Promise<void> => {
|
||||
const { path, packageInformationId }: PackageInformationQueryArgs = req.query
|
||||
await withDatabase(req, res, (database, ctx) =>
|
||||
database.packageInformation(path, packageInformationId, ctx)
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return router
|
||||
}
|
||||
@ -1,100 +0,0 @@
|
||||
import express from 'express'
|
||||
import { Logger } from 'winston'
|
||||
import { Span } from 'opentracing'
|
||||
import { wrap } from 'async-middleware'
|
||||
import { addTags, TracingContext, logAndTraceCall } from '../../shared/tracing'
|
||||
import { pipeline as _pipeline } from 'stream'
|
||||
import { promisify } from 'util'
|
||||
import * as fs from 'mz/fs'
|
||||
import * as settings from '../settings'
|
||||
import { dbFilename, uploadFilename } from '../../shared/paths'
|
||||
import { ThrottleGroup, Throttle } from 'stream-throttle'
|
||||
|
||||
const pipeline = promisify(_pipeline)
|
||||
|
||||
/**
|
||||
* Create a router containing the upload endpoints.
|
||||
*
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export function createUploadRouter(logger: Logger): express.Router {
|
||||
const router = express.Router()
|
||||
|
||||
const makeServeThrottle = makeThrottleFactory(
|
||||
settings.MAXIMUM_SERVE_BYTES_PER_SECOND,
|
||||
settings.MAXIMUM_SERVE_CHUNK_BYTES
|
||||
)
|
||||
|
||||
const makeUploadThrottle = makeThrottleFactory(
|
||||
settings.MAXIMUM_UPLOAD_BYTES_PER_SECOND,
|
||||
settings.MAXIMUM_UPLOAD_CHUNK_BYTES
|
||||
)
|
||||
|
||||
/**
|
||||
* Create a tracing context from the request logger and tracing span
|
||||
* tagged with the given values.
|
||||
*
|
||||
* @param req The express request.
|
||||
* @param tags The tags to apply to the logger and span.
|
||||
*/
|
||||
const createTracingContext = (
|
||||
req: express.Request & { span?: Span },
|
||||
tags: { [K: string]: unknown }
|
||||
): TracingContext => addTags({ logger, span: req.span }, tags)
|
||||
|
||||
router.get(
|
||||
'/uploads/:id([0-9]+)',
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<unknown>): Promise<void> => {
|
||||
const id = parseInt(req.params.id, 10)
|
||||
const ctx = createTracingContext(req, { id })
|
||||
const filename = uploadFilename(settings.STORAGE_ROOT, id)
|
||||
const stream = fs.createReadStream(filename)
|
||||
await logAndTraceCall(ctx, 'Serving payload', () => pipeline(stream, makeServeThrottle(), res))
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
router.post(
|
||||
'/uploads/:id([0-9]+)',
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<unknown>): Promise<void> => {
|
||||
const id = parseInt(req.params.id, 10)
|
||||
const ctx = createTracingContext(req, { id })
|
||||
const filename = uploadFilename(settings.STORAGE_ROOT, id)
|
||||
const stream = fs.createWriteStream(filename)
|
||||
await logAndTraceCall(ctx, 'Uploading payload', () => pipeline(req, makeUploadThrottle(), stream))
|
||||
res.send()
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
router.post(
|
||||
'/dbs/:id([0-9]+)',
|
||||
wrap(
|
||||
async (req: express.Request, res: express.Response<unknown>): Promise<void> => {
|
||||
const id = parseInt(req.params.id, 10)
|
||||
const ctx = createTracingContext(req, { id })
|
||||
const filename = dbFilename(settings.STORAGE_ROOT, id)
|
||||
const stream = fs.createWriteStream(filename)
|
||||
await logAndTraceCall(ctx, 'Uploading payload', () => pipeline(req, makeUploadThrottle(), stream))
|
||||
res.send()
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a function that will create a throttle that can be used as a stream
|
||||
* transformer. This transformer can limit both readable and writable streams.
|
||||
*
|
||||
* @param rate The maximum bit second of the stream.
|
||||
* @param chunksize The size of chunks used to break down larger slices of data.
|
||||
*/
|
||||
function makeThrottleFactory(rate: number, chunksize: number): () => Throttle {
|
||||
const opts = { rate, chunksize }
|
||||
const throttleGroup = new ThrottleGroup(opts)
|
||||
return () => throttleGroup.throttle(opts)
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
import { readEnvInt } from '../shared/settings'
|
||||
|
||||
/** Which port to run the bundle manager API on. Defaults to 3187. */
|
||||
export const HTTP_PORT = readEnvInt('HTTP_PORT', 3187)
|
||||
|
||||
/** HTTP address for internal precise code intel API. */
|
||||
export const PRECISE_CODE_INTEL_API_SERVER_URL =
|
||||
process.env.PRECISE_CODE_INTEL_API_SERVER_URL || 'http://localhost:3186'
|
||||
|
||||
/** Where on the file system to store LSIF files. This should be a persistent volume. */
|
||||
export const STORAGE_ROOT = process.env.LSIF_STORAGE_ROOT || 'lsif-storage'
|
||||
|
||||
/**
|
||||
* The number of SQLite connections that can be opened at once. This
|
||||
* value may be exceeded for a short period if many handles are held
|
||||
* at once.
|
||||
*/
|
||||
export const CONNECTION_CACHE_CAPACITY = readEnvInt('CONNECTION_CACHE_CAPACITY', 100)
|
||||
|
||||
/** The maximum number of documents that can be held in memory at once. */
|
||||
export const DOCUMENT_CACHE_CAPACITY = readEnvInt('DOCUMENT_CACHE_CAPACITY', 1024 * 1024 * 1024)
|
||||
|
||||
/** The maximum number of result chunks that can be held in memory at once. */
|
||||
export const RESULT_CHUNK_CACHE_CAPACITY = readEnvInt('RESULT_CHUNK_CACHE_CAPACITY', 1024 * 1024 * 1024)
|
||||
|
||||
/** The interval (in seconds) to clean the dbs directory. */
|
||||
export const PURGE_OLD_DUMPS_INTERVAL = readEnvInt('PURGE_OLD_DUMPS_INTERVAL', 60 * 30)
|
||||
|
||||
/** How many uploads to query at once when determining if a db file is unreferenced. */
|
||||
export const DEAD_DUMP_BATCH_SIZE = readEnvInt('DEAD_DUMP_BATCH_SIZE', 100)
|
||||
|
||||
/** The maximum space (in bytes) that the dbs directory can use. */
|
||||
export const DBS_DIR_MAXIMUM_SIZE_BYTES = readEnvInt('DBS_DIR_MAXIMUM_SIZE_BYTES', 1024 * 1024 * 1024 * 10)
|
||||
|
||||
/** The interval (in seconds) to invoke the cleanFailedUploads task. */
|
||||
export const CLEAN_FAILED_UPLOADS_INTERVAL = readEnvInt('CLEAN_FAILED_UPLOADS_INTERVAL', 60 * 60 * 8)
|
||||
|
||||
/** The maximum age (in seconds) that the files for an unprocessed upload can remain on disk. */
|
||||
export const FAILED_UPLOAD_MAX_AGE = readEnvInt('FAILED_UPLOAD_MAX_AGE', 24 * 60 * 60)
|
||||
|
||||
/** How many times to retry requests to precise-code-intel-api-server in the background. */
|
||||
export const MAX_REQUEST_RETRIES = readEnvInt('MAX_REQUEST_RETRIES', 60)
|
||||
|
||||
/** How long to wait (minimum, in seconds) between precise-code-intel-api-server request attempts. */
|
||||
export const MIN_REQUEST_RETRY_TIMEOUT = readEnvInt('MIN_REQUEST_RETRY_TIMEOUT', 1)
|
||||
|
||||
/** How long to wait (maximum, in seconds) between precise-code-intel-api-server request attempts. */
|
||||
export const MAX_REQUEST_RETRY_TIMEOUT = readEnvInt('MAX_REQUEST_RETRY_TIMEOUT', 30)
|
||||
|
||||
/** The maximum rate that the server will send upload payloads. */
|
||||
export const MAXIMUM_SERVE_BYTES_PER_SECOND = readEnvInt('MAXIMUM_SERVE_BYTES_PER_SECOND', 1024 * 1024 * 1024 * 10) // 10 GiB/sec
|
||||
|
||||
/** The maximum chunk size the server will use to send upload payloads. */
|
||||
export const MAXIMUM_SERVE_CHUNK_BYTES = readEnvInt('MAXIMUM_SERVE_CHUNK_BYTES', 1024 * 1024 * 10) // 10 MiB
|
||||
|
||||
/** The maximum rate that the server will receive upload payloads. */
|
||||
export const MAXIMUM_UPLOAD_BYTES_PER_SECOND = readEnvInt('MAXIMUM_UPLOAD_BYTES_PER_SECOND', 1024 * 1024 * 1024 * 10) // 10 GiB/sec
|
||||
|
||||
/** The maximum chunk size the server will use to receive upload payloads. */
|
||||
export const MAXIMUM_UPLOAD_CHUNK_BYTES = readEnvInt('MAXIMUM_UPLOAD_CHUNK_BYTES', 1024 * 1024 * 10) // 10 MiB
|
||||
@ -1,206 +0,0 @@
|
||||
import * as settings from './settings'
|
||||
import { Connection } from 'typeorm'
|
||||
import { Logger } from 'winston'
|
||||
import { ExclusivePeriodicTaskRunner } from '../shared/tasks'
|
||||
import * as constants from '../shared/constants'
|
||||
import * as fs from 'mz/fs'
|
||||
import * as path from 'path'
|
||||
import { chunk } from 'lodash'
|
||||
import { createSilentLogger } from '../shared/logging'
|
||||
import { TracingContext } from '../shared/tracing'
|
||||
import { dbFilename, idFromFilename } from '../shared/paths'
|
||||
import got from 'got'
|
||||
import pRetry from 'p-retry'
|
||||
import { parseJSON } from '../shared/encoding/json'
|
||||
|
||||
/**
|
||||
* Begin running cleanup tasks on a schedule in the background.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export function startTasks(connection: Connection, logger: Logger): void {
|
||||
const runner = new ExclusivePeriodicTaskRunner(connection, logger)
|
||||
|
||||
runner.register({
|
||||
name: 'Purging old dumps',
|
||||
intervalMs: settings.PURGE_OLD_DUMPS_INTERVAL,
|
||||
task: ({ ctx }) => purgeOldDumps(settings.STORAGE_ROOT, settings.DBS_DIR_MAXIMUM_SIZE_BYTES, ctx),
|
||||
})
|
||||
|
||||
runner.register({
|
||||
name: 'Cleaning failed uploads',
|
||||
intervalMs: settings.CLEAN_FAILED_UPLOADS_INTERVAL,
|
||||
task: ({ ctx }) => cleanFailedUploads(ctx),
|
||||
})
|
||||
|
||||
runner.run()
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove dumps until the space occupied by the dbs directory is below
|
||||
* the given limit.
|
||||
*
|
||||
* @param storageRoot The path where SQLite databases are stored.
|
||||
* @param maximumSizeBytes The maximum number of bytes (< 0 means no limit).
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
async function purgeOldDumps(
|
||||
storageRoot: string,
|
||||
maximumSizeBytes: number,
|
||||
{ logger = createSilentLogger() }: TracingContext = {}
|
||||
): Promise<void> {
|
||||
// First, remove all the files in the DB dir that don't have a corresponding
|
||||
// lsif_upload record in the database. This will happen in the cases where an
|
||||
// upload overlaps existing uploads which are deleted in batch from the db,
|
||||
// but not from disk. This can also happen if the db file is written during
|
||||
// processing but fails later while updating commits for that repo.
|
||||
await removeDeadDumps(storageRoot, { logger })
|
||||
|
||||
if (maximumSizeBytes < 0) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
let currentSizeBytes = await dirsize(path.join(storageRoot, constants.DBS_DIR))
|
||||
|
||||
while (currentSizeBytes > maximumSizeBytes) {
|
||||
// While our current data usage is too big, find candidate dumps to delete
|
||||
const payload: { id: number } = await makeServerRequest('/prune')
|
||||
if (!payload) {
|
||||
logger.warn(
|
||||
'Unable to reduce disk usage of the DB directory because deleting any single dump would drop in-use code intel for a repository.',
|
||||
{ currentSizeBytes, softMaximumSizeBytes: maximumSizeBytes }
|
||||
)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
// Delete this dump and subtract its size from the current dir size
|
||||
const filename = dbFilename(storageRoot, payload.id)
|
||||
currentSizeBytes -= await filesize(filename)
|
||||
await fs.unlink(filename)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove db files that are not reachable from a pending or completed upload record.
|
||||
*
|
||||
* @param storageRoot The path where SQLite databases are stored.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
async function removeDeadDumps(
|
||||
storageRoot: string,
|
||||
{ logger = createSilentLogger() }: TracingContext = {}
|
||||
): Promise<void> {
|
||||
let count = 0
|
||||
for (const basenames of chunk(
|
||||
await fs.readdir(path.join(storageRoot, constants.DBS_DIR)),
|
||||
settings.DEAD_DUMP_BATCH_SIZE
|
||||
)) {
|
||||
const pathsById = new Map<number, string>()
|
||||
for (const basename of basenames) {
|
||||
const id = idFromFilename(basename)
|
||||
if (!id) {
|
||||
continue
|
||||
}
|
||||
|
||||
pathsById.set(id, path.join(storageRoot, constants.DBS_DIR, basename))
|
||||
}
|
||||
|
||||
const states: Map<number, string> = await makeServerRequest('/uploads', { ids: Array.from(pathsById.keys()) })
|
||||
for (const [id, dbPath] of pathsById.entries()) {
|
||||
if (!states.has(id) || states.get(id) === 'errored') {
|
||||
count++
|
||||
await fs.unlink(dbPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (count > 0) {
|
||||
logger.debug('Removed dead dumps', { count })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove upload and temp files that are older than `FAILED_UPLOAD_MAX_AGE`. This assumes
|
||||
* that an upload conversion's total duration (from enqueue to completion) is less than this
|
||||
* interval during healthy operation.
|
||||
*
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
async function cleanFailedUploads({ logger = createSilentLogger() }: TracingContext): Promise<void> {
|
||||
let count = 0
|
||||
for (const basename of await fs.readdir(path.join(settings.STORAGE_ROOT, constants.UPLOADS_DIR))) {
|
||||
if (await purgeFile(path.join(settings.STORAGE_ROOT, constants.UPLOADS_DIR, basename))) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
if (count > 0) {
|
||||
logger.debug('Removed old files', { count })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the given file if it was last modified longer than `FAILED_UPLOAD_MAX_AGE` seconds
|
||||
* ago. Returns true if the file was removed and false otherwise.
|
||||
*
|
||||
* @param filename The file to remove.
|
||||
*/
|
||||
async function purgeFile(filename: string): Promise<boolean> {
|
||||
if (Date.now() - (await fs.stat(filename)).mtimeMs < settings.FAILED_UPLOAD_MAX_AGE * 1000) {
|
||||
return false
|
||||
}
|
||||
|
||||
await fs.unlink(filename)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the cumulative size of all plain files in a directory, non-recursively.
|
||||
*
|
||||
* @param directory The directory path.
|
||||
*/
|
||||
async function dirsize(directory: string): Promise<number> {
|
||||
return (
|
||||
await Promise.all((await fs.readdir(directory)).map(filename => filesize(path.join(directory, filename))))
|
||||
).reduce((a, b) => a + b, 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file size or zero if it doesn't exist.
|
||||
*
|
||||
* @param filename The filename.
|
||||
*/
|
||||
async function filesize(filename: string): Promise<number> {
|
||||
try {
|
||||
return (await fs.stat(filename)).size
|
||||
} catch (error) {
|
||||
if (!(error && error.code === 'ENOENT')) {
|
||||
throw error
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
async function makeServerRequest<T, R>(route: string, payload?: T): Promise<R> {
|
||||
return pRetry(
|
||||
async (): Promise<R> =>
|
||||
parseJSON(
|
||||
(
|
||||
await got.post(new URL(route, settings.PRECISE_CODE_INTEL_API_SERVER_URL).href, {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
).body
|
||||
),
|
||||
{
|
||||
factor: 1.5,
|
||||
randomize: true,
|
||||
retries: settings.MAX_REQUEST_RETRIES,
|
||||
minTimeout: settings.MIN_REQUEST_RETRY_TIMEOUT * 1000,
|
||||
maxTimeout: settings.MAX_REQUEST_RETRY_TIMEOUT * 1000,
|
||||
}
|
||||
)
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
import express from 'express'
|
||||
import promClient from 'prom-client'
|
||||
import { default as tracingMiddleware } from 'express-opentracing'
|
||||
import { errorHandler } from './middleware/errors'
|
||||
import { logger as loggingMiddleware } from 'express-winston'
|
||||
import { makeMetricsMiddleware } from './middleware/metrics'
|
||||
import { Tracer } from 'opentracing'
|
||||
import { Logger } from 'winston'
|
||||
import { jsonReplacer } from '../encoding/json'
|
||||
|
||||
export function startExpressApp({
|
||||
port,
|
||||
routers = [],
|
||||
logger,
|
||||
tracer,
|
||||
selectHistogram = () => undefined,
|
||||
}: {
|
||||
port: number
|
||||
routers?: express.Router[]
|
||||
logger: Logger
|
||||
tracer?: Tracer
|
||||
selectHistogram?: (route: string) => promClient.Histogram<string> | undefined
|
||||
}): void {
|
||||
const loggingOptions = {
|
||||
winstonInstance: logger,
|
||||
level: 'debug',
|
||||
ignoredRoutes: ['/ping', '/healthz', '/metrics'],
|
||||
requestWhitelist: ['method', 'url'],
|
||||
msg: 'Handled request',
|
||||
}
|
||||
|
||||
const app = express()
|
||||
app.use(tracingMiddleware({ tracer }))
|
||||
app.use(loggingMiddleware(loggingOptions))
|
||||
app.use(makeMetricsMiddleware(selectHistogram))
|
||||
app.use(createMetaRouter())
|
||||
|
||||
for (const route of routers) {
|
||||
app.use(route)
|
||||
}
|
||||
|
||||
// Error handler must be registered last so its exception handlers
|
||||
// will apply to all routes and other middleware.
|
||||
app.use(errorHandler(logger))
|
||||
|
||||
app.set('json replacer', jsonReplacer)
|
||||
|
||||
app.listen(port, () => logger.debug('API server listening', { port }))
|
||||
}
|
||||
|
||||
/** Create a router containing health and metrics endpoint. */
|
||||
function createMetaRouter(): express.Router {
|
||||
const router = express.Router()
|
||||
router.get('/ping', (_, res) => res.send('ok'))
|
||||
router.get('/healthz', (_, res) => res.send('ok'))
|
||||
router.get('/metrics', (_, res) => {
|
||||
res.writeHead(200, { 'Content-Type': 'text/plain' })
|
||||
res.end(promClient.register.metrics())
|
||||
})
|
||||
|
||||
return router
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
import express from 'express'
|
||||
import { Logger } from 'winston'
|
||||
|
||||
interface ErrorResponse {
|
||||
message: string
|
||||
}
|
||||
|
||||
export interface ApiError {
|
||||
message: string
|
||||
status?: number
|
||||
}
|
||||
|
||||
export const isApiError = (val: unknown): val is ApiError => typeof val === 'object' && !!val && 'message' in val
|
||||
|
||||
/**
|
||||
* Middleware function used to convert uncaught exceptions into 500 responses.
|
||||
*
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export const errorHandler = (
|
||||
logger: Logger
|
||||
): ((
|
||||
error: unknown,
|
||||
req: express.Request,
|
||||
res: express.Response<ErrorResponse>,
|
||||
next: express.NextFunction
|
||||
) => void) => (
|
||||
error: unknown,
|
||||
req: express.Request,
|
||||
res: express.Response<ErrorResponse>,
|
||||
// Express uses argument length to distinguish middleware and error handlers
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
next: express.NextFunction
|
||||
): void => {
|
||||
const status = (isApiError(error) && error.status) || 500
|
||||
const message = (isApiError(error) && error.message) || 'Unknown error'
|
||||
|
||||
if (status === 500) {
|
||||
logger.error('uncaught exception', { error })
|
||||
}
|
||||
|
||||
if (!res.headersSent) {
|
||||
res.status(status).send({ message })
|
||||
}
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
import express from 'express'
|
||||
import onFinished from 'on-finished'
|
||||
import promClient from 'prom-client'
|
||||
|
||||
/**
|
||||
* Creates a middleware function used to emit HTTP durations for LSIF functions.
|
||||
* Originally we used an express bundle, but that did not allow us to have different
|
||||
* histogram bucket for different endpoints, which makes half of the metrics useless
|
||||
* in the presence of large uploads.
|
||||
*/
|
||||
export const makeMetricsMiddleware = <T>(
|
||||
selectHistogram: (route: string) => promClient.Histogram<string> | undefined
|
||||
) => (req: express.Request, res: express.Response<T>, next: express.NextFunction): void => {
|
||||
const histogram = selectHistogram(req.path)
|
||||
|
||||
if (histogram !== undefined) {
|
||||
const labels = { code: 0 }
|
||||
const end = histogram.startTimer(labels)
|
||||
|
||||
onFinished(res, () => {
|
||||
labels.code = res.statusCode
|
||||
end()
|
||||
})
|
||||
}
|
||||
|
||||
next()
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
import express from 'express'
|
||||
import { query, ValidationChain, validationResult, ValidationError } from 'express-validator'
|
||||
import { parseCursor } from '../pagination/cursor'
|
||||
|
||||
/**
|
||||
* Create a query string validator for a required non-empty string value.
|
||||
*
|
||||
* @param key The query string key.
|
||||
*/
|
||||
export const validateNonEmptyString = (key: string): ValidationChain => query(key).isString().not().isEmpty()
|
||||
|
||||
/**
|
||||
* Create a query string validator for a possibly empty string value.
|
||||
*
|
||||
* @param key The query string key.
|
||||
*/
|
||||
export const validateOptionalString = (key: string): ValidationChain =>
|
||||
query(key)
|
||||
.optional()
|
||||
.customSanitizer(value => value || '')
|
||||
|
||||
/**
|
||||
* Create a query string validator for a possibly empty boolean value.
|
||||
*
|
||||
* @param key The query string key.
|
||||
*/
|
||||
export const validateOptionalBoolean = (key: string): ValidationChain => query(key).optional().isBoolean().toBoolean()
|
||||
|
||||
/**
|
||||
* Create a query string validator for an integer value.
|
||||
*
|
||||
* @param key The query string key.
|
||||
*/
|
||||
export const validateInt = (key: string): ValidationChain => query(key).isInt().toInt()
|
||||
|
||||
/**
|
||||
* Create a query string validator for a possibly empty integer value.
|
||||
*
|
||||
* @param key The query string key.
|
||||
*/
|
||||
export const validateOptionalInt = (key: string): ValidationChain => query(key).optional().isInt().toInt()
|
||||
|
||||
/** A validator used for a string query field. */
|
||||
export const validateQuery = validateOptionalString('query')
|
||||
|
||||
/**
|
||||
* Create a query string validator for an LSIF upload state.
|
||||
*
|
||||
* @param key The query string key.
|
||||
*/
|
||||
export const validateLsifUploadState = query('state').optional().isIn(['queued', 'completed', 'errored', 'processing'])
|
||||
|
||||
/** Create a validator for an integer limit field. */
|
||||
export const validateLimit = validateOptionalInt('limit')
|
||||
|
||||
/** A validator used for an integer offset field. */
|
||||
export const validateOffset = validateOptionalInt('offset')
|
||||
|
||||
/** Create a validator for a cursor that is serialized as the supplied generic type. */
|
||||
export const validateCursor = <T>(): ValidationChain =>
|
||||
validateOptionalString('cursor').customSanitizer(value => parseCursor<T>(value))
|
||||
|
||||
interface ValidationErrorResponse {
|
||||
errors: Record<string, ValidationError>
|
||||
}
|
||||
|
||||
/**
|
||||
* Middleware function used to apply a sequence of validators and then return
|
||||
* an unprocessable entity response with an error message if validation fails.
|
||||
*/
|
||||
export const validationMiddleware = (chains: ValidationChain[]) => async (
|
||||
req: express.Request,
|
||||
res: express.Response<ValidationErrorResponse>,
|
||||
next: express.NextFunction
|
||||
): Promise<void> => {
|
||||
await Promise.all(chains.map(chain => chain.run(req)))
|
||||
|
||||
const errors = validationResult(req)
|
||||
if (!errors.isEmpty()) {
|
||||
res.status(422).send({ errors: errors.mapped() })
|
||||
return
|
||||
}
|
||||
|
||||
next()
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
/**
|
||||
* Parse a base64-encoded JSON payload into the expected type.
|
||||
*
|
||||
* @param cursorRaw The raw cursor.
|
||||
*/
|
||||
export function parseCursor<T>(cursorRaw: string | undefined): T | undefined {
|
||||
if (cursorRaw === undefined) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(Buffer.from(cursorRaw, 'base64').toString())
|
||||
} catch {
|
||||
throw Object.assign(new Error(`Malformed cursor supplied ${cursorRaw}`), { status: 400 })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode an arbitrary pagination cursor value into a string.
|
||||
*
|
||||
* @param cursor The cursor value.
|
||||
*/
|
||||
export function encodeCursor<T>(cursor: T | undefined): string | undefined {
|
||||
if (!cursor) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return Buffer.from(JSON.stringify(cursor)).toString('base64')
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
/**
|
||||
* Normalize limit and offset values extracted from the query string.
|
||||
*
|
||||
* @param query Parameter bag.
|
||||
* @param defaultLimit The limit to use if one is not supplied.
|
||||
*/
|
||||
export const extractLimitOffset = (
|
||||
{
|
||||
limit,
|
||||
offset,
|
||||
}: {
|
||||
/** The limit value extracted from the query string. */
|
||||
limit?: number
|
||||
/** The offset value extracted from the query string. */
|
||||
offset?: number
|
||||
},
|
||||
defaultLimit: number
|
||||
): { limit: number; offset: number } => ({ limit: limit || defaultLimit, offset: offset || 0 })
|
||||
@ -1,23 +0,0 @@
|
||||
import express from 'express'
|
||||
|
||||
/**
|
||||
* Create a link header payload with a next link based on the previous endpoint.
|
||||
*
|
||||
* @param req The HTTP request.
|
||||
* @param params The query params to overwrite.
|
||||
*/
|
||||
export function nextLink(
|
||||
req: express.Request,
|
||||
params: { [name: string]: string | number | boolean | undefined }
|
||||
): string {
|
||||
// Requests always have a host header
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
const url = new URL(`${req.protocol}://${req.get('host')!}${req.originalUrl}`)
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
if (value !== undefined) {
|
||||
url.searchParams.set(key, String(value))
|
||||
}
|
||||
}
|
||||
|
||||
return `<${url.href}>; rel="next"`
|
||||
}
|
||||
@ -1,116 +0,0 @@
|
||||
import * as json5 from 'json5'
|
||||
import * as settings from './settings'
|
||||
import got from 'got'
|
||||
import { isEqual, pick } from 'lodash'
|
||||
import { Logger } from 'winston'
|
||||
import delay from 'delay'
|
||||
|
||||
/** Service configuration data pulled from the frontend. */
|
||||
export interface Configuration {
|
||||
/** The connection string for the Postgres database. */
|
||||
postgresDSN: string
|
||||
|
||||
/** Whether or not to enable Jaeger. */
|
||||
useJaeger: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a configuration fetcher function and block until the first payload
|
||||
* can be read from the frontend. Continue reading the configuration from the
|
||||
* frontend in the background. If one of the fields that cannot be updated while
|
||||
* the process remains up changes, it will forcibly exit the process to allow
|
||||
* whatever orchestrator is running this process restart it.
|
||||
*
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export async function waitForConfiguration(logger: Logger): Promise<() => Configuration> {
|
||||
let oldConfiguration: Configuration | undefined
|
||||
|
||||
await new Promise<void>(resolve => {
|
||||
updateConfiguration(logger, configuration => {
|
||||
if (oldConfiguration !== undefined && requireRestart(oldConfiguration, configuration)) {
|
||||
logger.error('Detected configuration change, restarting to take effect')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
oldConfiguration = configuration
|
||||
resolve()
|
||||
}).catch(() => {
|
||||
/* noop */
|
||||
})
|
||||
})
|
||||
|
||||
// This value is guaranteed to be set by the resolution of the promise above
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
return () => oldConfiguration!
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if the two configurations differ by a field that cannot be changed
|
||||
* while the process remains up and a restart would be required for the change to
|
||||
* take effect.
|
||||
*
|
||||
* @param oldConfiguration The old configuration instance.
|
||||
* @param newConfiguration The new configuration instance.
|
||||
*/
|
||||
function requireRestart(oldConfiguration: Configuration, newConfiguration: Configuration): boolean {
|
||||
const fields = ['postgresDSN', 'useJaeger']
|
||||
return !isEqual(pick(oldConfiguration, fields), pick(newConfiguration, fields))
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the configuration from the frontend on a loop. This function is async but does not
|
||||
* return any meaningful value (the returned promise neither resolves nor rejects).
|
||||
*
|
||||
* @param logger The logger instance.
|
||||
* @param onChange The callback to invoke each time the configuration is read.
|
||||
*/
|
||||
async function updateConfiguration(logger: Logger, onChange: (configuration: Configuration) => void): Promise<never> {
|
||||
const start = Date.now()
|
||||
let previousError: Error | undefined
|
||||
while (true) {
|
||||
try {
|
||||
onChange(await loadConfiguration())
|
||||
|
||||
// Clear old error run
|
||||
previousError = undefined
|
||||
} catch (error) {
|
||||
const elapsed = Date.now() - start
|
||||
|
||||
if (
|
||||
// Do not keep reporting the same error
|
||||
error.message !== previousError?.message &&
|
||||
// Ignore all connection errors during startup period to allow the frontend to become reachable
|
||||
(error.code !== 'ECONNREFUSED' || elapsed >= settings.DELAY_BEFORE_UNREACHABLE_LOG * 1000)
|
||||
) {
|
||||
logger.error(
|
||||
'Failed to retrieve configuration from frontend',
|
||||
error.code !== 'ECONNREFUSED' ? error : error.message
|
||||
)
|
||||
|
||||
previousError = error
|
||||
}
|
||||
}
|
||||
|
||||
// Do a jittery sleep _up to_ the config poll interval.
|
||||
const durationMs = Math.random() * settings.CONFIG_POLL_INTERVAL * 1000
|
||||
await delay(durationMs)
|
||||
}
|
||||
}
|
||||
|
||||
/** Read configuration from the frontend. */
|
||||
async function loadConfiguration(): Promise<Configuration> {
|
||||
const url = new URL(`http://${settings.SRC_FRONTEND_INTERNAL}/.internal/configuration`).href
|
||||
const resp = await got.post(url, { followRedirect: true })
|
||||
const payload = JSON.parse(resp.body)
|
||||
|
||||
// Already parsed
|
||||
const serviceConnections = payload.ServiceConnections
|
||||
// Need to parse but must support comments + trailing commas
|
||||
const site = json5.parse(payload.Site)
|
||||
|
||||
return {
|
||||
postgresDSN: serviceConnections.postgresDSN,
|
||||
useJaeger: site.useJaeger || false,
|
||||
}
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
import { readEnvInt } from '../settings'
|
||||
|
||||
/** HTTP address for internal frontend HTTP API. */
|
||||
export const SRC_FRONTEND_INTERNAL = process.env.SRC_FRONTEND_INTERNAL || 'sourcegraph-frontend-internal'
|
||||
|
||||
/**
|
||||
* How long to wait before emitting error logs when polling config (in seconds).
|
||||
* This needs to be long enough to allow the frontend to fully migrate the PostgreSQL
|
||||
* database in most cases, to avoid log spam when running sourcegraph/server for the
|
||||
* first time.
|
||||
*/
|
||||
export const DELAY_BEFORE_UNREACHABLE_LOG = readEnvInt('DELAY_BEFORE_UNREACHABLE_LOG', 15)
|
||||
|
||||
/** How long to wait between polling config. */
|
||||
export const CONFIG_POLL_INTERVAL = 5
|
||||
@ -1,27 +0,0 @@
|
||||
/** The directory relative to the storage where SQLite databases are located. */
|
||||
export const DBS_DIR = 'dbs'
|
||||
|
||||
/** The directory relative to the storage where raw dumps are uploaded. */
|
||||
export const UPLOADS_DIR = 'uploads'
|
||||
|
||||
/** The maximum number of rows to bulk insert in Postgres. */
|
||||
export const MAX_POSTGRES_BATCH_SIZE = 5000
|
||||
|
||||
/**
|
||||
* The maximum number of commits to visit breadth-first style when when finding
|
||||
* the closest commit.
|
||||
*/
|
||||
export const MAX_TRAVERSAL_LIMIT = 100
|
||||
|
||||
/** A random integer specific to the Postgres database used to generate advisory lock ids. */
|
||||
export const ADVISORY_LOCK_ID_SALT = 1688730858
|
||||
|
||||
/**
|
||||
* The number of commits to ask gitserver for when updating commit data for
|
||||
* a particular repository. This should be just slightly above the max traversal
|
||||
* limit.
|
||||
*/
|
||||
export const MAX_COMMITS_PER_UPDATE = MAX_TRAVERSAL_LIMIT * 1.5
|
||||
|
||||
/** The number of remote dumps we will query per page of reference results. */
|
||||
export const DEFAULT_REFERENCES_REMOTE_DUMP_LIMIT = 20
|
||||
@ -1,90 +0,0 @@
|
||||
import promClient from 'prom-client'
|
||||
import { EntityManager } from 'typeorm'
|
||||
import { instrument } from '../metrics'
|
||||
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
|
||||
|
||||
/**
|
||||
* A bag of prometheus metric objects that apply to a particular instance of
|
||||
* `TableInserter`.
|
||||
*/
|
||||
interface TableInserterMetrics {
|
||||
/** A histogram that is observed on each round-trip to the database. */
|
||||
durationHistogram: promClient.Histogram<string>
|
||||
|
||||
/** A counter that increments on each error that occurs during an insertion. */
|
||||
errorsCounter: promClient.Counter<string>
|
||||
}
|
||||
|
||||
/**
|
||||
* A batch inserter for a SQLite table. Inserting hundreds or thousands of rows in
|
||||
* a loop is too inefficient, but due to the limit of SQLITE_MAX_VARIABLE_NUMBER,
|
||||
* the entire set of values cannot be inserted in one bulk operation either.
|
||||
*
|
||||
* One inserter instance is created for each table that will receive a bulk
|
||||
* payload. The inserter will periodically perform the insert operation
|
||||
* when the number of values is at this maximum.
|
||||
*
|
||||
* See https://www.sqlite.org/limits.html#max_variable_number.
|
||||
*/
|
||||
export class TableInserter<T, M extends new () => T> {
|
||||
/** The set of entity values that will be inserted in the next invocation of `executeBatch`. */
|
||||
private batch: QueryDeepPartialEntity<T>[] = []
|
||||
|
||||
/**
|
||||
* Creates a new `TableInserter` with the given entity manager, the constructor
|
||||
* of the model object for the table, and the maximum batch size. This number
|
||||
* should be calculated by floor(MAX_VAR_NUMBER / fields_in_record).
|
||||
*
|
||||
* @param entityManager A transactional SQLite entity manager.
|
||||
* @param model The model object constructor.
|
||||
* @param maxBatchSize The maximum number of records that can be inserted at once.
|
||||
* @param metrics The bag of metrics to use for this instance of the inserter.
|
||||
* @param ignoreConflicts Whether or not to ignore conflicting data on unique constraint violations.
|
||||
*/
|
||||
constructor(
|
||||
private entityManager: EntityManager,
|
||||
private model: M,
|
||||
private maxBatchSize: number,
|
||||
private metrics: TableInserterMetrics,
|
||||
private ignoreConflicts: boolean = false
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Submit a model for insertion. This may happen immediately, on a
|
||||
* subsequent call to insert, or when the `flush` method is called.
|
||||
*
|
||||
* @param model The instance to save.
|
||||
*/
|
||||
public async insert(model: QueryDeepPartialEntity<T>): Promise<void> {
|
||||
this.batch.push(model)
|
||||
|
||||
if (this.batch.length >= this.maxBatchSize) {
|
||||
await this.executeBatch()
|
||||
}
|
||||
}
|
||||
|
||||
/** Ensure any outstanding records are inserted into the database. */
|
||||
public flush(): Promise<void> {
|
||||
return this.executeBatch()
|
||||
}
|
||||
|
||||
/**
|
||||
* If the current batch is non-empty, then perform an insert operation
|
||||
* and reset the batch array.
|
||||
*/
|
||||
private async executeBatch(): Promise<void> {
|
||||
if (this.batch.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
let query = this.entityManager.createQueryBuilder().insert().into(this.model).values(this.batch)
|
||||
|
||||
if (this.ignoreConflicts) {
|
||||
query = query.onConflict('do nothing')
|
||||
}
|
||||
|
||||
await instrument(this.metrics.durationHistogram, this.metrics.errorsCounter, () => query.execute())
|
||||
|
||||
this.batch = []
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
import { Logger as TypeORMLogger } from 'typeorm'
|
||||
import { PlatformTools } from 'typeorm/platform/PlatformTools'
|
||||
import { Logger as WinstonLogger } from 'winston'
|
||||
|
||||
/**
|
||||
* A custom TypeORM logger that only logs slow database queries.
|
||||
*
|
||||
* We had previously set the TypeORM logging config to `['warn', 'error']`.
|
||||
* This caused some issues as it would print the entire parameter set for large
|
||||
* batch inserts to stdout. These parameters often included gzipped json payloads,
|
||||
* which would lock up the terminal where the server was running.
|
||||
*
|
||||
* This logger will only print slow database queries. Any other error condition
|
||||
* will be printed with the query, parameters, and underlying constraint violation
|
||||
* as part of a thrown error, making it unnecessary to log here.
|
||||
*/
|
||||
export class DatabaseLogger implements TypeORMLogger {
|
||||
constructor(private logger: WinstonLogger) {}
|
||||
|
||||
public logQuerySlow(time: number, query: string, parameters?: unknown[]): void {
|
||||
this.logger.warn('Slow database query', {
|
||||
query: PlatformTools.highlightSql(query),
|
||||
parameters,
|
||||
executionTime: time,
|
||||
})
|
||||
}
|
||||
|
||||
public logQuery(): void {
|
||||
/* no-op */
|
||||
}
|
||||
|
||||
public logSchemaBuild(): void {
|
||||
/* no-op */
|
||||
}
|
||||
|
||||
public logMigration(): void {
|
||||
/* no-op */
|
||||
}
|
||||
|
||||
public logQueryError(): void {
|
||||
/* no-op */
|
||||
}
|
||||
|
||||
public log(): void {
|
||||
/* no-op */
|
||||
}
|
||||
}
|
||||
@ -1,29 +0,0 @@
|
||||
import promClient from 'prom-client'
|
||||
|
||||
//
|
||||
// Query Metrics
|
||||
|
||||
export const postgresQueryDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_xrepo_query_duration_seconds',
|
||||
help: 'Total time spent on Postgres database queries.',
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
export const postgresQueryErrorsCounter = new promClient.Counter({
|
||||
name: 'lsif_xrepo_query_errors_total',
|
||||
help: 'The number of errors that occurred during a Postgres database query.',
|
||||
})
|
||||
|
||||
//
|
||||
// Insertion Metrics
|
||||
|
||||
export const postgresInsertionDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_xrepo_insertion_duration_seconds',
|
||||
help: 'Total time spent on Postgres database insertions.',
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
export const postgresInsertionErrorsCounter = new promClient.Counter({
|
||||
name: 'lsif_xrepo_insertion_errors_total',
|
||||
help: 'The number of errors that occurred during a Postgres database insertion.',
|
||||
})
|
||||
@ -1,191 +0,0 @@
|
||||
import * as metrics from './metrics'
|
||||
import * as pgModels from '../models/pg'
|
||||
import pRetry from 'p-retry'
|
||||
import { Configuration } from '../config/config'
|
||||
import { Connection, createConnection as _createConnection, EntityManager } from 'typeorm'
|
||||
import { instrument } from '../metrics'
|
||||
import { Logger } from 'winston'
|
||||
import { PostgresConnectionCredentialsOptions } from 'typeorm/driver/postgres/PostgresConnectionCredentialsOptions'
|
||||
import { TlsOptions } from 'tls'
|
||||
import { DatabaseLogger } from './logger'
|
||||
import * as settings from './settings'
|
||||
|
||||
/**
|
||||
* The minimum migration version required by this instance of the LSIF process.
|
||||
* This should be updated any time a new LSIF migration is added to the migrations/
|
||||
* directory, as we watch the DB to ensure we're on at least this version prior to
|
||||
* making use of the DB (which the frontend may still be migrating).
|
||||
*/
|
||||
const MINIMUM_MIGRATION_VERSION = 1528395671
|
||||
|
||||
/**
|
||||
* Create a Postgres connection. This creates a typorm connection pool with
|
||||
* the name `lsif`. The connection configuration is constructed by the method
|
||||
* `createPostgresConnectionOptions`. This method blocks until the connection
|
||||
* is established, then blocks indefinitely while the database migration state
|
||||
* is behind the expected minimum, or dirty. If a connection is not made within
|
||||
* a configurable timeout, an exception is thrown.
|
||||
*
|
||||
* @param configuration The current configuration.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export async function createPostgresConnection(configuration: Configuration, logger: Logger): Promise<Connection> {
|
||||
// Parse current PostgresDSN into connection options usable by
|
||||
// the typeorm postgres adapter.
|
||||
const url = new URL(configuration.postgresDSN)
|
||||
|
||||
// TODO(efritz) - handle allow, prefer, require, 'verify-ca', and 'verify-full'
|
||||
const sslModes: { [name: string]: boolean | TlsOptions } = {
|
||||
disable: false,
|
||||
}
|
||||
|
||||
const host = url.hostname
|
||||
const port = parseInt(url.port, 10) || 5432
|
||||
const username = decodeURIComponent(url.username)
|
||||
const password = decodeURIComponent(url.password)
|
||||
const database = decodeURIComponent(url.pathname).substring(1) || username
|
||||
const sslMode = url.searchParams.get('sslmode')
|
||||
const ssl = sslMode ? sslModes[sslMode] : undefined
|
||||
|
||||
// Get a working connection
|
||||
const connection = await connect({ host, port, username, password, database, ssl }, logger)
|
||||
|
||||
// Poll the schema migrations table until we are up to date
|
||||
await waitForMigrations(connection, logger)
|
||||
|
||||
return connection
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a connection to Postgres. This will re-attempt to access the database while
|
||||
* the database does not exist. This is to give some time to the frontend to run the
|
||||
* migrations that create the LSIF tables. The retry interval and attempt count can
|
||||
* be tuned via `MAX_CONNECTION_RETRIES` and `CONNECTION_RETRY_INTERVAL` environment
|
||||
* variables.
|
||||
*
|
||||
* @param connectionOptions The connection options.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
function connect(connectionOptions: PostgresConnectionCredentialsOptions, logger: Logger): Promise<Connection> {
|
||||
return pRetry(
|
||||
() => {
|
||||
logger.debug('Connecting to Postgres')
|
||||
return connectPostgres(connectionOptions, '', logger)
|
||||
},
|
||||
{
|
||||
factor: 1.5,
|
||||
randomize: true,
|
||||
retries: settings.MAX_CONNECTION_RETRIES,
|
||||
minTimeout: settings.MIN_CONNECTION_RETRY_TIMEOUT * 1000,
|
||||
maxTimeout: settings.MAX_CONNECTION_RETRY_TIMEOUT * 1000,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a connection to Postgres.
|
||||
*
|
||||
* @param connectionOptions The connection options.
|
||||
* @param suffix The database suffix (used for testing).
|
||||
* @param logger The logger instance
|
||||
*/
|
||||
export function connectPostgres(
|
||||
connectionOptions: PostgresConnectionCredentialsOptions,
|
||||
suffix: string,
|
||||
logger: Logger
|
||||
): Promise<Connection> {
|
||||
return _createConnection({
|
||||
type: 'postgres',
|
||||
name: `lsif${suffix}`,
|
||||
entities: pgModels.entities,
|
||||
logger: new DatabaseLogger(logger),
|
||||
maxQueryExecutionTime: 1000,
|
||||
...connectionOptions,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Block until we can select a migration version from the database that is at
|
||||
* least as large as our minimum migration version.
|
||||
*
|
||||
* @param connection The connection to use.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
function waitForMigrations(connection: Connection, logger: Logger): Promise<void> {
|
||||
const check = async (): Promise<void> => {
|
||||
logger.debug('Checking database version', { requiredVersion: MINIMUM_MIGRATION_VERSION })
|
||||
|
||||
const version = parseInt(await getMigrationVersion(connection), 10)
|
||||
if (isNaN(version) || version < MINIMUM_MIGRATION_VERSION) {
|
||||
throw new Error('Postgres database not up to date')
|
||||
}
|
||||
}
|
||||
|
||||
return pRetry(check, {
|
||||
factor: 1,
|
||||
retries: settings.MAX_SCHEMA_POLL_RETRIES,
|
||||
minTimeout: settings.SCHEMA_POLL_INTERVAL * 1000,
|
||||
maxTimeout: settings.SCHEMA_POLL_INTERVAL * 1000,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the current migration version from the frontend database. Throws on query
|
||||
* error, if no migration version can be found, or if the current migration state
|
||||
* is dirty.
|
||||
*
|
||||
* @param connection The database connection.
|
||||
*/
|
||||
async function getMigrationVersion(connection: Connection): Promise<string> {
|
||||
const rows: {
|
||||
version: string
|
||||
dirty: boolean
|
||||
}[] = await connection.query('select * from schema_migrations')
|
||||
|
||||
if (rows.length > 0 && !rows[0].dirty) {
|
||||
return rows[0].version
|
||||
}
|
||||
|
||||
throw new Error('Unusable migration state.')
|
||||
}
|
||||
|
||||
/**
|
||||
* Instrument `callback` with Postgres query histogram and error counter.
|
||||
*
|
||||
* @param callback The function invoke with the connection.
|
||||
*/
|
||||
export function instrumentQuery<T>(callback: () => Promise<T>): Promise<T> {
|
||||
return instrument(metrics.postgresQueryDurationHistogram, metrics.postgresQueryErrorsCounter, callback)
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke `callback` with a transactional Postgres entity manager created
|
||||
* from the wrapped connection.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param callback The function invoke with the entity manager.
|
||||
*/
|
||||
export function withInstrumentedTransaction<T>(
|
||||
connection: Connection,
|
||||
callback: (connection: EntityManager) => Promise<T>
|
||||
): Promise<T> {
|
||||
return instrumentQuery(() => connection.transaction(callback))
|
||||
}
|
||||
|
||||
/**
|
||||
* Invokes the callback wrapped in instrumentQuery with the given entityManager, if supplied,
|
||||
* and runs the callback in a transaction with a fresh entityManager otherwise.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param entityManager The EntityManager to use as part of a transaction.
|
||||
* @param callback The function invoke with the entity manager.
|
||||
*/
|
||||
export function instrumentQueryOrTransaction<T>(
|
||||
connection: Connection,
|
||||
entityManager: EntityManager | undefined,
|
||||
callback: (connection: EntityManager) => Promise<T>
|
||||
): Promise<T> {
|
||||
return entityManager
|
||||
? instrumentQuery(() => callback(entityManager))
|
||||
: withInstrumentedTransaction(connection, callback)
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
import { readEnvInt } from '../settings'
|
||||
|
||||
/** How many times to try to check the current database migration version on startup. */
|
||||
export const MAX_SCHEMA_POLL_RETRIES = readEnvInt('MAX_SCHEMA_POLL_RETRIES', 60)
|
||||
|
||||
/** How long to wait (in seconds) between queries to check the current database migration version on startup. */
|
||||
export const SCHEMA_POLL_INTERVAL = readEnvInt('SCHEMA_POLL_INTERVAL', 5)
|
||||
|
||||
/** How many times to try to connect to Postgres on startup. */
|
||||
export const MAX_CONNECTION_RETRIES = readEnvInt('MAX_CONNECTION_RETRIES', 60)
|
||||
|
||||
/** How long to wait (minimum, in seconds) between Postgres connection attempts. */
|
||||
export const MIN_CONNECTION_RETRY_TIMEOUT = readEnvInt('MIN_CONNECTION_RETRY_TIMEOUT', 1)
|
||||
|
||||
/** How long to wait (maximum, in seconds) between Postgres connection attempts. */
|
||||
export const MAX_CONNECTION_RETRY_TIMEOUT = readEnvInt('MAX_CONNECTION_RETRY_TIMEOUT', 1)
|
||||
@ -1,28 +0,0 @@
|
||||
import { Connection, createConnection as _createConnection } from 'typeorm'
|
||||
import { Logger } from 'winston'
|
||||
import { DatabaseLogger } from './logger'
|
||||
|
||||
/**
|
||||
* Create a SQLite connection from the given filename.
|
||||
*
|
||||
* @param database The database filename.
|
||||
* @param entities The set of expected entities present in this schema.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
export function createSqliteConnection(
|
||||
database: string,
|
||||
// Decorators are not possible type check
|
||||
// eslint-disable-next-line @typescript-eslint/ban-types
|
||||
entities: Function[],
|
||||
logger: Logger
|
||||
): Promise<Connection> {
|
||||
return _createConnection({
|
||||
type: 'sqlite',
|
||||
name: database,
|
||||
database,
|
||||
entities,
|
||||
synchronize: true,
|
||||
logger: new DatabaseLogger(logger),
|
||||
maxQueryExecutionTime: 1000,
|
||||
})
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
import { createFilter, testFilter } from './bloom-filter'
|
||||
|
||||
describe('testFilter', () => {
|
||||
it('should test set membership', async () => {
|
||||
const filter = await createFilter(['foo', 'bar', 'baz'])
|
||||
expect(await testFilter(filter, 'foo')).toBeTruthy()
|
||||
expect(await testFilter(filter, 'bar')).toBeTruthy()
|
||||
expect(await testFilter(filter, 'baz')).toBeTruthy()
|
||||
expect(await testFilter(filter, 'bonk')).toBeFalsy()
|
||||
expect(await testFilter(filter, 'quux')).toBeFalsy()
|
||||
})
|
||||
})
|
||||
@ -1,39 +0,0 @@
|
||||
import { BloomFilter } from 'bloomfilter'
|
||||
import { gunzipJSON, gzipJSON } from '../encoding/json'
|
||||
import * as settings from './settings'
|
||||
|
||||
/** A type that describes a the encoded version of a bloom filter. */
|
||||
export type EncodedBloomFilter = Buffer
|
||||
|
||||
/**
|
||||
* Create a bloom filter containing the given values and return an encoded verion.
|
||||
*
|
||||
* @param values The values to add to the bloom filter.
|
||||
*/
|
||||
export function createFilter(values: string[]): Promise<EncodedBloomFilter> {
|
||||
const filter = new BloomFilter(settings.BLOOM_FILTER_BITS, settings.BLOOM_FILTER_NUM_HASH_FUNCTIONS)
|
||||
for (const value of values) {
|
||||
filter.add(value)
|
||||
}
|
||||
|
||||
// Need to shed the type of the array
|
||||
const buckets = Array.from(filter.buckets)
|
||||
|
||||
// Store the number of hash functions used to create this as it may change after
|
||||
// this value is serialized. We don't want to test with more hash functions than
|
||||
// it was created with, otherwise we'll get false negatives.
|
||||
return gzipJSON({ numHashFunctions: settings.BLOOM_FILTER_NUM_HASH_FUNCTIONS, buckets })
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode `filter` as created by `createFilter` and determine if `value` is a
|
||||
* possible element. This may return a false positive (returning true if the
|
||||
* element is not actually a member), but will not return false negatives.
|
||||
*
|
||||
* @param filter The encoded filter.
|
||||
* @param value The value to test membership.
|
||||
*/
|
||||
export async function testFilter(filter: EncodedBloomFilter, value: string): Promise<boolean> {
|
||||
const { numHashFunctions, buckets } = await gunzipJSON(filter)
|
||||
return new BloomFilter(buckets, numHashFunctions).test(value)
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
import { DefaultMap } from './default-map'
|
||||
|
||||
describe('DefaultMap', () => {
|
||||
it('should leave get unchanged', () => {
|
||||
const map = new DefaultMap<string, string>(() => 'bar')
|
||||
expect(map.get('foo')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should create values on access', () => {
|
||||
const map = new DefaultMap<string, string>(() => 'bar')
|
||||
expect(map.getOrDefault('foo')).toEqual('bar')
|
||||
})
|
||||
|
||||
it('should respect explicit set', () => {
|
||||
const map = new DefaultMap<string, string>(() => 'bar')
|
||||
map.set('foo', 'baz')
|
||||
expect(map.getOrDefault('foo')).toEqual('baz')
|
||||
})
|
||||
|
||||
it('should support nested gets', () => {
|
||||
const map = new DefaultMap<string, DefaultMap<string, string[]>>(
|
||||
() => new DefaultMap<string, string[]>(() => [])
|
||||
)
|
||||
|
||||
map.getOrDefault('foo').getOrDefault('bar').push('baz')
|
||||
|
||||
map.getOrDefault('foo').getOrDefault('bar').push('bonk')
|
||||
|
||||
const inner = map.get('foo')
|
||||
expect(inner?.get('bar')).toEqual(['baz', 'bonk'])
|
||||
})
|
||||
})
|
||||
@ -1,32 +0,0 @@
|
||||
/**
|
||||
* An extension of `Map` that defines `getOrDefault` for a type of stunted
|
||||
* autovivification. This saves a bunch of code that needs to check if a
|
||||
* nested type within a map is undefined on first access.
|
||||
*/
|
||||
export class DefaultMap<K, V> extends Map<K, V> {
|
||||
/**
|
||||
* Returns a new `DefaultMap`.
|
||||
*
|
||||
* @param defaultFactory The factory invoked when an undefined value is accessed.
|
||||
*/
|
||||
constructor(private defaultFactory: () => V) {
|
||||
super()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a key from the map. If the key does not exist, the default factory produces
|
||||
* a value and inserted into the map before being returned.
|
||||
*
|
||||
* @param key The key to retrieve.
|
||||
*/
|
||||
public getOrDefault(key: K): V {
|
||||
let value = super.get(key)
|
||||
if (value !== undefined) {
|
||||
return value
|
||||
}
|
||||
|
||||
value = this.defaultFactory()
|
||||
this.set(key, value)
|
||||
return value
|
||||
}
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
import { DisjointSet } from './disjoint-set'
|
||||
|
||||
describe('DisjointSet', () => {
|
||||
it('should traverse relations in both directions', () => {
|
||||
const set = new DisjointSet<number>()
|
||||
set.union(1, 2)
|
||||
set.union(3, 4)
|
||||
set.union(1, 3)
|
||||
set.union(5, 6)
|
||||
|
||||
expect(set.extractSet(1)).toEqual(new Set([1, 2, 3, 4]))
|
||||
expect(set.extractSet(2)).toEqual(new Set([1, 2, 3, 4]))
|
||||
expect(set.extractSet(3)).toEqual(new Set([1, 2, 3, 4]))
|
||||
expect(set.extractSet(4)).toEqual(new Set([1, 2, 3, 4]))
|
||||
expect(set.extractSet(5)).toEqual(new Set([5, 6]))
|
||||
expect(set.extractSet(6)).toEqual(new Set([5, 6]))
|
||||
})
|
||||
})
|
||||
@ -1,52 +0,0 @@
|
||||
import { DefaultMap } from './default-map'
|
||||
|
||||
/**
|
||||
* A modified disjoint set or union-find data structure. Allows linking
|
||||
* items together and retrieving the set of all items for a given set.
|
||||
*/
|
||||
export class DisjointSet<T> {
|
||||
/** For every linked value `v1` and `v2`, `v2` in `links[v1]` and `v1` in `links[v2]`. */
|
||||
private links = new DefaultMap<T, Set<T>>(() => new Set())
|
||||
|
||||
/** Return an iterator of all elements in the set. */
|
||||
public keys(): IterableIterator<T> {
|
||||
return this.links.keys()
|
||||
}
|
||||
|
||||
/**
|
||||
* Link two values into the same set. If one or the other value is
|
||||
* already in the set, then the sets of the two values will merge.
|
||||
*
|
||||
* @param v1 One linked value.
|
||||
* @param v2 The other linked value.
|
||||
*/
|
||||
public union(v1: T, v2: T): void {
|
||||
this.links.getOrDefault(v1).add(v2)
|
||||
this.links.getOrDefault(v2).add(v1)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the values in the same set as the given source value.
|
||||
*
|
||||
* @param v The source value.
|
||||
*/
|
||||
public extractSet(v: T): Set<T> {
|
||||
const set = new Set<T>()
|
||||
|
||||
let frontier = [v]
|
||||
while (frontier.length > 0) {
|
||||
const val = frontier.pop()
|
||||
if (val === undefined) {
|
||||
// hol up, frontier was non-empty!
|
||||
throw new Error('Impossible condition.')
|
||||
}
|
||||
|
||||
if (!set.has(val)) {
|
||||
set.add(val)
|
||||
frontier = frontier.concat(Array.from(this.links.get(val) || []))
|
||||
}
|
||||
}
|
||||
|
||||
return set
|
||||
}
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
import { OrderedSet } from './orderedset'
|
||||
|
||||
describe('OrderedSet', () => {
|
||||
it('should not contain duplicates', () => {
|
||||
const set = new OrderedSet<string>(value => value)
|
||||
set.push('foo')
|
||||
set.push('foo')
|
||||
set.push('bar')
|
||||
set.push('bar')
|
||||
|
||||
expect(set.values).toEqual(['foo', 'bar'])
|
||||
})
|
||||
|
||||
it('should retain insertion order', () => {
|
||||
const set = new OrderedSet<string>(value => value)
|
||||
set.push('bonk')
|
||||
set.push('baz')
|
||||
set.push('foo')
|
||||
set.push('bar')
|
||||
set.push('bar')
|
||||
set.push('baz')
|
||||
set.push('foo')
|
||||
set.push('bonk')
|
||||
|
||||
expect(set.values).toEqual(['bonk', 'baz', 'foo', 'bar'])
|
||||
})
|
||||
})
|
||||
@ -1,30 +0,0 @@
|
||||
/** A set of values that maintains insertion order. */
|
||||
export class OrderedSet<T> {
|
||||
private set = new Map<string, T>()
|
||||
|
||||
/**
|
||||
* Create a new ordered set.
|
||||
*
|
||||
* @param makeKey The function to generate a unique string key from a value.
|
||||
* @param values A set of values used to seed the set.
|
||||
*/
|
||||
constructor(private makeKey: (value: T) => string, values?: T[]) {
|
||||
if (!values) {
|
||||
return
|
||||
}
|
||||
|
||||
for (const value of values) {
|
||||
this.push(value)
|
||||
}
|
||||
}
|
||||
|
||||
/** The deduplicated values in insertion order. */
|
||||
public get values(): T[] {
|
||||
return Array.from(this.set.values())
|
||||
}
|
||||
|
||||
/** Insert a value into the set if it hasn't been seen before. */
|
||||
public push(value: T): void {
|
||||
this.set.set(this.makeKey(value), value)
|
||||
}
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
import { readEnvInt } from '../settings'
|
||||
|
||||
// These parameters give us a 1 in 1.38x10^9 false positive rate if we assume
|
||||
// that the number of unique URIs referrable by an external package is of the
|
||||
// order of 10k (....but I have no idea if that is a reasonable estimate....).
|
||||
//
|
||||
// See the following link for a bloom calculator: https://hur.st/bloomfilter
|
||||
|
||||
/** The number of bits allocated for new bloom filters. */
|
||||
export const BLOOM_FILTER_BITS = readEnvInt('BLOOM_FILTER_BITS', 64 * 1024)
|
||||
|
||||
/** The number of hash functions to use to determine if a value is a member of the filter. */
|
||||
export const BLOOM_FILTER_NUM_HASH_FUNCTIONS = readEnvInt('BLOOM_FILTER_NUM_HASH_FUNCTIONS', 16)
|
||||
@ -1,35 +0,0 @@
|
||||
import { gunzipJSON, gzipJSON } from './json'
|
||||
|
||||
describe('gzipJSON', () => {
|
||||
it('should preserve maps', async () => {
|
||||
const m = new Map<string, number>([
|
||||
['a', 1],
|
||||
['b', 2],
|
||||
['c', 3],
|
||||
])
|
||||
|
||||
const value = {
|
||||
foo: [1, 2, 3],
|
||||
bar: ['abc', 'xyz'],
|
||||
baz: m,
|
||||
}
|
||||
|
||||
const encoded = await gzipJSON(value)
|
||||
const decoded = await gunzipJSON(encoded)
|
||||
expect(decoded).toEqual(value)
|
||||
})
|
||||
|
||||
it('should preserve sets', async () => {
|
||||
const s = new Set<number>([1, 2, 3, 4, 5])
|
||||
|
||||
const value = {
|
||||
foo: [1, 2, 3],
|
||||
bar: ['abc', 'xyz'],
|
||||
baz: s,
|
||||
}
|
||||
|
||||
const encoded = await gzipJSON(value)
|
||||
const decoded = await gunzipJSON(encoded)
|
||||
expect(decoded).toEqual(value)
|
||||
})
|
||||
})
|
||||
@ -1,72 +0,0 @@
|
||||
import { gunzip, gzip } from 'mz/zlib'
|
||||
|
||||
/**
|
||||
* Return the gzipped JSON representation of `value`.
|
||||
*
|
||||
* @param value The value to encode.
|
||||
*/
|
||||
export function gzipJSON<T>(value: T): Promise<Buffer> {
|
||||
return gzip(Buffer.from(dumpJSON(value)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the operation of `gzipJSON`.
|
||||
*
|
||||
* @param value The value to decode.
|
||||
*/
|
||||
export async function gunzipJSON<T>(value: Buffer): Promise<T> {
|
||||
return parseJSON((await gunzip(value)).toString())
|
||||
}
|
||||
|
||||
/** The replacer used by dumpJSON to encode map and set values. */
|
||||
export const jsonReplacer = <T>(key: string, value: T): { type: string; value: unknown } | T => {
|
||||
if (value instanceof Map) {
|
||||
return {
|
||||
type: 'map',
|
||||
value: [...value],
|
||||
}
|
||||
}
|
||||
|
||||
if (value instanceof Set) {
|
||||
return {
|
||||
type: 'set',
|
||||
value: [...value],
|
||||
}
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the JSON representation of `value`. This has special logic to
|
||||
* convert ES6 map and set structures into a JSON-representable value.
|
||||
* This method, along with `parseJSON` should be used over the raw methods
|
||||
* if the payload may contain maps.
|
||||
*
|
||||
* @param value The value to jsonify.
|
||||
*/
|
||||
export function dumpJSON<T>(value: T): string {
|
||||
return JSON.stringify(value, jsonReplacer)
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the JSON representation of `value`. This has special logic to
|
||||
* unmarshal map and set objects as encoded by `dumpJSON`.
|
||||
*
|
||||
* @param value The value to unmarshal.
|
||||
*/
|
||||
export function parseJSON<T>(value: string): T {
|
||||
return JSON.parse(value, (_, oldValue) => {
|
||||
if (typeof oldValue === 'object' && oldValue !== null) {
|
||||
if (oldValue.type === 'map') {
|
||||
return new Map(oldValue.value)
|
||||
}
|
||||
|
||||
if (oldValue.type === 'set') {
|
||||
return new Set(oldValue.value)
|
||||
}
|
||||
}
|
||||
|
||||
return oldValue
|
||||
})
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
import nock from 'nock'
|
||||
import { flattenCommitParents, getCommitsNear, getDirectoryChildren } from './gitserver'
|
||||
|
||||
describe('getDirectoryChildren', () => {
|
||||
it('should parse response from gitserver', async () => {
|
||||
nock('http://frontend')
|
||||
.post('/.internal/git/42/exec', {
|
||||
args: ['ls-tree', '--name-only', 'c', '--', '.', 'foo/', 'bar/baz/'],
|
||||
})
|
||||
.reply(200, 'a\nb\nbar/baz/x\nbar/baz/y\nc\nfoo/1\nfoo/2\nfoo/3\n')
|
||||
|
||||
expect(
|
||||
await getDirectoryChildren({
|
||||
frontendUrl: 'frontend',
|
||||
repositoryId: 42,
|
||||
commit: 'c',
|
||||
dirnames: ['', 'foo', 'bar/baz/'],
|
||||
})
|
||||
).toEqual(
|
||||
new Map([
|
||||
['', new Set(['a', 'b', 'c'])],
|
||||
['foo', new Set(['foo/1', 'foo/2', 'foo/3'])],
|
||||
['bar/baz/', new Set(['bar/baz/x', 'bar/baz/y'])],
|
||||
])
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getCommitsNear', () => {
|
||||
it('should parse response from gitserver', async () => {
|
||||
nock('http://frontend')
|
||||
.post('/.internal/git/42/exec', { args: ['log', '--pretty=%H %P', 'l', '-150'] })
|
||||
.reply(200, 'a\nb c\nd e f\ng h i j k l')
|
||||
|
||||
expect(await getCommitsNear('frontend', 42, 'l')).toEqual(
|
||||
new Map([
|
||||
['a', new Set()],
|
||||
['b', new Set(['c'])],
|
||||
['d', new Set(['e', 'f'])],
|
||||
['g', new Set(['h', 'i', 'j', 'k', 'l'])],
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it('should handle request for unknown repository', async () => {
|
||||
nock('http://frontend').post('/.internal/git/42/exec').reply(404)
|
||||
|
||||
expect(await getCommitsNear('frontend', 42, 'l')).toEqual(new Map())
|
||||
})
|
||||
})
|
||||
|
||||
describe('flattenCommitParents', () => {
|
||||
it('should handle multiple commits', () => {
|
||||
expect(flattenCommitParents(['a', 'b c', 'd e f', '', 'g h i j k l', 'm '])).toEqual(
|
||||
new Map([
|
||||
['a', new Set()],
|
||||
['b', new Set(['c'])],
|
||||
['d', new Set(['e', 'f'])],
|
||||
['g', new Set(['h', 'i', 'j', 'k', 'l'])],
|
||||
['m', new Set()],
|
||||
])
|
||||
)
|
||||
})
|
||||
})
|
||||
@ -1,197 +0,0 @@
|
||||
import got from 'got'
|
||||
import { MAX_COMMITS_PER_UPDATE } from '../constants'
|
||||
import { TracingContext, logAndTraceCall } from '../tracing'
|
||||
import { instrument } from '../metrics'
|
||||
import * as metrics from './metrics'
|
||||
|
||||
/**
|
||||
* Get the children of a set of directories at a particular commit. This function
|
||||
* returns a list of sets `L` where `L[i]` is the set of children for `dirnames[i]`.
|
||||
*
|
||||
* Except for the root directory, denoted by the empty string, all supplied
|
||||
* directories should be disjoint (one should not contain another).
|
||||
*
|
||||
* @param args Parameter bag.
|
||||
*/
|
||||
export async function getDirectoryChildren({
|
||||
frontendUrl,
|
||||
repositoryId,
|
||||
commit,
|
||||
dirnames,
|
||||
ctx = {},
|
||||
}: {
|
||||
/** The url of the frontend internal API. */
|
||||
frontendUrl: string
|
||||
/** The repository identifier. */
|
||||
repositoryId: number
|
||||
/** The commit from which the gitserver queries should start. */
|
||||
commit: string
|
||||
/** A list of repo-root-relative directories. */
|
||||
dirnames: string[]
|
||||
/** The tracing context. */
|
||||
ctx?: TracingContext
|
||||
}): Promise<Map<string, Set<string>>> {
|
||||
const args = ['ls-tree', '--name-only', commit, '--']
|
||||
|
||||
for (const dirname of dirnames) {
|
||||
if (dirname === '') {
|
||||
args.push('.')
|
||||
} else {
|
||||
args.push(dirname.endsWith('/') ? dirname : dirname + '/')
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve a flat list of children of the dirnames constructed above. This returns a flat
|
||||
// list sorted alphabetically, which we then need to partition by parent directory.
|
||||
const uncategorizedChildren = await gitserverExecLines(frontendUrl, repositoryId, args, ctx)
|
||||
|
||||
const childMap = new Map()
|
||||
for (const dirname of dirnames) {
|
||||
childMap.set(
|
||||
dirname,
|
||||
dirname === ''
|
||||
? new Set(uncategorizedChildren.filter(line => !line.includes('/')))
|
||||
: new Set(uncategorizedChildren.filter(line => line.startsWith(dirname)))
|
||||
)
|
||||
}
|
||||
|
||||
return childMap
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a list of commits for the given repository with their parent starting at the
|
||||
* given commit and returning at most `MAX_COMMITS_PER_UPDATE` commits. The output
|
||||
* is a map from commits to a set of parent commits. The set of parents may be empty.
|
||||
*
|
||||
* If the repository or commit is unknown by gitserver, then the the results will be
|
||||
* empty but no error will be thrown. Any other error type will be thrown without
|
||||
* modification.
|
||||
*
|
||||
* @param frontendUrl The url of the frontend internal API.
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param commit The commit from which the gitserver queries should start.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
export async function getCommitsNear(
|
||||
frontendUrl: string,
|
||||
repositoryId: number,
|
||||
commit: string,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<Map<string, Set<string>>> {
|
||||
const args = ['log', '--pretty=%H %P', commit, `-${MAX_COMMITS_PER_UPDATE}`]
|
||||
|
||||
try {
|
||||
return flattenCommitParents(await gitserverExecLines(frontendUrl, repositoryId, args, ctx))
|
||||
} catch (error) {
|
||||
if (error.response && error.response.statusCode === 404) {
|
||||
// Unknown repository
|
||||
return new Map()
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert git log output into a parentage map. Each line of the input should have the
|
||||
* form `commit p1 p2 p3...`, where commits without a parent appear on a line of their
|
||||
* own. The output is a map from commits a set of parent commits. The set of parents may
|
||||
* be empty.
|
||||
*
|
||||
* @param lines The output lines of `git log`.
|
||||
*/
|
||||
export function flattenCommitParents(lines: string[]): Map<string, Set<string>> {
|
||||
const commits = new Map()
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim()
|
||||
if (trimmed === '') {
|
||||
continue
|
||||
}
|
||||
|
||||
const [child, ...parentCommits] = trimmed.split(' ')
|
||||
commits.set(child, new Set<string>(parentCommits))
|
||||
}
|
||||
|
||||
return commits
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current tip of the default branch of the given repository.
|
||||
*
|
||||
* @param frontendUrl The url of the frontend internal API.
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
export async function getHead(
|
||||
frontendUrl: string,
|
||||
repositoryId: number,
|
||||
ctx: TracingContext = {}
|
||||
): Promise<string | undefined> {
|
||||
const lines = await gitserverExecLines(frontendUrl, repositoryId, ['rev-parse', 'HEAD'], ctx)
|
||||
if (lines.length === 0) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return lines[0]
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a git command via gitserver and return its output split into non-empty lines.
|
||||
*
|
||||
* @param frontendUrl The url of the frontend internal API.
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param args The command to run in the repository's git directory.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
export async function gitserverExecLines(
|
||||
frontendUrl: string,
|
||||
repositoryId: number,
|
||||
args: string[],
|
||||
ctx: TracingContext = {}
|
||||
): Promise<string[]> {
|
||||
return (await gitserverExec(frontendUrl, repositoryId, args, ctx)).split('\n').filter(line => Boolean(line))
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a git command via gitserver and return its raw output.
|
||||
*
|
||||
* @param frontendUrl The url of the frontend internal API.
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param args The command to run in the repository's git directory.
|
||||
* @param ctx The tracing context.
|
||||
*/
|
||||
function gitserverExec(
|
||||
frontendUrl: string,
|
||||
repositoryId: number,
|
||||
args: string[],
|
||||
ctx: TracingContext = {}
|
||||
): Promise<string> {
|
||||
if (args[0] === 'git') {
|
||||
// Prevent this from happening again:
|
||||
// https://github.com/sourcegraph/sourcegraph/pull/5941
|
||||
// https://github.com/sourcegraph/sourcegraph/pull/6548
|
||||
throw new Error('Gitserver commands should not be prefixed with `git`')
|
||||
}
|
||||
|
||||
return logAndTraceCall(ctx, 'Executing git command', () =>
|
||||
instrument(metrics.gitserverDurationHistogram, metrics.gitserverErrorsCounter, async () => {
|
||||
// Perform request - this may fail with a 404 or 500
|
||||
const resp = await got.post(new URL(`http://${frontendUrl}/.internal/git/${repositoryId}/exec`).href, {
|
||||
body: JSON.stringify({ args }),
|
||||
})
|
||||
|
||||
// Read trailers on a 200-level response
|
||||
const status = resp.trailers['x-exec-exit-status']
|
||||
const stderr = resp.trailers['x-exec-stderr']
|
||||
|
||||
// Determine if underlying git command failed and throw an error
|
||||
// in that case. Status will be undefined in some of our tests and
|
||||
// will be the process exit code (given as a string) otherwise.
|
||||
if (status !== undefined && status !== '0') {
|
||||
throw new Error(`Failed to run git command ${['git', ...args].join(' ')}: ${String(stderr)}`)
|
||||
}
|
||||
|
||||
return resp.body
|
||||
})
|
||||
)
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
import promClient from 'prom-client'
|
||||
|
||||
//
|
||||
// Gitserver Metrics
|
||||
|
||||
export const gitserverDurationHistogram = new promClient.Histogram({
|
||||
name: 'lsif_gitserver_duration_seconds',
|
||||
help: 'Total time spent on gitserver exec queries.',
|
||||
buckets: [0.2, 0.5, 1, 2, 5, 10, 30],
|
||||
})
|
||||
|
||||
export const gitserverErrorsCounter = new promClient.Counter({
|
||||
name: 'lsif_gitserver_errors_total',
|
||||
help: 'The number of errors that occurred during a gitserver exec query.',
|
||||
})
|
||||
@ -1,133 +0,0 @@
|
||||
import * as fs from 'mz/fs'
|
||||
import * as path from 'path'
|
||||
import * as zlib from 'mz/zlib'
|
||||
import rmfr from 'rmfr'
|
||||
import { parseJsonLines, readGzippedJsonElementsFromFile, splitLines } from './input'
|
||||
import { Readable } from 'stream'
|
||||
|
||||
describe('readGzippedJsonElements', () => {
|
||||
let tempPath!: string
|
||||
|
||||
beforeAll(async () => {
|
||||
tempPath = await fs.mkdtemp('test-', { encoding: 'utf8' })
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await rmfr(tempPath)
|
||||
})
|
||||
|
||||
it('should decode gzip', async () => {
|
||||
const lines = [
|
||||
{ type: 'vertex', label: 'project' },
|
||||
{ type: 'vertex', label: 'document' },
|
||||
{ type: 'edge', label: 'item' },
|
||||
{ type: 'edge', label: 'moniker' },
|
||||
]
|
||||
|
||||
const filename = path.join(tempPath, 'gzip.txt')
|
||||
|
||||
const chunks = []
|
||||
for await (const chunk of Readable.from(lines.map(l => JSON.stringify(l)).join('\n')).pipe(zlib.createGzip())) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
|
||||
await fs.writeFile(filename, Buffer.concat(chunks))
|
||||
|
||||
const elements: unknown[] = []
|
||||
for await (const element of readGzippedJsonElementsFromFile(filename)) {
|
||||
elements.push(element)
|
||||
}
|
||||
|
||||
expect(elements).toEqual(lines)
|
||||
})
|
||||
|
||||
it('should fail without gzip', async () => {
|
||||
const lines = [
|
||||
'{"type": "vertex", "label": "project"}',
|
||||
'{"type": "vertex", "label": "document"}',
|
||||
'{"type": "edge", "label": "item"}',
|
||||
'{"type": "edge", "label": "moniker"}',
|
||||
]
|
||||
|
||||
const filename = path.join(tempPath, 'nogzip.txt')
|
||||
await fs.writeFile(filename, lines.join('\n'))
|
||||
|
||||
await expect(consume(readGzippedJsonElementsFromFile(filename))).rejects.toThrowError(
|
||||
new Error('incorrect header check')
|
||||
)
|
||||
})
|
||||
|
||||
it('should throw an error on IO error', async () => {
|
||||
const filename = path.join(tempPath, 'missing.txt')
|
||||
|
||||
await expect(consume(readGzippedJsonElementsFromFile(filename))).rejects.toThrowError(
|
||||
new Error(`ENOENT: no such file or directory, open '${filename}'`)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('splitLines', () => {
|
||||
it('should split input by newline', async () => {
|
||||
const chunks = ['foo\n', 'bar', '\nbaz\n\nbonk\nqu', 'ux']
|
||||
|
||||
const lines: string[] = []
|
||||
for await (const line of splitLines(generate(chunks))) {
|
||||
lines.push(line)
|
||||
}
|
||||
|
||||
expect(lines).toEqual(['foo', 'bar', 'baz', '', 'bonk', 'quux'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseJsonLines', () => {
|
||||
it('should parse JSON', async () => {
|
||||
const lines = [
|
||||
{ type: 'vertex', label: 'project' },
|
||||
{ type: 'vertex', label: 'document' },
|
||||
{ type: 'edge', label: 'item' },
|
||||
{ type: 'edge', label: 'moniker' },
|
||||
]
|
||||
|
||||
const elements: unknown[] = []
|
||||
for await (const element of parseJsonLines(generate(lines.map(l => JSON.stringify(l))))) {
|
||||
elements.push(element)
|
||||
}
|
||||
|
||||
expect(elements).toEqual(lines)
|
||||
})
|
||||
|
||||
it('should wrap parse errors', async () => {
|
||||
const input = [
|
||||
'{"type": "vertex", "label": "project"}',
|
||||
'{"type": "vertex", "label": "document"}',
|
||||
'{"type": "edge", "label": "item"}',
|
||||
'{"type": "edge" "label": "moniker"}', // missing comma
|
||||
]
|
||||
|
||||
await expect(consume(parseJsonLines(generate(input)))).rejects.toThrowError(
|
||||
new Error(
|
||||
'Failed to process line #4 ({"type": "edge" "label": "moniker"}): Unexpected string in JSON at position 16'
|
||||
)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
//
|
||||
// Helpers
|
||||
|
||||
async function* generate<T>(values: T[]): AsyncIterable<T> {
|
||||
// Make it actually async
|
||||
await Promise.resolve()
|
||||
|
||||
for (const value of values) {
|
||||
yield value
|
||||
}
|
||||
}
|
||||
|
||||
async function consume(iterable: AsyncIterable<unknown>): Promise<void> {
|
||||
// We need to consume the iterable but can't make a meaningful
|
||||
// binding for each element of the iteration.
|
||||
for await (const _ of iterable) {
|
||||
// no-op body, just consume iterable
|
||||
}
|
||||
}
|
||||
@ -1,68 +0,0 @@
|
||||
import * as fs from 'mz/fs'
|
||||
import { createGunzip } from 'zlib'
|
||||
|
||||
/**
|
||||
* Yield parsed JSON elements from a file containing the gzipped JSON lines.
|
||||
*
|
||||
* @param path The filepath containing a gzipped compressed stream of JSON lines composing the LSIF dump.
|
||||
*/
|
||||
export function readGzippedJsonElementsFromFile(path: string): AsyncIterable<unknown> {
|
||||
const input = fs.createReadStream(path)
|
||||
const piped = input.pipe(createGunzip())
|
||||
|
||||
// Ensure we forward errors opening/reading the file to the async
|
||||
// iterator opened below.
|
||||
input.on('error', error => piped.emit('error', error))
|
||||
|
||||
// Create the iterable
|
||||
return parseJsonLines(splitLines(piped))
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform an async iterable into an async iterable of lines. Each value
|
||||
* is stripped of its trailing newline. Lines may be empty.
|
||||
*
|
||||
* @param input The input buffer.
|
||||
*/
|
||||
export async function* splitLines(input: AsyncIterable<string | Buffer>): AsyncIterable<string> {
|
||||
let buffer = ''
|
||||
for await (const data of input) {
|
||||
buffer += data.toString()
|
||||
|
||||
do {
|
||||
const index = buffer.indexOf('\n')
|
||||
if (index < 0) {
|
||||
break
|
||||
}
|
||||
|
||||
yield buffer.substring(0, index)
|
||||
buffer = buffer.substring(index + 1)
|
||||
} while (true)
|
||||
}
|
||||
|
||||
yield buffer
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a stream of uncompressed JSON strings and yields each parsed line.
|
||||
* Ignores empty lines. Throws an exception with line index and content when
|
||||
* a non-empty line is not valid JSON.
|
||||
*
|
||||
* @param lines An iterable of JSON lines.
|
||||
*/
|
||||
export async function* parseJsonLines(lines: AsyncIterable<string>): AsyncIterable<unknown> {
|
||||
let index = 0
|
||||
for await (const data of lines) {
|
||||
index++
|
||||
|
||||
if (!data) {
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
yield JSON.parse(data)
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to process line #${index} (${data}): ${String(error?.message)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,83 +0,0 @@
|
||||
import { createLogger as _createLogger, Logger, transports } from 'winston'
|
||||
import { format, TransformableInfo } from 'logform'
|
||||
import { inspect } from 'util'
|
||||
import { MESSAGE } from 'triple-beam'
|
||||
|
||||
/**
|
||||
* Return a sanitized log level.
|
||||
*
|
||||
* @param value The raw log level.
|
||||
*/
|
||||
function toLogLevel(value: string): 'debug' | 'info' | 'warn' | 'error' {
|
||||
if (value === 'debug' || value === 'info' || value === 'warn' || value === 'error') {
|
||||
return value
|
||||
}
|
||||
|
||||
return 'info'
|
||||
}
|
||||
|
||||
/** The maximum level log message to output. */
|
||||
const LOG_LEVEL: 'debug' | 'info' | 'warn' | 'error' = toLogLevel((process.env.LOG_LEVEL || 'info').toLowerCase())
|
||||
|
||||
/** Create a structured logger. */
|
||||
export function createLogger(service: string): Logger {
|
||||
const formatTransformer = (info: TransformableInfo): TransformableInfo => {
|
||||
const attributes: { [name: string]: unknown } = {}
|
||||
for (const [key, value] of Object.entries(info)) {
|
||||
if (key !== 'level' && key !== 'message') {
|
||||
attributes[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
info[MESSAGE] = `${info.level} ${info.message} ${inspect(attributes)}`
|
||||
return info
|
||||
}
|
||||
|
||||
const uppercaseTransformer = (info: TransformableInfo): TransformableInfo => {
|
||||
info.level = info.level.toUpperCase()
|
||||
return info
|
||||
}
|
||||
|
||||
const colors = {
|
||||
debug: 'dim',
|
||||
info: 'cyan',
|
||||
warn: 'yellow',
|
||||
error: 'red',
|
||||
}
|
||||
|
||||
return _createLogger({
|
||||
level: LOG_LEVEL,
|
||||
// Need to upper case level before colorization or we destroy ANSI codes
|
||||
format: format.combine({ transform: uppercaseTransformer }, format.colorize({ level: true, colors }), {
|
||||
transform: formatTransformer,
|
||||
}),
|
||||
defaultMeta: { service },
|
||||
transports: [new transports.Console({})],
|
||||
})
|
||||
}
|
||||
|
||||
/** Creates a silent logger. */
|
||||
export function createSilentLogger(): Logger {
|
||||
return _createLogger({ silent: true })
|
||||
}
|
||||
|
||||
/**
|
||||
* Log the beginning, end, and exception of an operation.
|
||||
*
|
||||
* @param name The log message to output.
|
||||
* @param logger The logger instance.
|
||||
* @param f The operation to perform.
|
||||
*/
|
||||
export async function logCall<T>(name: string, logger: Logger, f: () => Promise<T> | T): Promise<T> {
|
||||
const timer = logger.startTimer()
|
||||
logger.debug(`${name}: starting`)
|
||||
|
||||
try {
|
||||
const value = await f()
|
||||
timer.done({ message: `${name}: finished`, level: 'debug' })
|
||||
return value
|
||||
} catch (error) {
|
||||
timer.done({ message: `${name}: failed`, level: 'error', error })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
import { mustGetFromEither } from './maps'
|
||||
|
||||
describe('mustGetFromEither', () => {
|
||||
it('should return first defined value', () => {
|
||||
const map1 = new Map<string, string>()
|
||||
const map2 = new Map<string, string>()
|
||||
|
||||
map2.set('foo', 'baz')
|
||||
expect(mustGetFromEither(map1, map2, 'foo', '')).toEqual('baz')
|
||||
map1.set('foo', 'bar')
|
||||
expect(mustGetFromEither(map1, map2, 'foo', '')).toEqual('bar')
|
||||
})
|
||||
})
|
||||
@ -1,42 +0,0 @@
|
||||
/**
|
||||
* Return the value of the given key from the given map. If the key does not
|
||||
* exist in the map, an exception is thrown with the given error text.
|
||||
*
|
||||
* @param map The map to query.
|
||||
* @param key The key to search for.
|
||||
* @param elementType The type of element (used for exception message).
|
||||
*/
|
||||
export function mustGet<K, V>(map: Map<K, V>, key: K, elementType: string): V {
|
||||
const value = map.get(key)
|
||||
if (value !== undefined) {
|
||||
return value
|
||||
}
|
||||
|
||||
throw new Error(`Unknown ${elementType} '${String(key)}'.`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the value of the given key from one of the given maps. The first
|
||||
* non-undefined value to be found is returned. If the key does not exist in
|
||||
* either map, an exception is thrown with the given error text.
|
||||
*
|
||||
* @param map1 The first map to query.
|
||||
* @param map2 The second map to query.
|
||||
* @param key The key to search for.
|
||||
* @param elementType The type of element (used for exception message).
|
||||
*/
|
||||
export function mustGetFromEither<K1, V1, K2, V2>(
|
||||
map1: Map<K1, V1>,
|
||||
map2: Map<K2, V2>,
|
||||
key: K1 & K2,
|
||||
elementType: string
|
||||
): V1 | V2 {
|
||||
for (const map of [map1, map2]) {
|
||||
const value = map.get(key)
|
||||
if (value !== undefined) {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Unknown ${elementType} '${String(key)}'.`)
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
import promClient from 'prom-client'
|
||||
|
||||
/**
|
||||
* Instrument the duration and error rate of the given function.
|
||||
*
|
||||
* @param durationHistogram The histogram for operation durations.
|
||||
* @param errorsCounter The counter for errors.
|
||||
* @param fn The function to instrument.
|
||||
*/
|
||||
export async function instrument<T>(
|
||||
durationHistogram: promClient.Histogram<string>,
|
||||
errorsCounter: promClient.Counter<string>,
|
||||
fn: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const end = durationHistogram.startTimer()
|
||||
try {
|
||||
return await fn()
|
||||
} catch (error) {
|
||||
errorsCounter.inc()
|
||||
throw error
|
||||
} finally {
|
||||
end()
|
||||
}
|
||||
}
|
||||
@ -1,22 +0,0 @@
|
||||
import * as sqliteModels from './sqlite'
|
||||
|
||||
/**
|
||||
* Hash a string or numeric identifier into the range `[0, maxIndex)`. The
|
||||
* hash algorithm here is similar to the one used in Java's String.hashCode.
|
||||
*
|
||||
* @param id The identifier to hash.
|
||||
* @param maxIndex The maximum of the range.
|
||||
*/
|
||||
export function hashKey(id: sqliteModels.DefinitionReferenceResultId, maxIndex: number): number {
|
||||
const s = `${id}`
|
||||
|
||||
let hash = 0
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
const chr = s.charCodeAt(i)
|
||||
hash = (hash << 5) - hash + chr
|
||||
hash |= 0
|
||||
}
|
||||
|
||||
// Hash value may be negative - must unset sign bit before modulus
|
||||
return Math.abs(hash) % maxIndex
|
||||
}
|
||||
@ -1,95 +0,0 @@
|
||||
import { MAX_TRAVERSAL_LIMIT } from '../constants'
|
||||
|
||||
/**
|
||||
* Return a recursive CTE `lineage` that returns ancestors of the commit for the given
|
||||
* repository. This assumes that the repository identifier is $1 and the commit is $2.
|
||||
*/
|
||||
export function ancestorLineage(): string {
|
||||
return `
|
||||
RECURSIVE lineage(id, "commit", parent, repository_id) AS (
|
||||
SELECT c.* FROM lsif_commits c WHERE c.repository_id = $1 AND c."commit" = $2
|
||||
UNION
|
||||
SELECT c.* FROM lineage a JOIN lsif_commits c ON a.repository_id = c.repository_id AND a.parent = c."commit"
|
||||
)
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a recursive CTE `lineage` that returns ancestors and descendants of the commit for
|
||||
* the given repository. This assumes that the repository identifier is $1 and the commit is $2.
|
||||
* This happens to evaluate in Postgres as a lazy generator, which allows us to pull the "next"
|
||||
* closest commit in either direction from the source commit as needed.
|
||||
*/
|
||||
export function bidirectionalLineage(): string {
|
||||
return `
|
||||
RECURSIVE lineage(id, "commit", parent_commit, repository_id, direction) AS (
|
||||
SELECT l.* FROM (
|
||||
-- seed recursive set with commit looking in ancestor direction
|
||||
SELECT c.*, 'A' FROM lsif_commits c WHERE c.repository_id = $1 AND c."commit" = $2
|
||||
UNION
|
||||
-- seed recursive set with commit looking in descendant direction
|
||||
SELECT c.*, 'D' FROM lsif_commits c WHERE c.repository_id = $1 AND c."commit" = $2
|
||||
) l
|
||||
|
||||
UNION
|
||||
|
||||
SELECT * FROM (
|
||||
WITH l_inner AS (SELECT * FROM lineage)
|
||||
-- get next ancestors (multiple parents for merge commits)
|
||||
SELECT c.*, 'A' FROM l_inner l JOIN lsif_commits c ON l.direction = 'A' AND c.repository_id = l.repository_id AND c."commit" = l.parent_commit
|
||||
UNION
|
||||
-- get next descendants
|
||||
SELECT c.*, 'D' FROM l_inner l JOIN lsif_commits c ON l.direction = 'D' and c.repository_id = l.repository_id AND c.parent_commit = l."commit"
|
||||
) subquery
|
||||
)
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a set of CTE definitions assuming the definition of a previous CTE named `lineage`.
|
||||
* This creates the CTE `visible_ids`, which gathers the set of LSIF dump identifiers whose
|
||||
* commit occurs in `lineage` (within the given traversal limit) and whose root does not
|
||||
* overlap another visible dump from the same indexer.
|
||||
*
|
||||
* @param limit The maximum number of dumps that can be extracted from `lineage`.
|
||||
*/
|
||||
export function visibleDumps(limit: number = MAX_TRAVERSAL_LIMIT): string {
|
||||
return `
|
||||
${lineageWithDumps(limit)},
|
||||
visible_ids AS (
|
||||
-- Remove dumps where there exists another visible dump of smaller depth with an
|
||||
-- overlapping root from the same indexer. Such dumps would not be returned with
|
||||
-- a closest commit query so we don't want to return results for them in global
|
||||
-- find-reference queries either.
|
||||
SELECT DISTINCT t1.dump_id as id FROM lineage_with_dumps t1 WHERE NOT EXISTS (
|
||||
SELECT 1 FROM lineage_with_dumps t2
|
||||
WHERE t2.n < t1.n AND t1.indexer = t2.indexer AND (
|
||||
t2.root LIKE (t1.root || '%') OR
|
||||
t1.root LIKE (t2.root || '%')
|
||||
)
|
||||
)
|
||||
)
|
||||
`
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a set of CTE definitions assuming the definition of a previous CTE named `lineage`.
|
||||
* This creates the CTE `lineage_with_dumps`, which gathers the set of LSIF dump identifiers
|
||||
* whose commit occurs in `lineage` (within the given traversal limit).
|
||||
*
|
||||
* @param limit The maximum number of dumps that can be extracted from `lineage`.
|
||||
*/
|
||||
function lineageWithDumps(limit: number = MAX_TRAVERSAL_LIMIT): string {
|
||||
return `
|
||||
-- Limit the visibility to the maximum traversal depth and approximate
|
||||
-- each commit's depth by its row number.
|
||||
limited_lineage AS (
|
||||
SELECT a.*, row_number() OVER() as n from lineage a LIMIT ${limit}
|
||||
),
|
||||
-- Correlate commits to dumps and filter out commits without LSIF data
|
||||
lineage_with_dumps AS (
|
||||
SELECT a.*, d.root, d.indexer, d.id as dump_id FROM limited_lineage a
|
||||
JOIN lsif_dumps d ON d.repository_id = a.repository_id AND d."commit" = a."commit"
|
||||
)
|
||||
`
|
||||
}
|
||||
@ -1,305 +0,0 @@
|
||||
import * as lsif from 'lsif-protocol'
|
||||
import { Column, Entity, Index, PrimaryColumn } from 'typeorm'
|
||||
import { calcSqliteBatchSize } from './util'
|
||||
|
||||
export type DocumentId = lsif.Id
|
||||
export type DocumentPath = string
|
||||
export type RangeId = lsif.Id
|
||||
export type DefinitionResultId = lsif.Id
|
||||
export type ReferenceResultId = lsif.Id
|
||||
export type DefinitionReferenceResultId = DefinitionResultId | ReferenceResultId
|
||||
export type HoverResultId = lsif.Id
|
||||
export type MonikerId = lsif.Id
|
||||
export type PackageInformationId = lsif.Id
|
||||
|
||||
/** A type that describes a gzipped and JSON-encoded value of type `T`. */
|
||||
export type JSONEncoded<T> = Buffer
|
||||
|
||||
/**
|
||||
* A type of hashed value created by hashing a value of type `T` and performing
|
||||
* the modulus with a value of type `U`. This is to link the index of a result
|
||||
* chunk to the hashed value of the identifiers stored within it.
|
||||
*/
|
||||
export type HashMod<T, U> = number
|
||||
|
||||
/**
|
||||
* An entity within the database describing LSIF data for a single repository
|
||||
* and commit pair. There should be only one metadata entity per database.
|
||||
*/
|
||||
@Entity({ name: 'meta' })
|
||||
export class MetaModel {
|
||||
/** The number of model instances that can be inserted at once. */
|
||||
public static BatchSize = calcSqliteBatchSize(4)
|
||||
|
||||
/** A unique ID required by typeorm entities: always zero here. */
|
||||
@PrimaryColumn('int')
|
||||
public id!: number
|
||||
|
||||
/** The version string of the input LSIF that created this database. */
|
||||
@Column('text')
|
||||
public lsifVersion!: string
|
||||
|
||||
/** The internal version of the LSIF system that created this database. */
|
||||
@Column('text')
|
||||
public sourcegraphVersion!: string
|
||||
|
||||
/**
|
||||
* The number of result chunks allocated when converting the dump stored
|
||||
* in this database. This is used as an upper bound for the hash into the
|
||||
* `resultChunks` table and must be record to keep the hash generation
|
||||
* stable.
|
||||
*/
|
||||
@Column('int')
|
||||
public numResultChunks!: number
|
||||
}
|
||||
|
||||
/**
|
||||
* An entity within the database describing LSIF data for a single repository and
|
||||
* commit pair. This contains a JSON-encoded `DocumentData` object that describes
|
||||
* relations within a single file of the dump.
|
||||
*/
|
||||
@Entity({ name: 'documents' })
|
||||
export class DocumentModel {
|
||||
/** The number of model instances that can be inserted at once. */
|
||||
public static BatchSize = calcSqliteBatchSize(2)
|
||||
|
||||
/** The root-relative path of the document. */
|
||||
@PrimaryColumn('text')
|
||||
public path!: DocumentPath
|
||||
|
||||
/** The JSON-encoded document data. */
|
||||
@Column('blob')
|
||||
public data!: JSONEncoded<DocumentData>
|
||||
}
|
||||
|
||||
/**
|
||||
* An entity within the database describing LSIF data for a single repository and
|
||||
* commit pair. This contains a JSON-encoded `ResultChunk` object that describes
|
||||
* a subset of the definition and reference results of the dump.
|
||||
*/
|
||||
@Entity({ name: 'resultChunks' })
|
||||
export class ResultChunkModel {
|
||||
/** The number of model instances that can be inserted at once. */
|
||||
public static BatchSize = calcSqliteBatchSize(2)
|
||||
|
||||
/**
|
||||
* The identifier of the chunk. This is also the index of the chunk during its
|
||||
* construction, and the identifiers contained in this chunk hash to this index
|
||||
* (modulo the total number of chunks for the dump).
|
||||
*/
|
||||
@PrimaryColumn('int')
|
||||
public id!: HashMod<DefinitionReferenceResultId, MetaModel['numResultChunks']>
|
||||
|
||||
/** The JSON-encoded chunk data. */
|
||||
@Column('blob')
|
||||
public data!: JSONEncoded<ResultChunkData>
|
||||
}
|
||||
|
||||
/**
|
||||
* The base class for `DefinitionModel` and `ReferenceModel` as they have identical
|
||||
* column descriptions.
|
||||
*/
|
||||
class Symbols {
|
||||
/** The number of model instances that can be inserted at once. */
|
||||
public static BatchSize = calcSqliteBatchSize(8)
|
||||
|
||||
/** A unique ID required by typeorm entities. */
|
||||
@PrimaryColumn('int')
|
||||
public id!: number
|
||||
|
||||
/** The name of the package type (e.g. npm, pip). */
|
||||
@Column('text')
|
||||
public scheme!: string
|
||||
|
||||
/** The unique identifier of the moniker. */
|
||||
@Column('text')
|
||||
public identifier!: string
|
||||
|
||||
/** The path of the document to which this reference belongs. */
|
||||
@Column('text')
|
||||
public documentPath!: DocumentPath
|
||||
|
||||
/** The zero-indexed line describing the start of this range. */
|
||||
@Column('int')
|
||||
public startLine!: number
|
||||
|
||||
/** The zero-indexed line describing the end of this range. */
|
||||
@Column('int')
|
||||
public endLine!: number
|
||||
|
||||
/** The zero-indexed line describing the start of this range. */
|
||||
@Column('int')
|
||||
public startCharacter!: number
|
||||
|
||||
/** The zero-indexed line describing the end of this range. */
|
||||
@Column('int')
|
||||
public endCharacter!: number
|
||||
}
|
||||
|
||||
/**
|
||||
* An entity within the database describing LSIF data for a single repository and commit
|
||||
* pair. This maps external monikers to their range and the document that contains the
|
||||
* definition of the moniker.
|
||||
*/
|
||||
@Entity({ name: 'definitions' })
|
||||
@Index(['scheme', 'identifier'])
|
||||
export class DefinitionModel extends Symbols {}
|
||||
|
||||
/**
|
||||
* An entity within the database describing LSIF data for a single repository and commit
|
||||
* pair. This maps imported monikers to their range and the document that contains a
|
||||
* reference to the moniker.
|
||||
*/
|
||||
@Entity({ name: 'references' })
|
||||
@Index(['scheme', 'identifier'])
|
||||
export class ReferenceModel extends Symbols {}
|
||||
|
||||
/**
|
||||
* Data for a single document within an LSIF dump. The data here can answer definitions,
|
||||
* references, and hover queries if the results are all contained within the same document.
|
||||
*/
|
||||
export interface DocumentData {
|
||||
/** A mapping from range identifiers to range data. */
|
||||
ranges: Map<RangeId, RangeData>
|
||||
|
||||
/**
|
||||
* A map of hover result identifiers to hover results normalized as a single
|
||||
* string.
|
||||
*/
|
||||
hoverResults: Map<HoverResultId, string>
|
||||
|
||||
/** A map of moniker identifiers to moniker data. */
|
||||
monikers: Map<MonikerId, MonikerData>
|
||||
|
||||
/** A map of package information identifiers to package information data. */
|
||||
packageInformation: Map<PackageInformationId, PackageInformationData>
|
||||
}
|
||||
|
||||
/**
|
||||
* A range identifier that also specifies the identifier of the document to
|
||||
* which it belongs. This is sometimes necessary as we hold definition and
|
||||
* reference results between packages, but the identifier of the range must be
|
||||
* looked up in a map of another encoded document.
|
||||
*/
|
||||
export interface DocumentIdRangeId {
|
||||
/**
|
||||
* The identifier of the document. The path of the document can be queried
|
||||
* by this identifier in the containing document.
|
||||
*/
|
||||
documentId: DocumentId
|
||||
|
||||
/** The identifier of the range in the referenced document. */
|
||||
rangeId: RangeId
|
||||
}
|
||||
|
||||
/**
|
||||
* A range identifier that also specifies the path of the document to which it
|
||||
* belongs. This is generally created by determining the path from an instance of
|
||||
* `DocumentIdRangeId`.
|
||||
*/
|
||||
export interface DocumentPathRangeId {
|
||||
/** The path of the document. */
|
||||
documentPath: DocumentPath
|
||||
|
||||
/** The identifier of the range in the referenced document. */
|
||||
rangeId: RangeId
|
||||
}
|
||||
|
||||
/**
|
||||
* A result chunk is a subset of the definition and reference result data for the
|
||||
* LSIF dump. Results are inserted into chunks based on the hash code of their
|
||||
* identifier (thus every chunk has a roughly proportional amount of data).
|
||||
*/
|
||||
export interface ResultChunkData {
|
||||
/**
|
||||
* A map from document identifiers to document paths. The document identifiers
|
||||
* in the `documentIdRangeIds` field reference a concrete path stored here.
|
||||
*/
|
||||
documentPaths: Map<DocumentId, DocumentPath>
|
||||
|
||||
/**
|
||||
* A map from definition or reference result identifiers to the ranges that
|
||||
* compose the result set. Each range is paired with the identifier of the
|
||||
* document in which it can be found.
|
||||
*/
|
||||
documentIdRangeIds: Map<DefinitionReferenceResultId, DocumentIdRangeId[]>
|
||||
}
|
||||
|
||||
/**
|
||||
* An internal representation of a range vertex from an LSIF dump. It contains the same
|
||||
* relevant edge data, which can be subsequently queried in the containing document. The
|
||||
* data that was reachable via a result set has been collapsed into this object during
|
||||
* import.
|
||||
*/
|
||||
export interface RangeData {
|
||||
/** The line on which the range starts (0-indexed, inclusive). */
|
||||
startLine: number
|
||||
|
||||
/** The line on which the range ends (0-indexed, inclusive). */
|
||||
startCharacter: number
|
||||
|
||||
/** The character on which the range starts (0-indexed, inclusive). */
|
||||
endLine: number
|
||||
|
||||
/** The character on which the range ends (0-indexed, inclusive). */
|
||||
endCharacter: number
|
||||
|
||||
/**
|
||||
* The identifier of the definition result attached to this range, if one exists.
|
||||
* The definition result object can be queried by its identifier within the containing
|
||||
* document.
|
||||
*/
|
||||
definitionResultId?: DefinitionResultId
|
||||
|
||||
/**
|
||||
* The identifier of the reference result attached to this range, if one exists.
|
||||
* The reference result object can be queried by its identifier within the containing
|
||||
* document.
|
||||
*/
|
||||
referenceResultId?: ReferenceResultId
|
||||
|
||||
/**
|
||||
* The identifier of the hover result attached to this range, if one exists. The
|
||||
* hover result object can be queried by its identifier within the containing
|
||||
* document.
|
||||
*/
|
||||
hoverResultId?: HoverResultId
|
||||
|
||||
/**
|
||||
* The set of moniker identifiers directly attached to this range. The moniker
|
||||
* object can be queried by its identifier within the
|
||||
* containing document.
|
||||
*/
|
||||
monikerIds: Set<MonikerId>
|
||||
}
|
||||
|
||||
/** Data about a moniker attached to a range. */
|
||||
export interface MonikerData {
|
||||
/** The kind of moniker (e.g. local, import, export). */
|
||||
kind: lsif.MonikerKind
|
||||
|
||||
/** The name of the package type (e.g. npm, pip). */
|
||||
scheme: string
|
||||
|
||||
/** The unique identifier of the moniker. */
|
||||
identifier: string
|
||||
|
||||
/**
|
||||
* The identifier of the package information to this moniker, if one exists.
|
||||
* The package information object can be queried by its identifier within the
|
||||
* containing document.
|
||||
*/
|
||||
packageInformationId?: PackageInformationId
|
||||
}
|
||||
|
||||
/** Additional data about a non-local moniker. */
|
||||
export interface PackageInformationData {
|
||||
/** The name of the package the moniker describes. */
|
||||
name: string
|
||||
|
||||
/** The version of the package the moniker describes. */
|
||||
version: string | null
|
||||
}
|
||||
|
||||
/** The entities composing the SQLite database models. */
|
||||
export const entities = [DefinitionModel, DocumentModel, MetaModel, ReferenceModel, ResultChunkModel]
|
||||
@ -1,12 +0,0 @@
|
||||
/**
|
||||
* Determine the table inserter batch size for an entity given the number of
|
||||
* fields inserted for that entity. We cannot perform an insert operation in
|
||||
* SQLite with more than 999 placeholder variables, so we need to flush our
|
||||
* batch before we reach that amount. If fields are added to the models, the
|
||||
* argument to this function also needs to change.
|
||||
*
|
||||
* @param numFields The number of fields for an entity.
|
||||
*/
|
||||
export function calcSqliteBatchSize(numFields: number): number {
|
||||
return Math.floor(999 / numFields)
|
||||
}
|
||||
@ -1,56 +0,0 @@
|
||||
import * as constants from './constants'
|
||||
import * as fs from 'mz/fs'
|
||||
import * as path from 'path'
|
||||
|
||||
/**
|
||||
* Construct the path of the SQLite database file for the given dump.
|
||||
*
|
||||
* @param storageRoot The path where SQLite databases are stored.
|
||||
* @param id The ID of the dump.
|
||||
*/
|
||||
export function dbFilename(storageRoot: string, id: number): string {
|
||||
return path.join(storageRoot, constants.DBS_DIR, `${id}.lsif.db`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct the path of the raw upload file for the given identifier.
|
||||
*
|
||||
* @param storageRoot The path where uploads are stored.
|
||||
* @param id The identifier of the upload.
|
||||
*/
|
||||
export function uploadFilename(storageRoot: string, id: number): string {
|
||||
return path.join(storageRoot, constants.UPLOADS_DIR, `${id}.lsif.gz`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the identifier of the database file. Handles both of the
|
||||
* following formats:
|
||||
*
|
||||
* - `{id}.lsif.db`
|
||||
* - `{id}-{repo}-{commit}.lsif.db`
|
||||
*
|
||||
* @param filename The filename.
|
||||
*/
|
||||
export function idFromFilename(filename: string): number | undefined {
|
||||
const id = parseInt(path.parse(filename).name.split('-')[0], 10)
|
||||
if (!isNaN(id)) {
|
||||
return id
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the directory exists.
|
||||
*
|
||||
* @param directoryPath The directory path.
|
||||
*/
|
||||
export async function ensureDirectory(directoryPath: string): Promise<void> {
|
||||
try {
|
||||
await fs.mkdir(directoryPath)
|
||||
} catch (error) {
|
||||
if (!(error && error.code === 'EEXIST')) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
/**
|
||||
* Reads an integer from an environment variable or defaults to the given value.
|
||||
*
|
||||
* @param key The environment variable name.
|
||||
* @param defaultValue The default value.
|
||||
*/
|
||||
export function readEnvInt(key: string, defaultValue: number): number {
|
||||
return (process.env[key] && parseInt(process.env[key] || '', 10)) || defaultValue
|
||||
}
|
||||
@ -1,236 +0,0 @@
|
||||
import * as util from '../test-util'
|
||||
import * as pgModels from '../models/pg'
|
||||
import { Connection } from 'typeorm'
|
||||
import { fail } from 'assert'
|
||||
import { DumpManager } from './dumps'
|
||||
import { DependencyManager } from './dependencies'
|
||||
|
||||
describe('DependencyManager', () => {
|
||||
let connection!: Connection
|
||||
let cleanup!: () => Promise<void>
|
||||
let dumpManager!: DumpManager
|
||||
let dependencyManager!: DependencyManager
|
||||
|
||||
const repositoryId1 = 100
|
||||
const repositoryId2 = 101
|
||||
|
||||
beforeAll(async () => {
|
||||
;({ connection, cleanup } = await util.createCleanPostgresDatabase())
|
||||
dumpManager = new DumpManager(connection)
|
||||
dependencyManager = new DependencyManager(connection)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
beforeEach(async () => {
|
||||
if (connection) {
|
||||
await util.truncatePostgresTables(connection)
|
||||
}
|
||||
})
|
||||
|
||||
it('should respect bloom filter', async () => {
|
||||
if (!dependencyManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
const cf = util.createCommit()
|
||||
|
||||
const updatePackages = async (
|
||||
commit: string,
|
||||
root: string,
|
||||
identifiers: string[]
|
||||
): Promise<pgModels.LsifDump> => {
|
||||
const dump = await util.insertDump(connection, dumpManager, repositoryId1, commit, root, 'test')
|
||||
|
||||
await dependencyManager.addPackagesAndReferences(
|
||||
dump.id,
|
||||
[],
|
||||
[
|
||||
{
|
||||
package: {
|
||||
scheme: 'npm',
|
||||
name: 'p1',
|
||||
version: '0.1.0',
|
||||
},
|
||||
identifiers,
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
return dump
|
||||
}
|
||||
|
||||
// Note: roots must be unique so dumps are visible
|
||||
const dumpa = await updatePackages(ca, 'r1', ['x', 'y', 'z'])
|
||||
const dumpb = await updatePackages(cb, 'r2', ['y', 'z'])
|
||||
const dumpf = await updatePackages(cf, 'r3', ['y', 'z'])
|
||||
await updatePackages(cc, 'r4', ['x', 'z'])
|
||||
await updatePackages(cd, 'r5', ['x'])
|
||||
await updatePackages(ce, 'r6', ['x', 'z'])
|
||||
|
||||
const getReferencedDumpIds = async () => {
|
||||
const { packageReferences } = await dependencyManager.getPackageReferences({
|
||||
repositoryId: repositoryId2,
|
||||
scheme: 'npm',
|
||||
name: 'p1',
|
||||
version: '0.1.0',
|
||||
identifier: 'y',
|
||||
limit: 50,
|
||||
offset: 0,
|
||||
})
|
||||
|
||||
return packageReferences.map(packageReference => packageReference.dump_id).sort()
|
||||
}
|
||||
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId1,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
[cc, new Set([cb])],
|
||||
[cd, new Set([cc])],
|
||||
[ce, new Set([cd])],
|
||||
[cf, new Set([ce])],
|
||||
])
|
||||
)
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId1, cf)
|
||||
|
||||
// only references containing identifier y
|
||||
expect(await getReferencedDumpIds()).toEqual([dumpa.id, dumpb.id, dumpf.id])
|
||||
})
|
||||
|
||||
it('should re-query if bloom filter prunes too many results', async () => {
|
||||
if (!dependencyManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
const updatePackages = async (
|
||||
commit: string,
|
||||
root: string,
|
||||
identifiers: string[]
|
||||
): Promise<pgModels.LsifDump> => {
|
||||
const dump = await util.insertDump(connection, dumpManager, repositoryId1, commit, root, 'test')
|
||||
|
||||
await dependencyManager.addPackagesAndReferences(
|
||||
dump.id,
|
||||
[],
|
||||
[
|
||||
{
|
||||
package: {
|
||||
scheme: 'npm',
|
||||
name: 'p1',
|
||||
version: '0.1.0',
|
||||
},
|
||||
identifiers,
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
return dump
|
||||
}
|
||||
|
||||
const dumps = []
|
||||
for (let i = 0; i < 250; i++) {
|
||||
// Spread out uses of `y` so that we pull back a series of pages that are
|
||||
// empty and half-empty after being filtered by the bloom filter. We will
|
||||
// have to empty pages (i < 100) followed by three pages where very third
|
||||
// uses the identifier. In all, there are fifty uses spread over 5 pages.
|
||||
const isUse = i >= 100 && i % 3 === 0
|
||||
|
||||
const dump = await updatePackages(util.createCommit(), `r${i}`, ['x', isUse ? 'y' : 'z'])
|
||||
dump.visibleAtTip = true
|
||||
await connection.getRepository(pgModels.LsifUpload).save(dump)
|
||||
|
||||
if (isUse) {
|
||||
// Save use ids
|
||||
dumps.push(dump.id)
|
||||
}
|
||||
}
|
||||
|
||||
const { packageReferences } = await dependencyManager.getPackageReferences({
|
||||
repositoryId: repositoryId2,
|
||||
scheme: 'npm',
|
||||
name: 'p1',
|
||||
version: '0.1.0',
|
||||
identifier: 'y',
|
||||
limit: 50,
|
||||
offset: 0,
|
||||
})
|
||||
|
||||
expect(packageReferences.map(packageReference => packageReference.dump_id).sort()).toEqual(dumps)
|
||||
})
|
||||
|
||||
it('references only returned if dumps visible at tip', async () => {
|
||||
if (!dependencyManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
|
||||
const references = [
|
||||
{
|
||||
package: {
|
||||
scheme: 'npm',
|
||||
name: 'p1',
|
||||
version: '0.1.0',
|
||||
},
|
||||
identifiers: ['x', 'y', 'z'],
|
||||
},
|
||||
]
|
||||
|
||||
const dumpa = await util.insertDump(connection, dumpManager, repositoryId1, ca, '', 'test')
|
||||
const dumpb = await util.insertDump(connection, dumpManager, repositoryId1, cb, '', 'test')
|
||||
const dumpc = await util.insertDump(connection, dumpManager, repositoryId1, cc, '', 'test')
|
||||
|
||||
await dependencyManager.addPackagesAndReferences(dumpa.id, [], references)
|
||||
await dependencyManager.addPackagesAndReferences(dumpb.id, [], references)
|
||||
await dependencyManager.addPackagesAndReferences(dumpc.id, [], references)
|
||||
|
||||
const getReferencedDumpIds = async () =>
|
||||
(
|
||||
await dependencyManager.getPackageReferences({
|
||||
repositoryId: repositoryId2,
|
||||
scheme: 'npm',
|
||||
name: 'p1',
|
||||
version: '0.1.0',
|
||||
identifier: 'y',
|
||||
limit: 50,
|
||||
offset: 0,
|
||||
})
|
||||
).packageReferences
|
||||
.map(packageReference => packageReference.dump_id)
|
||||
.sort()
|
||||
|
||||
const updateVisibility = async (visibleA: boolean, visibleB: boolean, visibleC: boolean) => {
|
||||
dumpa.visibleAtTip = visibleA
|
||||
dumpb.visibleAtTip = visibleB
|
||||
dumpc.visibleAtTip = visibleC
|
||||
await connection.getRepository(pgModels.LsifUpload).save(dumpa)
|
||||
await connection.getRepository(pgModels.LsifUpload).save(dumpb)
|
||||
await connection.getRepository(pgModels.LsifUpload).save(dumpc)
|
||||
}
|
||||
|
||||
// Set a, b visible from tip
|
||||
await updateVisibility(true, true, false)
|
||||
expect(await getReferencedDumpIds()).toEqual([dumpa.id, dumpb.id])
|
||||
|
||||
// Clear a, b visible from tip, set c visible fro
|
||||
await updateVisibility(false, false, true)
|
||||
expect(await getReferencedDumpIds()).toEqual([dumpc.id])
|
||||
|
||||
// Clear all visible from tip
|
||||
await updateVisibility(false, false, false)
|
||||
expect(await getReferencedDumpIds()).toEqual([])
|
||||
})
|
||||
})
|
||||
@ -1,766 +0,0 @@
|
||||
import * as util from '../test-util'
|
||||
import * as pgModels from '../models/pg'
|
||||
import nock from 'nock'
|
||||
import { Connection } from 'typeorm'
|
||||
import { DumpManager } from './dumps'
|
||||
import { fail } from 'assert'
|
||||
import { MAX_TRAVERSAL_LIMIT } from '../constants'
|
||||
|
||||
describe('DumpManager', () => {
|
||||
let connection!: Connection
|
||||
let cleanup!: () => Promise<void>
|
||||
let dumpManager!: DumpManager
|
||||
|
||||
let counter = 100
|
||||
const nextId = () => {
|
||||
counter++
|
||||
return counter
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
;({ connection, cleanup } = await util.createCleanPostgresDatabase())
|
||||
dumpManager = new DumpManager(connection)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
beforeEach(async () => {
|
||||
if (connection) {
|
||||
await util.truncatePostgresTables(connection)
|
||||
}
|
||||
})
|
||||
|
||||
it('should find closest commits with LSIF data (first commit graph)', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This database has the following commit graph:
|
||||
//
|
||||
// [a] --+--- b --------+--e -- f --+-- [g]
|
||||
// | | |
|
||||
// +-- [c] -- d --+ +--- h
|
||||
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
const cf = util.createCommit()
|
||||
const cg = util.createCommit()
|
||||
const ch = util.createCommit()
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
[cc, new Set([ca])],
|
||||
[cd, new Set([cc])],
|
||||
[ce, new Set([cb])],
|
||||
[ce, new Set([cd])],
|
||||
[cf, new Set([ce])],
|
||||
[cg, new Set([cf])],
|
||||
[ch, new Set([cf])],
|
||||
])
|
||||
)
|
||||
|
||||
// Add dumps
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, '', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cc, '', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cg, '', 'test')
|
||||
|
||||
const d1 = await dumpManager.findClosestDumps(repositoryId, ca, 'file.ts')
|
||||
const d2 = await dumpManager.findClosestDumps(repositoryId, cb, 'file.ts')
|
||||
const d3 = await dumpManager.findClosestDumps(repositoryId, cc, 'file.ts')
|
||||
const d4 = await dumpManager.findClosestDumps(repositoryId, cd, 'file.ts')
|
||||
const d5 = await dumpManager.findClosestDumps(repositoryId, cf, 'file.ts')
|
||||
const d6 = await dumpManager.findClosestDumps(repositoryId, cg, 'file.ts')
|
||||
const d7 = await dumpManager.findClosestDumps(repositoryId, ce, 'file.ts')
|
||||
const d8 = await dumpManager.findClosestDumps(repositoryId, ch, 'file.ts')
|
||||
|
||||
expect(d1).toHaveLength(1)
|
||||
expect(d2).toHaveLength(1)
|
||||
expect(d3).toHaveLength(1)
|
||||
expect(d4).toHaveLength(1)
|
||||
expect(d5).toHaveLength(1)
|
||||
expect(d6).toHaveLength(1)
|
||||
expect(d7).toHaveLength(1)
|
||||
expect(d8).toHaveLength(1)
|
||||
|
||||
// Test closest commit
|
||||
expect(d1[0].commit).toEqual(ca)
|
||||
expect(d2[0].commit).toEqual(ca)
|
||||
expect(d3[0].commit).toEqual(cc)
|
||||
expect(d4[0].commit).toEqual(cc)
|
||||
expect(d5[0].commit).toEqual(cg)
|
||||
expect(d6[0].commit).toEqual(cg)
|
||||
|
||||
// Multiple nearest are chosen arbitrarily
|
||||
expect([ca, cc, cg]).toContain(d7[0].commit)
|
||||
expect([ca, cc]).toContain(d8[0].commit)
|
||||
})
|
||||
|
||||
it('should find closest commits with LSIF data (second commit graph)', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This database has the following commit graph:
|
||||
//
|
||||
// a --+-- [b] ---- c
|
||||
// |
|
||||
// +--- d --+-- e -- f
|
||||
// |
|
||||
// +-- g -- h
|
||||
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
const cf = util.createCommit()
|
||||
const cg = util.createCommit()
|
||||
const ch = util.createCommit()
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
[cc, new Set([cb])],
|
||||
[cd, new Set([ca])],
|
||||
[ce, new Set([cd])],
|
||||
[cf, new Set([ce])],
|
||||
[cg, new Set([cd])],
|
||||
[ch, new Set([cg])],
|
||||
])
|
||||
)
|
||||
|
||||
// Add dumps
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, '', 'test')
|
||||
|
||||
const d1 = await dumpManager.findClosestDumps(repositoryId, ca, 'file.ts')
|
||||
const d2 = await dumpManager.findClosestDumps(repositoryId, cb, 'file.ts')
|
||||
const d3 = await dumpManager.findClosestDumps(repositoryId, cc, 'file.ts')
|
||||
|
||||
expect(d1).toHaveLength(1)
|
||||
expect(d2).toHaveLength(1)
|
||||
expect(d3).toHaveLength(1)
|
||||
|
||||
// Test closest commit
|
||||
expect(d1[0].commit).toEqual(cb)
|
||||
expect(d2[0].commit).toEqual(cb)
|
||||
expect(d3[0].commit).toEqual(cb)
|
||||
expect(await dumpManager.findClosestDumps(repositoryId, cd, 'file.ts')).toHaveLength(0)
|
||||
expect(await dumpManager.findClosestDumps(repositoryId, ce, 'file.ts')).toHaveLength(0)
|
||||
expect(await dumpManager.findClosestDumps(repositoryId, cf, 'file.ts')).toHaveLength(0)
|
||||
expect(await dumpManager.findClosestDumps(repositoryId, cg, 'file.ts')).toHaveLength(0)
|
||||
expect(await dumpManager.findClosestDumps(repositoryId, ch, 'file.ts')).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('should find closest commits with LSIF data (distinct roots)', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This database has the following commit graph:
|
||||
//
|
||||
// a --+-- [b]
|
||||
//
|
||||
// Where LSIF dumps exist at b at roots: root1/ and root2/.
|
||||
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
])
|
||||
)
|
||||
|
||||
// Add dumps
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, 'root1/', '')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, 'root2/', '')
|
||||
|
||||
// Test closest commit
|
||||
const d1 = await dumpManager.findClosestDumps(repositoryId, ca, 'blah')
|
||||
const d2 = await dumpManager.findClosestDumps(repositoryId, cb, 'root1/file.ts')
|
||||
const d3 = await dumpManager.findClosestDumps(repositoryId, cb, 'root2/file.ts')
|
||||
const d4 = await dumpManager.findClosestDumps(repositoryId, ca, 'root2/file.ts')
|
||||
|
||||
expect(d1).toHaveLength(0)
|
||||
expect(d2).toHaveLength(1)
|
||||
expect(d3).toHaveLength(1)
|
||||
expect(d4).toHaveLength(1)
|
||||
|
||||
expect(d2[0].commit).toEqual(cb)
|
||||
expect(d2[0].root).toEqual('root1/')
|
||||
expect(d3[0].commit).toEqual(cb)
|
||||
expect(d3[0].root).toEqual('root2/')
|
||||
expect(d4[0].commit).toEqual(cb)
|
||||
expect(d4[0].root).toEqual('root2/')
|
||||
|
||||
const d5 = await dumpManager.findClosestDumps(repositoryId, ca, 'root3/file.ts')
|
||||
expect(d5).toHaveLength(0)
|
||||
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, '', '')
|
||||
const d6 = await dumpManager.findClosestDumps(repositoryId, ca, 'root2/file.ts')
|
||||
const d7 = await dumpManager.findClosestDumps(repositoryId, ca, 'root3/file.ts')
|
||||
|
||||
expect(d6).toHaveLength(1)
|
||||
expect(d7).toHaveLength(1)
|
||||
|
||||
expect(d6[0].commit).toEqual(cb)
|
||||
expect(d6[0].root).toEqual('')
|
||||
expect(d7[0].commit).toEqual(cb)
|
||||
expect(d7[0].root).toEqual('')
|
||||
})
|
||||
|
||||
it('should find closest commits with LSIF data (overlapping roots)', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This database has the following commit graph:
|
||||
//
|
||||
// a -- b --+-- c --+-- e -- f
|
||||
// | |
|
||||
// +-- d --+
|
||||
//
|
||||
// With the following LSIF dumps:
|
||||
//
|
||||
// | Commit | Root | Indexer |
|
||||
// | ------ + ------- + ------- |
|
||||
// | a | root3/ | A |
|
||||
// | a | root4/ | B |
|
||||
// | b | root1/ | A |
|
||||
// | b | root2/ | A |
|
||||
// | b | | B | (overwrites root4/ at commit a)
|
||||
// | c | root1/ | A | (overwrites root1/ at commit b)
|
||||
// | d | | B | (overwrites (root) at commit b)
|
||||
// | e | root2/ | A | (overwrites root2/ at commit b)
|
||||
// | f | root1/ | A | (overwrites root1/ at commit b)
|
||||
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
const cf = util.createCommit()
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
[cc, new Set([cb])],
|
||||
[cd, new Set([cb])],
|
||||
[ce, new Set([cc, cd])],
|
||||
[cf, new Set([ce])],
|
||||
])
|
||||
)
|
||||
|
||||
// Add dumps
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, 'root3/', 'A')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, 'root4/', 'B')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, 'root1/', 'A')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, 'root2/', 'A')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, '', 'B')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cc, 'root1/', 'A')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cd, '', 'B')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ce, 'root2/', 'A')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cf, 'root1/', 'A')
|
||||
|
||||
// Test closest commit
|
||||
const d1 = await dumpManager.findClosestDumps(repositoryId, cd, 'root1/file.ts')
|
||||
expect(d1).toHaveLength(2)
|
||||
expect(d1[0].commit).toEqual(cd)
|
||||
expect(d1[0].root).toEqual('')
|
||||
expect(d1[0].indexer).toEqual('B')
|
||||
expect(d1[1].commit).toEqual(cb)
|
||||
expect(d1[1].root).toEqual('root1/')
|
||||
expect(d1[1].indexer).toEqual('A')
|
||||
|
||||
const d2 = await dumpManager.findClosestDumps(repositoryId, ce, 'root2/file.ts')
|
||||
expect(d2).toHaveLength(2)
|
||||
expect(d2[0].commit).toEqual(ce)
|
||||
expect(d2[0].root).toEqual('root2/')
|
||||
expect(d2[0].indexer).toEqual('A')
|
||||
expect(d2[1].commit).toEqual(cd)
|
||||
expect(d2[1].root).toEqual('')
|
||||
expect(d2[1].indexer).toEqual('B')
|
||||
|
||||
const d3 = await dumpManager.findClosestDumps(repositoryId, cc, 'root3/file.ts')
|
||||
expect(d3).toHaveLength(2)
|
||||
expect(d3[0].commit).toEqual(cb)
|
||||
expect(d3[0].root).toEqual('')
|
||||
expect(d3[0].indexer).toEqual('B')
|
||||
expect(d3[1].commit).toEqual(ca)
|
||||
expect(d3[1].root).toEqual('root3/')
|
||||
expect(d3[1].indexer).toEqual('A')
|
||||
|
||||
const d4 = await dumpManager.findClosestDumps(repositoryId, ca, 'root4/file.ts')
|
||||
expect(d4).toHaveLength(1)
|
||||
expect(d4[0].commit).toEqual(ca)
|
||||
expect(d4[0].root).toEqual('root4/')
|
||||
expect(d4[0].indexer).toEqual('B')
|
||||
|
||||
const d5 = await dumpManager.findClosestDumps(repositoryId, cb, 'root4/file.ts')
|
||||
expect(d5).toHaveLength(1)
|
||||
expect(d5[0].commit).toEqual(cb)
|
||||
expect(d5[0].root).toEqual('')
|
||||
expect(d5[0].indexer).toEqual('B')
|
||||
})
|
||||
|
||||
it('should not return elements farther than MAX_TRAVERSAL_LIMIT', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This repository has the following commit graph (ancestors to the left):
|
||||
//
|
||||
// MAX_TRAVERSAL_LIMIT -- ... -- 2 -- 1 -- 0
|
||||
//
|
||||
// Note: we use 'a' as a suffix for commit numbers on construction so that
|
||||
// we can distinguish `1` and `11` (`1a1a1a...` and `11a11a11a..`).
|
||||
|
||||
const repositoryId = nextId()
|
||||
const c0 = util.createCommit(0)
|
||||
const c1 = util.createCommit(1)
|
||||
const cpen = util.createCommit(MAX_TRAVERSAL_LIMIT / 2 - 1)
|
||||
const cmax = util.createCommit(MAX_TRAVERSAL_LIMIT / 2)
|
||||
|
||||
const commits = new Map<string, Set<string>>(
|
||||
Array.from({ length: MAX_TRAVERSAL_LIMIT }, (_, i) => [
|
||||
util.createCommit(i),
|
||||
new Set([util.createCommit(i + 1)]),
|
||||
])
|
||||
)
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(repositoryId, commits)
|
||||
|
||||
// Add dumps
|
||||
await util.insertDump(connection, dumpManager, repositoryId, c0, '', 'test')
|
||||
|
||||
const d1 = await dumpManager.findClosestDumps(repositoryId, c0, 'file.ts')
|
||||
const d2 = await dumpManager.findClosestDumps(repositoryId, c1, 'file.ts')
|
||||
const d3 = await dumpManager.findClosestDumps(repositoryId, cpen, 'file.ts')
|
||||
|
||||
expect(d1).toHaveLength(1)
|
||||
expect(d2).toHaveLength(1)
|
||||
expect(d3).toHaveLength(1)
|
||||
|
||||
// Test closest commit
|
||||
expect(d1[0].commit).toEqual(c0)
|
||||
expect(d2[0].commit).toEqual(c0)
|
||||
expect(d3[0].commit).toEqual(c0)
|
||||
|
||||
// (Assuming MAX_TRAVERSAL_LIMIT = 100)
|
||||
// At commit `50`, the traversal limit will be reached before visiting commit `0`
|
||||
// because commits are visited in this order:
|
||||
//
|
||||
// | depth | commit |
|
||||
// | ----- | ------ |
|
||||
// | 1 | 50 | (with direction 'A')
|
||||
// | 2 | 50 | (with direction 'D')
|
||||
// | 3 | 51 |
|
||||
// | 4 | 49 |
|
||||
// | 5 | 52 |
|
||||
// | 6 | 48 |
|
||||
// | ... | |
|
||||
// | 99 | 99 |
|
||||
// | 100 | 1 | (limit reached)
|
||||
|
||||
expect(await dumpManager.findClosestDumps(repositoryId, cmax, 'file.ts')).toHaveLength(0)
|
||||
|
||||
// Add closer dump
|
||||
await util.insertDump(connection, dumpManager, repositoryId, c1, '', 'test')
|
||||
|
||||
// Now commit 1 should be found
|
||||
const dumps = await dumpManager.findClosestDumps(repositoryId, cmax, 'file.ts')
|
||||
expect(dumps).toHaveLength(1)
|
||||
expect(dumps[0].commit).toEqual(c1)
|
||||
})
|
||||
|
||||
it('should prune overlapping roots during visibility check', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This database has the following commit graph:
|
||||
//
|
||||
// a -- b -- c -- d -- e -- f -- g
|
||||
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
const cf = util.createCommit()
|
||||
const cg = util.createCommit()
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
[cc, new Set([cb])],
|
||||
[cd, new Set([cc])],
|
||||
[ce, new Set([cd])],
|
||||
[cf, new Set([ce])],
|
||||
[cg, new Set([cf])],
|
||||
])
|
||||
)
|
||||
|
||||
// Add dumps
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, 'r1', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, 'r2', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cc, '', 'test') // overwrites r1, r2
|
||||
const d1 = await util.insertDump(connection, dumpManager, repositoryId, cd, 'r3', 'test') // overwrites ''
|
||||
const d2 = await util.insertDump(connection, dumpManager, repositoryId, cf, 'r4', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cg, 'r5', 'test') // not traversed
|
||||
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, cf)
|
||||
const visibleDumps = await dumpManager.getVisibleDumps(repositoryId)
|
||||
expect(visibleDumps.map((dump: pgModels.LsifDump) => dump.id).sort()).toEqual([d1.id, d2.id])
|
||||
})
|
||||
|
||||
it('should prune overlapping roots of the same indexer during visibility check', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
// This database has the following commit graph:
|
||||
//
|
||||
// a -- b -- c -- d -- e -- f -- g
|
||||
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
const cf = util.createCommit()
|
||||
const cg = util.createCommit()
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
[cc, new Set([cb])],
|
||||
[cd, new Set([cc])],
|
||||
[ce, new Set([cd])],
|
||||
[cf, new Set([ce])],
|
||||
[cg, new Set([cf])],
|
||||
])
|
||||
)
|
||||
|
||||
// Add dumps from indexer A
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, 'r1', 'A')
|
||||
const d1 = await util.insertDump(connection, dumpManager, repositoryId, cc, 'r2', 'A')
|
||||
const d2 = await util.insertDump(connection, dumpManager, repositoryId, cd, 'r1', 'A') // overwrites r1
|
||||
const d3 = await util.insertDump(connection, dumpManager, repositoryId, cf, 'r3', 'A')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cg, 'r4', 'A') // not traversed
|
||||
|
||||
// Add dumps from indexer B
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, 'r1', 'B')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cc, 'r2', 'B')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cd, '', 'B') // overwrites r1, r2
|
||||
const d5 = await util.insertDump(connection, dumpManager, repositoryId, ce, 'r3', 'B') // overwrites ''
|
||||
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, cf)
|
||||
const visibleDumps = await dumpManager.getVisibleDumps(repositoryId)
|
||||
expect(visibleDumps.map((dump: pgModels.LsifDump) => dump.id).sort()).toEqual([d1.id, d2.id, d3.id, d5.id])
|
||||
})
|
||||
|
||||
it('should traverse branching paths during visibility check', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This database has the following commit graph:
|
||||
//
|
||||
// a --+-- [b] --- c ---+
|
||||
// | |
|
||||
// +--- d --- [e] --+ -- [h] --+-- [i]
|
||||
// | |
|
||||
// +-- [f] --- g --------------+
|
||||
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
const ch = util.createCommit()
|
||||
const ci = util.createCommit()
|
||||
const cf = util.createCommit()
|
||||
const cg = util.createCommit()
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set()],
|
||||
[cb, new Set([ca])],
|
||||
[cc, new Set([cb])],
|
||||
[cd, new Set([ca])],
|
||||
[ce, new Set([cd])],
|
||||
[ch, new Set([cc, ce])],
|
||||
[ci, new Set([ch, cg])],
|
||||
[cf, new Set([ca])],
|
||||
[cg, new Set([cf])],
|
||||
])
|
||||
)
|
||||
|
||||
// Add dumps
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, 'r2', 'test')
|
||||
const dump1 = await util.insertDump(connection, dumpManager, repositoryId, ce, 'r2/a', 'test') // overwrites r2 in commit b
|
||||
const dump2 = await util.insertDump(connection, dumpManager, repositoryId, ce, 'r2/b', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cf, 'r1/a', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cf, 'r1/b', 'test')
|
||||
const dump3 = await util.insertDump(connection, dumpManager, repositoryId, ch, 'r1', 'test') // overwrites r1/{a,b} in commit f
|
||||
const dump4 = await util.insertDump(connection, dumpManager, repositoryId, ci, 'r3', 'test')
|
||||
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, ci)
|
||||
const visibleDumps = await dumpManager.getVisibleDumps(repositoryId)
|
||||
expect(visibleDumps.map((dump: pgModels.LsifDump) => dump.id).sort()).toEqual([
|
||||
dump1.id,
|
||||
dump2.id,
|
||||
dump3.id,
|
||||
dump4.id,
|
||||
])
|
||||
})
|
||||
|
||||
it('should not set dumps visible farther than MAX_TRAVERSAL_LIMIT', async () => {
|
||||
if (!dumpManager) {
|
||||
fail('failed beforeAll')
|
||||
}
|
||||
|
||||
// This repository has the following commit graph (ancestors to the left):
|
||||
//
|
||||
// (MAX_TRAVERSAL_LIMIT + 1) -- ... -- 2 -- 1 -- 0
|
||||
//
|
||||
// Note: we use 'a' as a suffix for commit numbers on construction so that
|
||||
// we can distinguish `1` and `11` (`1a1a1a...` and `11a11a11a...`).
|
||||
|
||||
const repositoryId = nextId()
|
||||
const c0 = util.createCommit(0)
|
||||
const c1 = util.createCommit(1)
|
||||
const cpen = util.createCommit(MAX_TRAVERSAL_LIMIT - 1)
|
||||
const cmax = util.createCommit(MAX_TRAVERSAL_LIMIT)
|
||||
|
||||
const commits = new Map<string, Set<string>>(
|
||||
Array.from({ length: MAX_TRAVERSAL_LIMIT + 1 }, (_, i) => [
|
||||
util.createCommit(i),
|
||||
new Set([util.createCommit(i + 1)]),
|
||||
])
|
||||
)
|
||||
|
||||
// Add relations
|
||||
await dumpManager.updateCommits(repositoryId, commits)
|
||||
|
||||
// Add dumps
|
||||
const dump1 = await util.insertDump(connection, dumpManager, repositoryId, cmax, '', 'test')
|
||||
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, cmax)
|
||||
let visibleDumps = await dumpManager.getVisibleDumps(repositoryId)
|
||||
expect(visibleDumps.map((dump: pgModels.LsifDump) => dump.id).sort()).toEqual([dump1.id])
|
||||
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, c1)
|
||||
visibleDumps = await dumpManager.getVisibleDumps(repositoryId)
|
||||
expect(visibleDumps.map((dump: pgModels.LsifDump) => dump.id).sort()).toEqual([dump1.id])
|
||||
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, c0)
|
||||
visibleDumps = await dumpManager.getVisibleDumps(repositoryId)
|
||||
expect(visibleDumps.map((dump: pgModels.LsifDump) => dump.id).sort()).toEqual([])
|
||||
|
||||
// Add closer dump
|
||||
const dump2 = await util.insertDump(connection, dumpManager, repositoryId, cpen, '', 'test')
|
||||
|
||||
// Now commit cpen should be found
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, c0)
|
||||
visibleDumps = await dumpManager.getVisibleDumps(repositoryId)
|
||||
expect(visibleDumps.map((dump: pgModels.LsifDump) => dump.id).sort()).toEqual([dump2.id])
|
||||
})
|
||||
})
|
||||
|
||||
describe('discoverAndUpdateCommit', () => {
|
||||
let counter = 200
|
||||
const nextId = () => {
|
||||
counter++
|
||||
return counter
|
||||
}
|
||||
|
||||
it('should update tracked commits', async () => {
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
|
||||
nock('http://frontend')
|
||||
.post(`/.internal/git/${repositoryId}/exec`)
|
||||
.reply(200, `${ca}\n${cb} ${ca}\n${cc} ${cb}`)
|
||||
|
||||
const { connection, cleanup } = await util.createCleanPostgresDatabase()
|
||||
|
||||
try {
|
||||
const dumpManager = new DumpManager(connection)
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, '', 'test')
|
||||
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
await dumpManager.discoverCommits({
|
||||
repositoryId,
|
||||
commit: cc,
|
||||
frontendUrl: 'frontend',
|
||||
})
|
||||
)
|
||||
|
||||
// Ensure all commits are now tracked
|
||||
expect((await connection.getRepository(pgModels.Commit).find()).map(c => c.commit).sort()).toEqual([
|
||||
ca,
|
||||
cb,
|
||||
cc,
|
||||
])
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
it('should early-out if commit is tracked', async () => {
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
|
||||
const { connection, cleanup } = await util.createCleanPostgresDatabase()
|
||||
|
||||
try {
|
||||
const dumpManager = new DumpManager(connection)
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, '', 'test')
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([[cb, new Set()]])
|
||||
)
|
||||
|
||||
// This test ensures the following does not make a gitserver request.
|
||||
// As we did not register a nock interceptor, any request will result
|
||||
// in an exception being thrown.
|
||||
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
await dumpManager.discoverCommits({
|
||||
repositoryId,
|
||||
commit: cb,
|
||||
frontendUrl: 'frontend',
|
||||
})
|
||||
)
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
it('should early-out if repository is unknown', async () => {
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
|
||||
const { connection, cleanup } = await util.createCleanPostgresDatabase()
|
||||
|
||||
try {
|
||||
const dumpManager = new DumpManager(connection)
|
||||
|
||||
// This test ensures the following does not make a gitserver request.
|
||||
// As we did not register a nock interceptor, any request will result
|
||||
// in an exception being thrown.
|
||||
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
await dumpManager.discoverCommits({
|
||||
repositoryId,
|
||||
commit: ca,
|
||||
frontendUrl: 'frontend',
|
||||
})
|
||||
)
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('discoverAndUpdateTips', () => {
|
||||
let counter = 300
|
||||
const nextId = () => {
|
||||
counter++
|
||||
return counter
|
||||
}
|
||||
|
||||
it('should update tips', async () => {
|
||||
const repositoryId = nextId()
|
||||
const ca = util.createCommit()
|
||||
const cb = util.createCommit()
|
||||
const cc = util.createCommit()
|
||||
const cd = util.createCommit()
|
||||
const ce = util.createCommit()
|
||||
|
||||
nock('http://frontend')
|
||||
.post(`/.internal/git/${repositoryId}/exec`, { args: ['rev-parse', 'HEAD'] })
|
||||
.reply(200, ce)
|
||||
|
||||
const { connection, cleanup } = await util.createCleanPostgresDatabase()
|
||||
|
||||
try {
|
||||
const dumpManager = new DumpManager(connection)
|
||||
await dumpManager.updateCommits(
|
||||
repositoryId,
|
||||
new Map<string, Set<string>>([
|
||||
[ca, new Set<string>()],
|
||||
[cb, new Set<string>([ca])],
|
||||
[cc, new Set<string>([cb])],
|
||||
[cd, new Set<string>([cc])],
|
||||
[ce, new Set<string>([cd])],
|
||||
])
|
||||
)
|
||||
await util.insertDump(connection, dumpManager, repositoryId, ca, 'foo', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cb, 'foo', 'test')
|
||||
await util.insertDump(connection, dumpManager, repositoryId, cc, 'bar', 'test')
|
||||
|
||||
const tipCommit = await dumpManager.discoverTip({
|
||||
repositoryId,
|
||||
frontendUrl: 'frontend',
|
||||
})
|
||||
if (!tipCommit) {
|
||||
throw new Error('Expected a tip commit')
|
||||
}
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, tipCommit)
|
||||
|
||||
const d1 = await dumpManager.getDump(repositoryId, ca, 'foo/test.ts')
|
||||
const d2 = await dumpManager.getDump(repositoryId, cb, 'foo/test.ts')
|
||||
const d3 = await dumpManager.getDump(repositoryId, cc, 'bar/test.ts')
|
||||
|
||||
expect(d1?.visibleAtTip).toBeFalsy()
|
||||
expect(d2?.visibleAtTip).toBeTruthy()
|
||||
expect(d3?.visibleAtTip).toBeTruthy()
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
})
|
||||
@ -1,368 +0,0 @@
|
||||
import { uniq } from 'lodash'
|
||||
import * as sharedMetrics from '../database/metrics'
|
||||
import * as pgModels from '../models/pg'
|
||||
import { getCommitsNear, getHead } from '../gitserver/gitserver'
|
||||
import { Brackets, Connection, EntityManager } from 'typeorm'
|
||||
import { logAndTraceCall, TracingContext } from '../tracing'
|
||||
import { instrumentQuery, instrumentQueryOrTransaction, withInstrumentedTransaction } from '../database/postgres'
|
||||
import { TableInserter } from '../database/inserter'
|
||||
import { visibleDumps, ancestorLineage, bidirectionalLineage } from '../models/queries'
|
||||
import { isDefined } from '../util'
|
||||
|
||||
/** The insertion metrics for Postgres. */
|
||||
const insertionMetrics = {
|
||||
durationHistogram: sharedMetrics.postgresInsertionDurationHistogram,
|
||||
errorsCounter: sharedMetrics.postgresQueryErrorsCounter,
|
||||
}
|
||||
|
||||
/** A wrapper around the database tables that control dumps and commits. */
|
||||
export class DumpManager {
|
||||
/**
|
||||
* Create a new `DumpManager` backed by the given database connection.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
*/
|
||||
constructor(private connection: Connection) {}
|
||||
|
||||
/**
|
||||
* Find the dump for the given repository and commit.
|
||||
*
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param commit The commit.
|
||||
* @param file A filename that should be included in the dump.
|
||||
*/
|
||||
public getDump(repositoryId: number, commit: string, file: string): Promise<pgModels.LsifDump | undefined> {
|
||||
return instrumentQuery(() =>
|
||||
this.connection
|
||||
.getRepository(pgModels.LsifDump)
|
||||
.createQueryBuilder()
|
||||
.select()
|
||||
.where({ repositoryId, commit })
|
||||
.andWhere(":file LIKE (root || '%')", { file })
|
||||
.getOne()
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a dump by identifier.
|
||||
*
|
||||
* @param id The dump identifier.
|
||||
*/
|
||||
public getDumpById(id: pgModels.DumpId): Promise<pgModels.LsifDump | undefined> {
|
||||
return instrumentQuery(() => this.connection.getRepository(pgModels.LsifDump).findOne({ id }))
|
||||
}
|
||||
|
||||
/**
|
||||
* Bulk get dumps by identifier.
|
||||
*
|
||||
* @param ids The dump identifiers.
|
||||
*/
|
||||
public async getDumpsByIds(ids: pgModels.DumpId[]): Promise<Map<pgModels.DumpId, pgModels.LsifDump>> {
|
||||
if (ids.length === 0) {
|
||||
return new Map()
|
||||
}
|
||||
|
||||
const dumps = await instrumentQuery(() =>
|
||||
this.connection.getRepository(pgModels.LsifDump).createQueryBuilder().select().whereInIds(ids).getMany()
|
||||
)
|
||||
|
||||
return new Map(dumps.map(d => [d.id, d]))
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a map from upload ids to their state.
|
||||
*
|
||||
* @param ids The upload ids to fetch.
|
||||
*/
|
||||
public async getUploadStates(ids: pgModels.DumpId[]): Promise<Map<pgModels.DumpId, pgModels.LsifUploadState>> {
|
||||
if (ids.length === 0) {
|
||||
return new Map()
|
||||
}
|
||||
|
||||
const result: { id: pgModels.DumpId; state: pgModels.LsifUploadState }[] = await instrumentQuery(() =>
|
||||
this.connection
|
||||
.getRepository(pgModels.LsifUpload)
|
||||
.createQueryBuilder()
|
||||
.select(['id', 'state'])
|
||||
.where('id IN (:...ids)', { ids })
|
||||
.getRawMany()
|
||||
)
|
||||
|
||||
return new Map<pgModels.DumpId, pgModels.LsifUploadState>(result.map(u => [u.id, u.state]))
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the visible dumps. This method is used for testing.
|
||||
*
|
||||
* @param repositoryId The repository identifier.
|
||||
*/
|
||||
public getVisibleDumps(repositoryId: number): Promise<pgModels.LsifDump[]> {
|
||||
return instrumentQuery(() =>
|
||||
this.connection
|
||||
.getRepository(pgModels.LsifDump)
|
||||
.createQueryBuilder()
|
||||
.select()
|
||||
.where({ repositoryId, visibleAtTip: true })
|
||||
.getMany()
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the oldest dump that is not visible at the tip of its repository.
|
||||
*
|
||||
* @param entityManager The EntityManager to use as part of a transaction.
|
||||
*/
|
||||
public getOldestPrunableDump(
|
||||
entityManager: EntityManager = this.connection.createEntityManager()
|
||||
): Promise<pgModels.LsifDump | undefined> {
|
||||
return instrumentQuery(() =>
|
||||
entityManager
|
||||
.getRepository(pgModels.LsifDump)
|
||||
.createQueryBuilder()
|
||||
.select()
|
||||
.where({ visibleAtTip: false })
|
||||
.orderBy('uploaded_at')
|
||||
.getOne()
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the dump 'closest' to the given target commit (a direct descendant or ancestor of
|
||||
* the target commit). If no closest commit can be determined, this method returns undefined.
|
||||
*
|
||||
* This method returns dumps ordered by commit distance (nearest first).
|
||||
*
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param commit The target commit.
|
||||
* @param file One of the files in the dump.
|
||||
* @param ctx The tracing context.
|
||||
* @param frontendUrl The url of the frontend internal API.
|
||||
*/
|
||||
public async findClosestDumps(
|
||||
repositoryId: number,
|
||||
commit: string,
|
||||
file: string,
|
||||
ctx: TracingContext = {},
|
||||
frontendUrl?: string
|
||||
): Promise<pgModels.LsifDump[]> {
|
||||
// Request updated commit data from gitserver if this commit isn't already
|
||||
// tracked. This will pull back ancestors for this commit up to a certain
|
||||
// (configurable) depth and insert them into the database. This populates
|
||||
// the necessary data for the following query.
|
||||
if (frontendUrl) {
|
||||
await this.updateCommits(
|
||||
repositoryId,
|
||||
await this.discoverCommits({ repositoryId, commit, frontendUrl, ctx }),
|
||||
ctx
|
||||
)
|
||||
}
|
||||
|
||||
return logAndTraceCall(ctx, 'Finding closest dump', async () => {
|
||||
const query = `
|
||||
WITH
|
||||
${bidirectionalLineage()},
|
||||
${visibleDumps()}
|
||||
|
||||
SELECT d.dump_id FROM lineage_with_dumps d
|
||||
WHERE $3 LIKE (d.root || '%') AND d.dump_id IN (SELECT * FROM visible_ids)
|
||||
ORDER BY d.n
|
||||
`
|
||||
|
||||
return withInstrumentedTransaction(this.connection, async entityManager => {
|
||||
const results: { dump_id: number }[] = await entityManager.query(query, [repositoryId, commit, file])
|
||||
const dumpIds = results.map(({ dump_id }) => dump_id)
|
||||
if (dumpIds.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
const uniqueDumpIds = uniq(dumpIds)
|
||||
|
||||
const dumps = await entityManager
|
||||
.getRepository(pgModels.LsifDump)
|
||||
.createQueryBuilder()
|
||||
.select()
|
||||
.where('id IN (:...ids)', { ids: uniqueDumpIds })
|
||||
.getMany()
|
||||
|
||||
const dumpByID = new Map(dumps.map(dump => [dump.id, dump]))
|
||||
return uniqueDumpIds.map(id => dumpByID.get(id)).filter(isDefined)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the set of dumps which are 'visible' from the given commit and set the
|
||||
* `visible_at_tip` flags. Unset the flag for each invisible dump for this repository.
|
||||
* This will traverse all ancestor commits but not descendants, as the given commit
|
||||
* is assumed to be the tip of the default branch. For each dump that is filtered out
|
||||
* of the set of results, there must be a dump with a smaller depth from the given
|
||||
* commit that has a root that overlaps with the filtered dump. The other such dump
|
||||
* is necessarily a dump associated with a closer commit for the same root.
|
||||
*
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param commit The head of the default branch.
|
||||
* @param ctx The tracing context.
|
||||
* @param entityManager The EntityManager to use as part of a transaction.
|
||||
*/
|
||||
public updateDumpsVisibleFromTip(
|
||||
repositoryId: number,
|
||||
commit: string,
|
||||
ctx: TracingContext = {},
|
||||
entityManager: EntityManager = this.connection.createEntityManager()
|
||||
): Promise<void> {
|
||||
const query = `
|
||||
WITH
|
||||
${ancestorLineage()},
|
||||
${visibleDumps()}
|
||||
|
||||
-- Update dump records by:
|
||||
-- (1) unsetting the visibility flag of all previously visible dumps, and
|
||||
-- (2) setting the visibility flag of all currently visible dumps
|
||||
UPDATE lsif_dumps d
|
||||
SET visible_at_tip = id IN (SELECT * from visible_ids)
|
||||
WHERE d.repository_id = $1 AND (d.id IN (SELECT * from visible_ids) OR d.visible_at_tip)
|
||||
`
|
||||
|
||||
return logAndTraceCall(ctx, 'Updating dumps visible from tip', () =>
|
||||
instrumentQuery(() => entityManager.query(query, [repositoryId, commit]))
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the known commits for a repository. The input commits must be a map from commits to
|
||||
* a set of parent commits. Commits without a parent should have an empty set of parents, but
|
||||
* should still be present in the map.
|
||||
*
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param commits The commit parentage data.
|
||||
* @param ctx The tracing context.
|
||||
* @param entityManager The EntityManager to use as part of a transaction.
|
||||
*/
|
||||
public updateCommits(
|
||||
repositoryId: number,
|
||||
commits: Map<string, Set<string>>,
|
||||
ctx: TracingContext = {},
|
||||
entityManager?: EntityManager
|
||||
): Promise<void> {
|
||||
return logAndTraceCall(ctx, 'Updating commits', () =>
|
||||
instrumentQueryOrTransaction(this.connection, entityManager, async definiteEntityManager => {
|
||||
const commitInserter = new TableInserter(
|
||||
definiteEntityManager,
|
||||
pgModels.Commit,
|
||||
pgModels.Commit.BatchSize,
|
||||
insertionMetrics,
|
||||
true // Do nothing on conflict
|
||||
)
|
||||
|
||||
for (const [commit, parentCommits] of commits) {
|
||||
if (parentCommits.size === 0) {
|
||||
await commitInserter.insert({ repositoryId, commit, parentCommit: null })
|
||||
}
|
||||
|
||||
for (const parentCommit of parentCommits) {
|
||||
await commitInserter.insert({ repositoryId, commit, parentCommit })
|
||||
}
|
||||
}
|
||||
|
||||
await commitInserter.flush()
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a list of commits for the given repository with their parent starting at the
|
||||
* given commit and returning at most `MAX_COMMITS_PER_UPDATE` commits. The output
|
||||
* is a map from commits to a set of parent commits. The set of parents may be empty.
|
||||
* If we already have commit parentage information for this commit, this function
|
||||
* will do nothing.
|
||||
*
|
||||
* @param args Parameter bag.
|
||||
*/
|
||||
public async discoverCommits({
|
||||
repositoryId,
|
||||
commit,
|
||||
frontendUrl,
|
||||
ctx = {},
|
||||
}: {
|
||||
/** The repository identifier. */
|
||||
repositoryId: number
|
||||
/** The commit from which the gitserver queries should start. */
|
||||
commit: string
|
||||
/** The url of the frontend internal API. */
|
||||
frontendUrl: string
|
||||
/** The tracing context. */
|
||||
ctx?: TracingContext
|
||||
}): Promise<Map<string, Set<string>>> {
|
||||
const matchingRepos = await instrumentQuery(() =>
|
||||
this.connection.getRepository(pgModels.LsifUpload).count({ where: { repositoryId } })
|
||||
)
|
||||
if (matchingRepos === 0) {
|
||||
return new Map()
|
||||
}
|
||||
|
||||
const matchingCommits = await instrumentQuery(() =>
|
||||
this.connection.getRepository(pgModels.Commit).count({ where: { repositoryId, commit } })
|
||||
)
|
||||
if (matchingCommits > 0) {
|
||||
return new Map()
|
||||
}
|
||||
|
||||
return getCommitsNear(frontendUrl, repositoryId, commit, ctx)
|
||||
}
|
||||
|
||||
/**
|
||||
* Query gitserver for the head of the default branch for the given repository.
|
||||
*
|
||||
* @param args Parameter bag.
|
||||
*/
|
||||
public discoverTip({
|
||||
repositoryId,
|
||||
frontendUrl,
|
||||
ctx = {},
|
||||
}: {
|
||||
/** The repository identifier. */
|
||||
repositoryId: number
|
||||
/** The url of the frontend internal API. */
|
||||
frontendUrl: string
|
||||
/** The tracing context. */
|
||||
ctx?: TracingContext
|
||||
}): Promise<string | undefined> {
|
||||
return logAndTraceCall(ctx, 'Getting repository metadata', () => getHead(frontendUrl, repositoryId, ctx))
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete existing dumps from the same repo@commit and indexer that overlap with the
|
||||
* current root (where the existing root is a prefix of the current root, or vice versa).
|
||||
*
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param commit The commit.
|
||||
* @param root The root of all files that are in the dump.
|
||||
* @param indexer The indexer used to produce the dump.
|
||||
* @param ctx The tracing context.
|
||||
* @param entityManager The EntityManager to use as part of a transaction.
|
||||
*/
|
||||
public async deleteOverlappingDumps(
|
||||
repositoryId: number,
|
||||
commit: string,
|
||||
root: string,
|
||||
indexer: string | undefined,
|
||||
ctx: TracingContext = {},
|
||||
entityManager: EntityManager = this.connection.createEntityManager()
|
||||
): Promise<void> {
|
||||
return logAndTraceCall(ctx, 'Clearing overlapping dumps', () =>
|
||||
instrumentQuery(async () => {
|
||||
await entityManager
|
||||
.getRepository(pgModels.LsifUpload)
|
||||
.createQueryBuilder()
|
||||
.delete()
|
||||
.where({ repositoryId, commit, indexer, state: 'completed' })
|
||||
.andWhere(
|
||||
new Brackets(qb =>
|
||||
qb.where(":root LIKE (root || '%')", { root }).orWhere("root LIKE (:root || '%')", { root })
|
||||
)
|
||||
)
|
||||
.execute()
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,71 +0,0 @@
|
||||
import * as crc32 from 'crc-32'
|
||||
import { ADVISORY_LOCK_ID_SALT } from '../constants'
|
||||
import { Connection } from 'typeorm'
|
||||
|
||||
/**
|
||||
* Hold a Postgres advisory lock while executing the given function. Note that acquiring
|
||||
* an advisory lock is an (indefinitely) blocking operation.
|
||||
*
|
||||
* For more information, see
|
||||
* https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param name The name of the lock.
|
||||
* @param f The function to execute while holding the lock.
|
||||
*/
|
||||
export async function withLock<T>(connection: Connection, name: string, f: () => Promise<T>): Promise<T> {
|
||||
const lockId = createLockId(name)
|
||||
await connection.query('SELECT pg_advisory_lock($1)', [lockId])
|
||||
try {
|
||||
return await f()
|
||||
} finally {
|
||||
await connection.query('SELECT pg_advisory_unlock($1)', [lockId])
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hold a Postgres advisory lock while executing the given function. If the lock cannot be
|
||||
* acquired immediately, the function will return undefined without invoking the function.
|
||||
*
|
||||
* For more information, see
|
||||
* https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param name The name of the lock.
|
||||
* @param f The function to execute while holding the lock.
|
||||
*/
|
||||
export async function tryWithLock<T>(
|
||||
connection: Connection,
|
||||
name: string,
|
||||
f: () => Promise<T>
|
||||
): Promise<T | undefined> {
|
||||
const lockId = createLockId(name)
|
||||
if (await connection.query('SELECT pg_try_advisory_lock($1)', [lockId])) {
|
||||
try {
|
||||
return await f()
|
||||
} finally {
|
||||
await connection.query('SELECT pg_advisory_unlock($1)', [lockId])
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an integer identifier that will be unique to this app, but will always be the same for this given
|
||||
* name within the application.
|
||||
*
|
||||
* We base our advisory lock identifier generation technique on golang-migrate. For the original source, see
|
||||
* https://github.com/golang-migrate/migrate/blob/6c96ef02dfbf9430f7286b58afc15718588f2e13/database/util.go#L12.
|
||||
*
|
||||
* Advisory lock ids should be deterministically generated such that a single app will return the same lock id
|
||||
* for the same name, but distinct apps are unlikely to generate the same id (using the same name or not). To
|
||||
* accomplish this, we hash the name into an integer, then multiply it by some app-specific salt to reduce the
|
||||
* collision space with another application. Each app should choose a salt uniformly at random. This
|
||||
* application's salt is distinct from the golang-migrate salt.
|
||||
*
|
||||
* @param name The name of the lock.
|
||||
*/
|
||||
function createLockId(name: string): number {
|
||||
return crc32.str(name) * ADVISORY_LOCK_ID_SALT
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
import promClient from 'prom-client'
|
||||
|
||||
//
|
||||
// Bloom Filter Metrics
|
||||
|
||||
export const bloomFilterEventsCounter = new promClient.Counter({
|
||||
name: 'lsif_bloom_filter_events_total',
|
||||
help: 'The number of bloom filter hits and misses.',
|
||||
labelNames: ['type'],
|
||||
})
|
||||
@ -1,71 +0,0 @@
|
||||
import AsyncPolling from 'async-polling'
|
||||
import { Connection } from 'typeorm'
|
||||
import { logAndTraceCall, TracingContext } from './tracing'
|
||||
import { Logger } from 'winston'
|
||||
import { tryWithLock } from './store/locks'
|
||||
|
||||
interface Task {
|
||||
intervalMs: number
|
||||
handler: () => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* A collection of tasks that are invoked periodically, each holding an
|
||||
* exclusive advisory lock on a Postgres database connection.
|
||||
*/
|
||||
export class ExclusivePeriodicTaskRunner {
|
||||
private tasks: Task[] = []
|
||||
|
||||
/**
|
||||
* Create a new task runner.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param logger The logger instance.
|
||||
*/
|
||||
constructor(private connection: Connection, private logger: Logger) {}
|
||||
|
||||
/**
|
||||
* Register a task to be performed while holding an exclusive advisory lock in Postgres.
|
||||
*
|
||||
* @param args Parameter bag
|
||||
*/
|
||||
public register({
|
||||
/** The task name. */
|
||||
name,
|
||||
/** The interval between task invocations. */
|
||||
intervalMs,
|
||||
/** The function to invoke. */
|
||||
task,
|
||||
/** Whether or not to silence logging. */
|
||||
silent = false,
|
||||
}: {
|
||||
name: string
|
||||
intervalMs: number
|
||||
task: ({ connection, ctx }: { connection: Connection; ctx: TracingContext }) => Promise<void>
|
||||
silent?: boolean
|
||||
}): void {
|
||||
const taskArgs = { connection: this.connection, ctx: {} }
|
||||
|
||||
this.tasks.push({
|
||||
intervalMs,
|
||||
handler: () =>
|
||||
tryWithLock(this.connection, name, () =>
|
||||
silent
|
||||
? task(taskArgs)
|
||||
: logAndTraceCall({ logger: this.logger }, name, ctx => task({ ...taskArgs, ctx }))
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
/** Start running all registered tasks on the specified interval. */
|
||||
public run(): void {
|
||||
for (const { intervalMs, handler } of this.tasks) {
|
||||
const fn = async (end: () => void): Promise<void> => {
|
||||
await handler()
|
||||
end()
|
||||
}
|
||||
|
||||
AsyncPolling(fn, intervalMs * 1000).run()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,170 +0,0 @@
|
||||
import * as fs from 'mz/fs'
|
||||
import * as nodepath from 'path'
|
||||
import * as uuid from 'uuid'
|
||||
import * as pgModels from './models/pg'
|
||||
import { child_process } from 'mz'
|
||||
import { Connection } from 'typeorm'
|
||||
import { connectPostgres } from './database/postgres'
|
||||
import { userInfo } from 'os'
|
||||
import { DumpManager } from './store/dumps'
|
||||
import { createSilentLogger } from './logging'
|
||||
|
||||
/**
|
||||
* Create a new postgres database with a random suffix, apply the frontend
|
||||
* migrations (via the ./dev/migrate.sh script) and return an open connection.
|
||||
* This uses the PG* environment variables for host, port, user, and password.
|
||||
* This also returns a cleanup function that will destroy the database, which
|
||||
* should be called at the end of the test.
|
||||
*/
|
||||
export async function createCleanPostgresDatabase(): Promise<{ connection: Connection; cleanup: () => Promise<void> }> {
|
||||
// Each test has a random dbname suffix
|
||||
const suffix = uuid.v4().substring(0, 8)
|
||||
|
||||
// Pull test db config from environment
|
||||
const host = process.env.PGHOST || 'localhost'
|
||||
const port = parseInt(process.env.PGPORT || '5432', 10)
|
||||
const username = process.env.PGUSER || userInfo().username || 'postgres'
|
||||
const password = process.env.PGPASSWORD || ''
|
||||
const database = `sourcegraph-test-lsif-${suffix}`
|
||||
|
||||
// Determine the path of the migrate script. This will cover the case where `yarn test` is
|
||||
// run from within the root or from the precise-code-intel directory.
|
||||
const migrationsPath = nodepath.join((await fs.exists('migrations')) ? '' : '../..', 'migrations')
|
||||
|
||||
// Ensure environment gets passed to child commands
|
||||
const env = {
|
||||
...process.env,
|
||||
PGHOST: host,
|
||||
PGPORT: `${port}`,
|
||||
PGUSER: username,
|
||||
PGPASSWORD: password,
|
||||
PGSSLMODE: 'disable',
|
||||
PGDATABASE: database,
|
||||
}
|
||||
|
||||
// Construct postgres connection string using environment above. We disable this
|
||||
// eslint rule because we want it to use bash interpolation, not typescript string
|
||||
// templates.
|
||||
//
|
||||
// eslint-disable-next-line no-template-curly-in-string
|
||||
const connectionString = 'postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE}?sslmode=disable'
|
||||
|
||||
// Define command text
|
||||
const createCommand = `createdb ${database}`
|
||||
const dropCommand = `dropdb --if-exists ${database}`
|
||||
const migrateCommand = `migrate -database "${connectionString}" -path ${migrationsPath} up`
|
||||
|
||||
// Create cleanup function to run after test. This will close the connection
|
||||
// created below (if successful), then destroy the database that was created
|
||||
// for the test. It is necessary to close the database first, otherwise we
|
||||
// get failures during the after hooks:
|
||||
//
|
||||
// dropdb: database removal failed: ERROR: database "sourcegraph-test-lsif-5033c9e8" is being accessed by other users
|
||||
|
||||
let connection: Connection
|
||||
const cleanup = async (): Promise<void> => {
|
||||
if (connection) {
|
||||
await connection.close()
|
||||
}
|
||||
|
||||
await child_process.exec(dropCommand, { env }).then(() => undefined)
|
||||
}
|
||||
|
||||
// Try to create database
|
||||
await child_process.exec(createCommand, { env })
|
||||
|
||||
try {
|
||||
// Run migrations then connect to database
|
||||
await child_process.exec(migrateCommand, { env })
|
||||
connection = await connectPostgres(
|
||||
{ host, port, username, password, database, ssl: false },
|
||||
suffix,
|
||||
createSilentLogger()
|
||||
)
|
||||
return { connection, cleanup }
|
||||
} catch (error) {
|
||||
// We made a database but can't use it - try to clean up
|
||||
// before throwing the original error.
|
||||
|
||||
try {
|
||||
await cleanup()
|
||||
} catch (_) {
|
||||
// If a new error occurs, swallow it
|
||||
}
|
||||
|
||||
// Throw the original error
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate all tables that do not match `schema_migrations`.
|
||||
*
|
||||
* @param connection The connection to use.
|
||||
*/
|
||||
export async function truncatePostgresTables(connection: Connection): Promise<void> {
|
||||
const results: { table_name: string }[] = await connection.query(
|
||||
"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE' AND table_name != 'schema_migrations'"
|
||||
)
|
||||
|
||||
const tableNames = results.map(row => row.table_name).join(', ')
|
||||
await connection.query(`truncate ${tableNames} restart identity`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert an upload entity and return the corresponding dump entity.
|
||||
*
|
||||
* @param connection The Postgres connection.
|
||||
* @param dumpManager The dumps manager instance.
|
||||
* @param repositoryId The repository identifier.
|
||||
* @param commit The commit.
|
||||
* @param root The root of all files in the dump.
|
||||
* @param indexer The type of indexer used to produce this dump.
|
||||
*/
|
||||
export async function insertDump(
|
||||
connection: Connection,
|
||||
dumpManager: DumpManager,
|
||||
repositoryId: number,
|
||||
commit: string,
|
||||
root: string,
|
||||
indexer: string
|
||||
): Promise<pgModels.LsifDump> {
|
||||
await dumpManager.deleteOverlappingDumps(repositoryId, commit, root, indexer, {})
|
||||
|
||||
const upload = new pgModels.LsifUpload()
|
||||
upload.repositoryId = repositoryId
|
||||
upload.commit = commit
|
||||
upload.root = root
|
||||
upload.indexer = indexer
|
||||
upload.uploadedAt = new Date()
|
||||
upload.state = 'completed'
|
||||
upload.numParts = 1
|
||||
upload.uploadedParts = [0]
|
||||
await connection.createEntityManager().save(upload)
|
||||
|
||||
const dump = new pgModels.LsifDump()
|
||||
dump.id = upload.id
|
||||
dump.repositoryId = repositoryId
|
||||
dump.commit = commit
|
||||
dump.root = root
|
||||
dump.indexer = indexer
|
||||
return dump
|
||||
}
|
||||
|
||||
/** A counter used for unique commit generation. */
|
||||
let commitBase = 0
|
||||
|
||||
/**
|
||||
* Create a 40-character commit.
|
||||
*
|
||||
* @param base A unique numeric base to repeat.
|
||||
*/
|
||||
export function createCommit(base?: number): string {
|
||||
if (base === undefined) {
|
||||
base = commitBase
|
||||
commitBase++
|
||||
}
|
||||
|
||||
// Add 'a' to differentiate between similar numeric bases such as `1a1a...` and `11a11a...`.
|
||||
return (base + 'a').repeat(40).substring(0, 40)
|
||||
}
|
||||
@ -1,117 +0,0 @@
|
||||
import { createSilentLogger, logCall } from './logging'
|
||||
import { ERROR } from 'opentracing/lib/ext/tags'
|
||||
import { initTracerFromEnv } from 'jaeger-client'
|
||||
import { Logger } from 'winston'
|
||||
import { Span, Tracer } from 'opentracing'
|
||||
|
||||
/**
|
||||
* A bag of logging and tracing instances passed around a current
|
||||
* HTTP request or upload conversion.
|
||||
*/
|
||||
export interface TracingContext {
|
||||
/** The current tagged logger instance. Optional for testing. */
|
||||
logger?: Logger
|
||||
|
||||
/** The current opentracing span. Optional for testing. */
|
||||
span?: Span
|
||||
}
|
||||
|
||||
/**
|
||||
* Add tags to the logger and span. Returns a new context.
|
||||
*
|
||||
* @param ctx The tracing context.
|
||||
* @param tags The tags to add to the logger and span.
|
||||
*/
|
||||
export function addTags(
|
||||
{ logger = createSilentLogger(), span = new Span() }: TracingContext,
|
||||
tags: { [name: string]: unknown }
|
||||
): TracingContext {
|
||||
return { logger: logger.child(tags), span: span.addTags(tags) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs an event to the span of The tracing context, if its defined.
|
||||
*
|
||||
* @param ctx The tracing context.
|
||||
* @param event The name of the event.
|
||||
* @param pairs The values to log.
|
||||
*/
|
||||
export function logSpan(
|
||||
{ span = new Span() }: TracingContext,
|
||||
event: string,
|
||||
pairs: { [name: string]: unknown }
|
||||
): void {
|
||||
span.log({ event, ...pairs })
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a distributed tracer.
|
||||
*
|
||||
* @param serviceName The name of the process.
|
||||
* @param configuration The current configuration.
|
||||
*/
|
||||
export function createTracer(
|
||||
serviceName: string,
|
||||
{
|
||||
useJaeger,
|
||||
}: {
|
||||
/** Whether or not to enable Jaeger. */
|
||||
useJaeger: boolean
|
||||
}
|
||||
): Tracer | undefined {
|
||||
if (useJaeger) {
|
||||
const config = {
|
||||
serviceName,
|
||||
sampler: {
|
||||
type: 'const',
|
||||
param: 1,
|
||||
},
|
||||
}
|
||||
|
||||
return initTracerFromEnv(config, {})
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Trace an operation.
|
||||
*
|
||||
* @param name The log message to output.
|
||||
* @param parent The parent span instance.
|
||||
* @param f The operation to perform.
|
||||
*/
|
||||
export async function traceCall<T>(name: string, parent: Span, f: (span: Span) => Promise<T> | T): Promise<T> {
|
||||
const span = parent.tracer().startSpan(name, { childOf: parent })
|
||||
|
||||
try {
|
||||
return await f(span)
|
||||
} catch (error) {
|
||||
span.setTag(ERROR, true)
|
||||
span.log({
|
||||
event: 'error',
|
||||
'error.object': error,
|
||||
stack: error.stack,
|
||||
message: error.message,
|
||||
})
|
||||
|
||||
throw error
|
||||
} finally {
|
||||
span.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Log and trace the execution of a function.
|
||||
*
|
||||
* @param ctx The tracing context.
|
||||
* @param name The name of the span and text of the log message.
|
||||
* @param f The function to invoke.
|
||||
*/
|
||||
export function logAndTraceCall<T>(
|
||||
{ logger = createSilentLogger(), span = new Span() }: TracingContext,
|
||||
name: string,
|
||||
f: (ctx: TracingContext) => Promise<T> | T
|
||||
): Promise<T> {
|
||||
return logCall(name, logger, () => traceCall(name, span, childSpan => f({ logger, span: childSpan })))
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
/**
|
||||
* Returns true if the given value is not undefined.
|
||||
*
|
||||
* @param value The value to test.
|
||||
*/
|
||||
export function isDefined<T>(value: T | undefined): value is T {
|
||||
return value !== undefined
|
||||
}
|
||||
@ -1,79 +0,0 @@
|
||||
import { EntityManager } from 'typeorm'
|
||||
import { DumpManager } from './store/dumps'
|
||||
import { TracingContext } from './tracing'
|
||||
|
||||
/**
|
||||
* Update the commits for this repo, and update the visible_at_tip flag on the dumps
|
||||
* of this repository. This will query for commits starting from both the current tip
|
||||
* of the repo and from given commit.
|
||||
*
|
||||
* @param args Parameter bag.
|
||||
*/
|
||||
export async function updateCommitsAndDumpsVisibleFromTip({
|
||||
entityManager,
|
||||
dumpManager,
|
||||
frontendUrl,
|
||||
repositoryId,
|
||||
commit,
|
||||
ctx = {},
|
||||
}: {
|
||||
/** The EntityManager to use as part of a transaction. */
|
||||
entityManager: EntityManager
|
||||
/** The dumps manager instance. */
|
||||
dumpManager: DumpManager
|
||||
/** The url of the frontend internal API. */
|
||||
frontendUrl: string
|
||||
/** The repository id. */
|
||||
repositoryId: number
|
||||
/**
|
||||
* An optional commit. This should be supplied if an upload was just
|
||||
* processed. If no commit is supplied, then the commits will be queried
|
||||
* only from the tip commit of the default branch.
|
||||
*/
|
||||
commit?: string
|
||||
/** The tracing context. */
|
||||
ctx?: TracingContext
|
||||
}): Promise<void> {
|
||||
const tipCommit = await dumpManager.discoverTip({
|
||||
repositoryId,
|
||||
frontendUrl,
|
||||
ctx,
|
||||
})
|
||||
if (tipCommit === undefined) {
|
||||
throw new Error('No tip commit available for repository')
|
||||
}
|
||||
|
||||
const commits = commit
|
||||
? await dumpManager.discoverCommits({
|
||||
repositoryId,
|
||||
commit,
|
||||
frontendUrl,
|
||||
ctx,
|
||||
})
|
||||
: new Map()
|
||||
|
||||
if (tipCommit !== commit) {
|
||||
// If the tip is ahead of this commit, we also want to discover all of
|
||||
// the commits between this commit and the tip so that we can accurately
|
||||
// determine what is visible from the tip. If we do not do this before the
|
||||
// updateDumpsVisibleFromTip call below, no dumps will be reachable from
|
||||
// the tip and all dumps will be invisible.
|
||||
|
||||
const tipCommits = await dumpManager.discoverCommits({
|
||||
repositoryId,
|
||||
commit: tipCommit,
|
||||
frontendUrl,
|
||||
ctx,
|
||||
})
|
||||
|
||||
for (const [k, v] of tipCommits.entries()) {
|
||||
commits.set(
|
||||
k,
|
||||
new Set<string>([...(commits.get(k) || []), ...v])
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
await dumpManager.updateCommits(repositoryId, commits, ctx, entityManager)
|
||||
await dumpManager.updateDumpsVisibleFromTip(repositoryId, tipCommit, ctx, entityManager)
|
||||
}
|
||||
@ -1,107 +0,0 @@
|
||||
import { createBatcher } from './batch'
|
||||
import { range } from 'lodash'
|
||||
|
||||
describe('createBatcher', () => {
|
||||
it('should traverse entire tree', () => {
|
||||
const values = gatherValues('foo', [
|
||||
...range(10).map(i => `bar/${i}.ts`),
|
||||
...range(10).map(i => `bar/baz/${i}.ts`),
|
||||
...range(10).map(i => `bar/baz/bonk/${i}.ts`),
|
||||
])
|
||||
|
||||
expect(values).toEqual([[''], ['foo'], ['foo/bar'], ['foo/bar/baz'], ['foo/bar/baz/bonk']])
|
||||
})
|
||||
|
||||
it('should batch entries at same depth', () => {
|
||||
const values = gatherValues(
|
||||
'foo',
|
||||
['bar', 'baz', 'bonk'].map(d => `${d}/sub/file.ts`)
|
||||
)
|
||||
|
||||
expect(values).toEqual([
|
||||
[''],
|
||||
['foo'],
|
||||
['foo/bar', 'foo/baz', 'foo/bonk'],
|
||||
['foo/bar/sub', 'foo/baz/sub', 'foo/bonk/sub'],
|
||||
])
|
||||
})
|
||||
|
||||
it('should batch entries at same depth (wide)', () => {
|
||||
const is = range(0, 5)
|
||||
const ds = ['bar', 'baz', 'bonk']
|
||||
|
||||
const values = gatherValues(
|
||||
'foo',
|
||||
ds.flatMap(d => is.map(i => `${d}/${i}/file.ts`))
|
||||
)
|
||||
|
||||
expect(values).toEqual([
|
||||
[''],
|
||||
['foo'],
|
||||
['foo/bar', 'foo/baz', 'foo/bonk'],
|
||||
[
|
||||
'foo/bar/0',
|
||||
'foo/bar/1',
|
||||
'foo/bar/2',
|
||||
'foo/bar/3',
|
||||
'foo/bar/4',
|
||||
'foo/baz/0',
|
||||
'foo/baz/1',
|
||||
'foo/baz/2',
|
||||
'foo/baz/3',
|
||||
'foo/baz/4',
|
||||
'foo/bonk/0',
|
||||
'foo/bonk/1',
|
||||
'foo/bonk/2',
|
||||
'foo/bonk/3',
|
||||
'foo/bonk/4',
|
||||
],
|
||||
])
|
||||
})
|
||||
|
||||
it('should cut subtrees that do not exist', () => {
|
||||
const ds = ['bar', 'baz', 'bonk']
|
||||
const ss = ['a', 'b', 'c']
|
||||
const is = range(1, 4)
|
||||
|
||||
const blacklist = ['foo/bar', 'foo/baz/a', 'foo/bonk/a/1', 'foo/bonk/a/2', 'foo/bonk/b/1', 'foo/bonk/b/3']
|
||||
|
||||
const values = gatherValues(
|
||||
'foo',
|
||||
ds.flatMap(d => ss.flatMap(s => is.map(i => `${d}/${s}/${i}/sub/file.ts`))),
|
||||
blacklist
|
||||
)
|
||||
|
||||
const prune = (paths: string[]): string[] =>
|
||||
// filter out all proper descendants of the blacklist
|
||||
paths.filter(p => blacklist.includes(p) || !blacklist.some(b => p.includes(b)))
|
||||
|
||||
expect(values).toEqual([
|
||||
[''],
|
||||
['foo'],
|
||||
prune(ds.map(d => `foo/${d}`)),
|
||||
prune(ds.flatMap(d => ss.map(s => `foo/${d}/${s}`))),
|
||||
prune(ds.flatMap(d => ss.flatMap(s => is.map(i => `foo/${d}/${s}/${i}`)))),
|
||||
prune(ds.flatMap(d => ss.flatMap(s => is.map(i => `foo/${d}/${s}/${i}/sub`)))),
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
function gatherValues(root: string, documentPaths: string[], blacklist: string[] = []): string[][] {
|
||||
const batcher = createBatcher(root, documentPaths)
|
||||
|
||||
let done: boolean | undefined
|
||||
let batch: string[] | void | undefined
|
||||
|
||||
const all = []
|
||||
while (true) {
|
||||
;({ value: batch, done } = batcher.next((batch || []).filter(x => !blacklist?.includes(x))))
|
||||
if (done || !batch) {
|
||||
break
|
||||
}
|
||||
|
||||
all.push(batch)
|
||||
}
|
||||
|
||||
return all
|
||||
}
|
||||
@ -1,100 +0,0 @@
|
||||
import * as path from 'path'
|
||||
import { dirnameWithoutDot } from './paths'
|
||||
|
||||
/**
|
||||
* Create a directory tree from the complete set of file paths in an LSIF dump.
|
||||
* Returns a generator that will perform a breadth-first traversal of the tree.
|
||||
*
|
||||
* @param root The LSIF dump root.
|
||||
* @param documentPaths The set of file paths in an LSIF dump.
|
||||
*/
|
||||
export function createBatcher(root: string, documentPaths: string[]): Generator<string[], void, string[]> {
|
||||
return traverse(createTree(root, documentPaths))
|
||||
}
|
||||
|
||||
/** A node in a directory path tree. */
|
||||
interface Node {
|
||||
/** The name of a directory in this level of the directory tree. */
|
||||
dirname: string
|
||||
|
||||
/** The segments directly nested in this node. */
|
||||
children: Node[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a directory tree from the complete set of file paths in an LSIF dump.
|
||||
*
|
||||
* @param root The LSIF dump root.
|
||||
* @param documentPaths The set of file paths in an LSIF dump.
|
||||
*/
|
||||
function createTree(root: string, documentPaths: string[]): Node {
|
||||
// Construct the the set of root-relative parent directories for each path
|
||||
const documentDirs = documentPaths.map(documentPath => dirnameWithoutDot(path.join(root, documentPath)))
|
||||
// Deduplicate and throw out any directory that is outside of the dump root
|
||||
const dirs = Array.from(new Set(documentDirs)).filter(dirname => !dirname.startsWith('..'))
|
||||
|
||||
// Construct the canned root node
|
||||
const rootNode: Node = { dirname: '', children: [] }
|
||||
|
||||
for (const dir of dirs) {
|
||||
// Skip the dump root as the following loop body would make
|
||||
// it a child of itself. This is a non-obvious edge condition.
|
||||
if (dir === '') {
|
||||
continue
|
||||
}
|
||||
|
||||
let node = rootNode
|
||||
|
||||
// For each path segment in this directory, traverse down the
|
||||
// tree. Any node that doesn't exist is created with empty
|
||||
// children.
|
||||
|
||||
for (const dirname of dir.split('/')) {
|
||||
let child = node.children.find(n => n.dirname === dirname)
|
||||
if (!child) {
|
||||
child = { dirname, children: [] }
|
||||
node.children.push(child)
|
||||
}
|
||||
|
||||
node = child
|
||||
}
|
||||
}
|
||||
|
||||
return rootNode
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform a breadth-first traversal of a directory tree. Returns a generator that
|
||||
* acts as a coroutine visitor. The generator first yields the root (empty) path,
|
||||
* then waits for the caller to return the set of paths from the last batch that
|
||||
* exist in git. The next batch returned will only include the paths that are
|
||||
* properly nested under a previously returned value.
|
||||
*
|
||||
* @param root The root node.
|
||||
*/
|
||||
function* traverse(root: Node): Generator<string[], void, string[]> {
|
||||
// The frontier is the candidate batch that will be returned in the next
|
||||
// call to the generator. This is a superset of paths that will actually
|
||||
// be returned after pruning non-existent paths.
|
||||
let frontier: [string, Node[]][] = [['', root.children]]
|
||||
|
||||
while (frontier.length > 0) {
|
||||
// Yield our current batch and wait for the caller to return the set
|
||||
// of previously yielded paths that exist.
|
||||
const exists = yield frontier.map(([parent]) => parent)
|
||||
|
||||
frontier = frontier
|
||||
// Remove any children from the frontier that are not a proper
|
||||
// descendant of some path that was confirmed to be exist. This
|
||||
// will stop us from iterating children that definitely don't
|
||||
// exist as the parent is already known to be an un-tracked path.
|
||||
.filter(([parent]) => exists.includes(parent))
|
||||
.flatMap(([parent, children]) =>
|
||||
// Join the current path to a node with the node's segment to
|
||||
// get the full path to that node. Create the new frontier from
|
||||
// the current frontier's children after pruning the non-existent
|
||||
// paths.
|
||||
children.map((child): [string, Node[]] => [path.join(parent, child.dirname), child.children])
|
||||
)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user