mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 17:51:57 +00:00
codeintel: Register SCIP oobmigration (#45106)
This commit is contained in:
Eric Fritz
GitHub
parent
e7d1a9a57a
commit
a528304b51
@ -19,6 +19,7 @@ All notable changes to Sourcegraph are documented in this file.
|
||||
|
||||
- The default author and email for changesets will now be pulled from user account details when possible. [#46385](https://github.com/sourcegraph/sourcegraph/pull/46385)
|
||||
- Code Insights has a new display option: "Max number of series points to display". This setting controls the number of data points you see per series on an insight. [#46653](https://github.com/sourcegraph/sourcegraph/pull/46653)
|
||||
- Added out-of-band migration that will migrate all existing data from LSIF to SCIP (see additional [migration documentation](https://docs.sourcegraph.com/admin/how-to/lsif_scip_migration)). [#45106](https://github.com/sourcegraph/sourcegraph/pull/45106)
|
||||
|
||||
### Changed
|
||||
|
||||
|
||||
25
dev/ci/integration/code-intel/install-src.sh
Executable file
25
dev/ci/integration/code-intel/install-src.sh
Executable file
@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script is called by test.sh and preprod-run.sh to install an up-to-date
|
||||
# version of src-cli as required by the codeintel-qa pipeline. The target binary
|
||||
# is installed to {REPO_ROOT}/.bin/src.
|
||||
|
||||
set -eux
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../../.."
|
||||
root_dir="$(pwd)"
|
||||
|
||||
# By default, the commit that added handleSCIP support
|
||||
VERSION=${1:-'1c70d536b4ab3187b5aed41af8f259f1b8ceba6b'}
|
||||
|
||||
TEMP=$(mktemp -d -t sgdockerbuild_XXXXXXX)
|
||||
cleanup() {
|
||||
rm -rf "${TEMP}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
git clone git@github.com:sourcegraph/src-cli.git "${TEMP}" --depth 1
|
||||
pushd "${TEMP}"
|
||||
git checkout "${VERSION}"
|
||||
mkdir -p "${root_dir}/.bin"
|
||||
go build -o "${root_dir}/.bin" ./cmd/src
|
||||
popd
|
||||
@ -4,8 +4,8 @@
|
||||
# against the preprod.
|
||||
|
||||
set -eux
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../../.."
|
||||
root_dir=$(pwd)
|
||||
|
||||
SOURCEGRAPH_BASE_URL="https://preview.sgdev.dev"
|
||||
TEST_USER_EMAIL="testadmin@preview.sgdev.dev"
|
||||
@ -18,21 +18,30 @@ export SOURCEGRAPH_SUDO_USER
|
||||
export TEST_USER_PASSWORD
|
||||
export SOURCEGRAPH_SUDO_TOKEN
|
||||
|
||||
echo "--- :go: Building init-sg"
|
||||
echo '--- :go: Building init-sg'
|
||||
go build -o init-sg ./internal/cmd/init-sg/...
|
||||
|
||||
echo "--- :horse: Running init-sg addRepos"
|
||||
echo '--- :horse: Running init-sg addRepos'
|
||||
./init-sg addRepos -config ./dev/ci/integration/code-intel/repos.json
|
||||
|
||||
echo '--- Installing local src-cli'
|
||||
./dev/ci/integration/code-intel/install-src.sh
|
||||
which src
|
||||
src version
|
||||
|
||||
echo '--- :brain: Running the test suite'
|
||||
pushd dev/codeintel-qa
|
||||
|
||||
echo "--- :brain: Running the test suite"
|
||||
echo '--- :zero: downloading test data from GCS'
|
||||
go run ./cmd/download
|
||||
|
||||
echo '--- :one: clearing existing state'
|
||||
go run ./cmd/clear
|
||||
|
||||
echo '--- :two: integration test ./dev/codeintel-qa/cmd/upload'
|
||||
go run ./cmd/upload --timeout=5m
|
||||
env PATH="${root_dir}/.bin:${PATH}" go run ./cmd/upload --timeout=5m
|
||||
|
||||
# make queries but do not assert against expected locations
|
||||
echo '--- :three: integration test ./dev/codeintel-qa/cmd/query'
|
||||
go run ./cmd/query -check-query-result=false # make queries but do not assert against expected locations
|
||||
go run ./cmd/query -check-query-result=false
|
||||
popd
|
||||
|
||||
@ -8,7 +8,11 @@
|
||||
"sourcegraph-testing/etcd",
|
||||
"sourcegraph-testing/tidb",
|
||||
"sourcegraph-testing/titan",
|
||||
"sourcegraph-testing/zap"
|
||||
"sourcegraph-testing/zap",
|
||||
"sourcegraph-testing/nacelle",
|
||||
"sourcegraph-testing/nacelle-config",
|
||||
"sourcegraph-testing/nacelle-service",
|
||||
"sourcegraph/code-intel-extensions"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,36 +3,56 @@
|
||||
# This script runs the codeintel-qa tests against a running server.
|
||||
# This script is invoked by ./dev/ci/integration/run-integration.sh after running an instance.
|
||||
|
||||
set -eux
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/../../../.."
|
||||
root_dir=$(pwd)
|
||||
set -e
|
||||
|
||||
export SOURCEGRAPH_BASE_URL="${1:-"http://localhost:7080"}"
|
||||
SOURCEGRAPH_BASE_URL="${1:-"http://localhost:7080"}"
|
||||
export SOURCEGRAPH_BASE_URL
|
||||
|
||||
echo '--- initializing Sourcegraph instance'
|
||||
echo '--- :go: Building init-sg'
|
||||
go build -o init-sg ./internal/cmd/init-sg/...
|
||||
|
||||
pushd internal/cmd/init-sg
|
||||
go build -o "${root_dir}/init-sg"
|
||||
popd
|
||||
|
||||
pushd dev/ci/integration/code-intel
|
||||
echo '--- Initializing instance'
|
||||
"${root_dir}/init-sg" initSG
|
||||
# Disable `-x` to avoid printing secrets
|
||||
set +x
|
||||
|
||||
echo '--- Loading secrets'
|
||||
set +x # Avoid printing secrets
|
||||
# shellcheck disable=SC1091
|
||||
source /root/.sg_envrc
|
||||
"${root_dir}/init-sg" addRepos -config repos.json
|
||||
popd
|
||||
set -x
|
||||
|
||||
echo '--- :horse: Running init-sg addRepos'
|
||||
"${root_dir}/init-sg" addRepos -config ./dev/ci/integration/code-intel/repos.json
|
||||
|
||||
echo '--- Installing local src-cli'
|
||||
./dev/ci/integration/code-intel/install-src.sh
|
||||
which src
|
||||
src version
|
||||
|
||||
echo '--- :brain: Running the test suite'
|
||||
pushd dev/codeintel-qa
|
||||
|
||||
echo "--- :brain: Running the test suite"
|
||||
echo '--- :zero: downloading test data from GCS'
|
||||
go run ./cmd/download
|
||||
|
||||
echo '--- :one: clearing existing state'
|
||||
go run ./cmd/clear
|
||||
echo '--- :two: integration test ./dev/codeintel-qa/cmd/upload'
|
||||
go run ./cmd/upload --timeout=5m -verbose
|
||||
echo '--- :three: integration test ./dev/codeintel-qa/cmd/query'
|
||||
go run ./cmd/query -verbose
|
||||
|
||||
# Disable migration #20 (LSIF -> SCIP)
|
||||
echo '--- :two: Disabling LSIF -> SCIP migration'
|
||||
"${root_dir}/init-sg" oobmigration -id T3V0T2ZCYW5kTWlncmF0aW9uOjIw -down
|
||||
|
||||
echo '--- :three: integration test ./dev/codeintel-qa/cmd/upload'
|
||||
env PATH="${root_dir}/.bin:${PATH}" go run ./cmd/upload --timeout=5m
|
||||
|
||||
echo '--- :four: integration test ./dev/codeintel-qa/cmd/query'
|
||||
go run ./cmd/query
|
||||
|
||||
# Enable migration #20 (LSIF -> SCIP) and wait for it to complete
|
||||
echo '--- :five: Running LSIF -> SCIP migration'
|
||||
"${root_dir}/init-sg" oobmigration -id T3V0T2ZCYW5kTWlncmF0aW9uOjIw
|
||||
|
||||
echo '--- :six: integration test ./dev/codeintel-qa/cmd/query'
|
||||
go run ./cmd/query
|
||||
popd
|
||||
|
||||
@ -8,6 +8,8 @@ Ensure that the following tools are available on your path:
|
||||
|
||||
- [`src`](https://github.com/sourcegraph/src-cli)
|
||||
- [`lsif-go`](https://github.com/sourcegraph/lsif-go)
|
||||
- [`scip-go`](https://github.com/sourcegraph/scip-go)
|
||||
- [`scip-typescript`](https://github.com/sourcegraph/scip-typescript)
|
||||
|
||||
You should have enviornment variables that authenticate you to the `sourcegraph-dev` GCS project if you plan to upload or download index files (as we do in CI).
|
||||
|
||||
@ -27,7 +29,11 @@ SOURCEGRAPH_SUDO_TOKEN=<YOUR SOURCEGRAPH API ACCESS TOKEN>
|
||||
"sourcegraph-testing/etcd",
|
||||
"sourcegraph-testing/tidb",
|
||||
"sourcegraph-testing/titan",
|
||||
"sourcegraph-testing/zap"
|
||||
"sourcegraph-testing/zap",
|
||||
"sourcegraph-testing/nacelle",
|
||||
"sourcegraph-testing/nacelle-config",
|
||||
"sourcegraph-testing/nacelle-service",
|
||||
"sourcegraph/code-intel-extensions"
|
||||
],
|
||||
```
|
||||
|
||||
|
||||
@ -80,6 +80,42 @@ var repositoryMeta = []struct {
|
||||
"f8307e394c512b4263fc0cd67ccf9fd46f1ad9a5",
|
||||
},
|
||||
},
|
||||
|
||||
// These repositories have their module names modified and new tags created to refer to each other
|
||||
{
|
||||
org: "sourcegraph-testing",
|
||||
name: "nacelle",
|
||||
indexer: "scip-go",
|
||||
revisions: []string{
|
||||
"68d3125fb03d4aec540714577401f9f01adffa8a",
|
||||
},
|
||||
},
|
||||
{
|
||||
org: "sourcegraph-testing",
|
||||
name: "nacelle-config",
|
||||
indexer: "scip-go",
|
||||
revisions: []string{
|
||||
"4d4864d3b5b046fe12154f3aae7a86a04690c4ae",
|
||||
},
|
||||
},
|
||||
{
|
||||
org: "sourcegraph-testing",
|
||||
name: "nacelle-service",
|
||||
indexer: "scip-go",
|
||||
revisions: []string{
|
||||
"0652f3023c1bc7e7466a487f20bbe4b5e28fdcc7",
|
||||
},
|
||||
},
|
||||
|
||||
// This repository is archived in-practice and as a good candidate for a low-effort scip-typescript test
|
||||
{
|
||||
org: "sourcegraph",
|
||||
name: "code-intel-extensions",
|
||||
indexer: "scip-typescript",
|
||||
revisions: []string{
|
||||
"c66e756d3d68a1e19048c3f7515ba42a7e793767",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func mainErr(ctx context.Context) error {
|
||||
@ -175,7 +211,9 @@ type IndexerPair struct {
|
||||
}
|
||||
|
||||
var indexFunMap = map[string]IndexerPair{
|
||||
"lsif-go": {"dump", indexGoWithLSIF},
|
||||
"lsif-go": {"dump", indexGoWithLSIF},
|
||||
"scip-go": {"scip", indexGoWithSCIP},
|
||||
"scip-typescript": {"scip", indexTypeScriptWithSCIP},
|
||||
}
|
||||
|
||||
func indexGoWithLSIF(ctx context.Context, reposDir, targetFile, name, revision string) error {
|
||||
@ -196,6 +234,31 @@ func indexGoWithLSIF(ctx context.Context, reposDir, targetFile, name, revision s
|
||||
})
|
||||
}
|
||||
|
||||
func indexGoWithSCIP(ctx context.Context, reposDir, targetFile, name, revision string) error {
|
||||
return indexGeneric(ctx, reposDir, targetFile, name, revision, func(repoCopyDir string) error {
|
||||
// --repository-root=. is necessary here as the temp dir might be within a strange
|
||||
// nest of symlinks on MacOS, which confuses the repository root detection in scip-go.
|
||||
if err := run.Bash(ctx, "scip-go", "--repository-root=.", "-o", targetFile).Dir(repoCopyDir).Run().Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func indexTypeScriptWithSCIP(ctx context.Context, reposDir, targetFile, name, revision string) error {
|
||||
return indexGeneric(ctx, reposDir, targetFile, name, revision, func(repoCopyDir string) error {
|
||||
if err := run.Bash(ctx, "yarn").Dir(repoCopyDir).Run().Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := run.Bash(ctx, "scip-typescript", "index", "--output", targetFile).Dir(repoCopyDir).Run().Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func indexGeneric(ctx context.Context, reposDir, targetFile, name, revision string, index func(repoCopyDir string) error) error {
|
||||
if ok, err := internal.FileExists(targetFile); err != nil {
|
||||
return err
|
||||
|
||||
@ -26,7 +26,7 @@ func main() {
|
||||
}
|
||||
|
||||
const (
|
||||
bucketName = "precise-code-intel-integration-testdata"
|
||||
bucketName = "codeintel-qa-indexes"
|
||||
relativeIndexesDir = "dev/codeintel-qa/testdata/indexes"
|
||||
)
|
||||
|
||||
@ -89,7 +89,7 @@ func downloadAll(ctx context.Context, bucket *storage.BucketHandle, paths []stri
|
||||
}
|
||||
|
||||
func downloadIndex(ctx context.Context, bucket *storage.BucketHandle, indexesDir, name string) (err error) {
|
||||
targetFile := filepath.Join(indexesDir, "sourcegraph-testing."+strings.TrimSuffix(name, ".gz"))
|
||||
targetFile := filepath.Join(indexesDir, strings.TrimSuffix(name, ".gz"))
|
||||
|
||||
if ok, err := internal.FileExists(targetFile); err != nil {
|
||||
return err
|
||||
|
||||
@ -21,12 +21,17 @@ func checkInstanceState(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func instanceStateDiff(ctx context.Context) (string, error) {
|
||||
commitsByRepo, err := internal.CommitsByRepo(indexDir)
|
||||
extensionAndCommitsByRepo, err := internal.ExtensionAndCommitsByRepo(indexDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
expectedCommitsByRepo := map[string][]string{}
|
||||
for repoName, commits := range commitsByRepo {
|
||||
for repoName, extensionAndCommits := range extensionAndCommitsByRepo {
|
||||
commits := make([]string, 0, len(extensionAndCommits))
|
||||
for _, e := range extensionAndCommits {
|
||||
commits = append(commits, e.Commit)
|
||||
}
|
||||
|
||||
sort.Strings(commits)
|
||||
expectedCommitsByRepo[internal.MakeTestRepoName(repoName)] = commits
|
||||
}
|
||||
|
||||
@ -10,15 +10,21 @@ type Location struct {
|
||||
}
|
||||
|
||||
const (
|
||||
repo_etcd = "github.com/sourcegraph-testing/etcd"
|
||||
repo_tidb = "github.com/sourcegraph-testing/tidb"
|
||||
repo_titan = "github.com/sourcegraph-testing/titan"
|
||||
repo_zap = "github.com/sourcegraph-testing/zap"
|
||||
repo_etcd = "github.com/sourcegraph-testing/etcd"
|
||||
repo_tidb = "github.com/sourcegraph-testing/tidb"
|
||||
repo_titan = "github.com/sourcegraph-testing/titan"
|
||||
repo_zap = "github.com/sourcegraph-testing/zap"
|
||||
repo_nacelle = "github.com/sourcegraph-testing/nacelle"
|
||||
repo_nacelle_config = "github.com/sourcegraph-testing/nacelle-config"
|
||||
repo_nacelle_service = "github.com/sourcegraph-testing/nacelle-service"
|
||||
repo_extensions = "github.com/sourcegraph/code-intel-extensions"
|
||||
|
||||
commit_2aa9fa2 = "2aa9fa25da83bdfff756c36a91442edc9a84576c"
|
||||
commit_4d4864d = "4d4864d3b5b046fe12154f3aae7a86a04690c4ae"
|
||||
commit_9aab491 = "9aab49176993f9dc0ed2fcb9ef7e5125518e8b98"
|
||||
commit_a6015e1 = "a6015e13fab9b744d96085308ce4e8f11bad1996"
|
||||
commit_ad78480 = "ad7848014a051dbe3fcd6a4cff2c7befdd16d5a8"
|
||||
commit_c66e756 = "c66e756d3d68a1e19048c3f7515ba42a7e793767"
|
||||
commit_f8307e3 = "f8307e394c512b4263fc0cd67ccf9fd46f1ad9a5"
|
||||
)
|
||||
|
||||
@ -36,6 +42,10 @@ const (
|
||||
// All the test cases for those repos find definitions and references for
|
||||
// the symbol `zap.String`, a global function defined in fields.go in both
|
||||
// versions.
|
||||
//
|
||||
// go-nacelle release tags:
|
||||
// - `v5.0.0` for `nacelle-config at commit `4d4864d`, and
|
||||
// - `v5.0.0` for `nacelle-service` at commit `0652f30`.
|
||||
|
||||
var testCases = []struct {
|
||||
Definition Location // Symbol definition
|
||||
@ -1679,4 +1689,110 @@ var testCases = []struct {
|
||||
{Repo: repo_zap, Rev: commit_a6015e1, Path: "zaptest/observer/observer_test.go", Line: 170, Character: 34},
|
||||
},
|
||||
},
|
||||
{
|
||||
Definition: Location{
|
||||
Repo: repo_nacelle_config,
|
||||
Rev: commit_4d4864d,
|
||||
Path: "sourcer.go",
|
||||
Line: 6,
|
||||
Character: 5,
|
||||
},
|
||||
References: []Location{
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "config.go", Line: 31, Character: 12},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "config.go", Line: 37, Character: 23},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 11, Character: 10},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 14, Character: 6},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 19, Character: 107},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 33, Character: 99},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 55, Character: 41},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 72, Character: 15},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "env_sourcer.go", Line: 13, Character: 6},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "env_sourcer.go", Line: 23, Character: 34},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 20, Character: 6},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 33, Character: 98},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 47, Character: 90},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 113, Character: 41},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 118, Character: 41},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "flag_sourcer.go", Line: 12, Character: 6},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "flag_sourcer.go", Line: 16, Character: 54},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "glob_sourcer.go", Line: 6, Character: 89},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "glob_sourcer.go", Line: 9, Character: 15},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 7, Character: 12},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 11, Character: 6},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 15, Character: 33},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 15, Character: 42},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "test_env_sourcer.go", Line: 10, Character: 6},
|
||||
{Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "test_env_sourcer.go", Line: 14, Character: 49},
|
||||
},
|
||||
},
|
||||
{
|
||||
Definition: Location{
|
||||
Repo: repo_extensions,
|
||||
Rev: commit_c66e756,
|
||||
Path: "template/src/util/graphql.ts",
|
||||
Line: 16,
|
||||
Character: 12,
|
||||
},
|
||||
References: []Location{
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/api.ts", Character: 9},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/api.ts", Line: 23, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 5, Character: 9},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 13, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 46, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 68, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 91, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.ts", Line: 4, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.ts", Line: 81, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/implementations.ts", Line: 3, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/implementations.ts", Line: 75, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/implementations.ts", Line: 92, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/locations.ts", Line: 2, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/locations.ts", Line: 43, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/locations.ts", Line: 54, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 5, Character: 9},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 33, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 76, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 113, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 126, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 175, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 205, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 221, Character: 27},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 222, Character: 27},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 224, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 270, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 303, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 334, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 358, Character: 45},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 7, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 46, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 108, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 152, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 207, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 262, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 5, Character: 9},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 33, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 67, Character: 42},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 78, Character: 42},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 189, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 249, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 3, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 63, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 139, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 268, Character: 44},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 365, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 5, Character: 9},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 24, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 47, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 56, Character: 23},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 57, Character: 23},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 59, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 102, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.ts", Line: 3, Character: 41},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.ts", Line: 75, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.ts", Line: 87, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/stencil.ts", Line: 4, Character: 9},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/stencil.ts", Line: 11, Character: 18},
|
||||
{Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/stencil.ts", Line: 47, Character: 18},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@ -54,13 +54,13 @@ func mainErr(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
commitsByRepo, err := internal.CommitsByRepo(indexDir)
|
||||
extensionAndCommitsByRepo, err := internal.ExtensionAndCommitsByRepo(indexDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
repoNames := make([]string, 0, len(commitsByRepo))
|
||||
for name := range commitsByRepo {
|
||||
repoNames := make([]string, 0, len(extensionAndCommitsByRepo))
|
||||
for name := range extensionAndCommitsByRepo {
|
||||
repoNames = append(repoNames, name)
|
||||
}
|
||||
sort.Strings(repoNames)
|
||||
@ -68,7 +68,7 @@ func mainErr(ctx context.Context) error {
|
||||
limiter := internal.NewLimiter(numConcurrentUploads)
|
||||
defer limiter.Close()
|
||||
|
||||
uploads, err := uploadAll(ctx, commitsByRepo, limiter)
|
||||
uploads, err := uploadAll(ctx, extensionAndCommitsByRepo, limiter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -25,9 +25,9 @@ type uploadMeta struct {
|
||||
// Uploads are performed concurrently given the limiter instance as well as the set of
|
||||
// flags supplied by the user. This function returns a slice of uploadMeta containing
|
||||
// the graphql identifier of the uploaded resources.
|
||||
func uploadAll(ctx context.Context, commitsByRepo map[string][]string, limiter *internal.Limiter) ([]uploadMeta, error) {
|
||||
func uploadAll(ctx context.Context, extensionAndCommitsByRepo map[string][]internal.ExtensionAndCommit, limiter *internal.Limiter) ([]uploadMeta, error) {
|
||||
n := 0
|
||||
for _, commits := range commitsByRepo {
|
||||
for _, commits := range extensionAndCommitsByRepo {
|
||||
n += len(commits)
|
||||
}
|
||||
|
||||
@ -35,8 +35,11 @@ func uploadAll(ctx context.Context, commitsByRepo map[string][]string, limiter *
|
||||
errCh := make(chan error, n)
|
||||
uploadCh := make(chan uploadMeta, n)
|
||||
|
||||
for repoName, commits := range commitsByRepo {
|
||||
for i, commit := range commits {
|
||||
for repoName, extensionAndCommits := range extensionAndCommitsByRepo {
|
||||
for i, extensionAndCommit := range extensionAndCommits {
|
||||
commit := extensionAndCommit.Commit
|
||||
extension := extensionAndCommit.Extension
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
go func(repoName, commit, file string) {
|
||||
@ -63,7 +66,7 @@ func uploadAll(ctx context.Context, commitsByRepo map[string][]string, limiter *
|
||||
repoName: repoName,
|
||||
commit: commit,
|
||||
}
|
||||
}(repoName, commit, fmt.Sprintf("%s.%d.%s.dump", strings.Replace(repoName, "/", ".", 1), i, commit))
|
||||
}(repoName, commit, fmt.Sprintf("%s.%d.%s.%s", strings.Replace(repoName, "/", ".", 1), i, commit, extension))
|
||||
}
|
||||
}
|
||||
|
||||
@ -99,6 +102,7 @@ func upload(ctx context.Context, repoName, commit, file string) (string, error)
|
||||
for k, v := range argMap {
|
||||
args = append(args, fmt.Sprintf("-%s=%s", k, v))
|
||||
}
|
||||
args = append(args, "--skip-scip")
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "codeintel-qa")
|
||||
if err != nil {
|
||||
|
||||
@ -7,22 +7,28 @@ import (
|
||||
"github.com/grafana/regexp"
|
||||
)
|
||||
|
||||
var indexFilenamePattern = regexp.MustCompile(`^([^.]+)\.([^.]+)\.\d+\.([0-9A-Fa-f]{40})\.dump$`)
|
||||
var indexFilenamePattern = regexp.MustCompile(`^([^.]+)\.([^.]+)\.\d+\.([0-9A-Fa-f]{40})\.(scip|dump)$`)
|
||||
|
||||
// CommitsByRepo returns a map from org+repository name to a slice of commits for that repository. The
|
||||
// repositories and commits are read from the filesystem state of the index directory supplied by the user.
|
||||
// This method assumes that index files have been downloaded or generated locally.
|
||||
func CommitsByRepo(indexDir string) (map[string][]string, error) {
|
||||
type ExtensionAndCommit struct {
|
||||
Extension string
|
||||
Commit string
|
||||
}
|
||||
|
||||
// ExtensionAndCommitsByRepo returns a map from org+repository name to a slice of commit and extension
|
||||
// pairs for that repository. The repositories and commits are read from the filesystem state of the
|
||||
// index directory supplied by the user. This method assumes that index files have been downloaded or
|
||||
// generated locally.
|
||||
func ExtensionAndCommitsByRepo(indexDir string) (map[string][]ExtensionAndCommit, error) {
|
||||
infos, err := os.ReadDir(indexDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
commitsByRepo := map[string][]string{}
|
||||
commitsByRepo := map[string][]ExtensionAndCommit{}
|
||||
for _, info := range infos {
|
||||
if matches := indexFilenamePattern.FindStringSubmatch(info.Name()); len(matches) > 0 {
|
||||
orgRepo := fmt.Sprintf("%s/%s", matches[1], matches[2])
|
||||
commitsByRepo[orgRepo] = append(commitsByRepo[orgRepo], matches[3])
|
||||
commitsByRepo[orgRepo] = append(commitsByRepo[orgRepo], ExtensionAndCommit{Extension: matches[4], Commit: matches[3]})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
69
doc/admin/how-to/clear_codeintel_data.md
Normal file
69
doc/admin/how-to/clear_codeintel_data.md
Normal file
@ -0,0 +1,69 @@
|
||||
# Clearing code intelligence data
|
||||
|
||||
Clear all precise code intelligence data from your instance and start fresh.
|
||||
|
||||
## Clearing the `frontend` database
|
||||
|
||||
The following commands assume a connection to the `frontend` database. Refer to our guide on [`psql`](/admin/how-to/run-psql) if you do not use an [external database](/admin/external_services).
|
||||
|
||||
If you are clearing **all** code intelligence data, then you will need to clear the metadata information in the frontend database. This will clear the high-level information about code intelligence indexes, but the raw data will remain in the `codeintel-db` database, which will also need to be cleared (see the next section).
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
TRUNCATE lsif_uploads;
|
||||
TRUNCATE lsif_uploads_audit_logs;
|
||||
TRUNCATE lsif_uploads_reference_counts;
|
||||
TRUNCATE codeintel_commit_dates;
|
||||
TRUNCATE lsif_packages;
|
||||
TRUNCATE lsif_references;
|
||||
TRUNCATE lsif_nearest_uploads;
|
||||
TRUNCATE lsif_nearest_uploads_links;
|
||||
TRUNCATE lsif_uploads_visible_at_tip;
|
||||
TRUNCATE lsif_dirty_repositories;
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
## Clearing the `codeintel-db` database
|
||||
|
||||
The following commands assume a connection to the `codeintel-db` database. Refer to our guide on [`psql`](/admin/how-to/run-psql) if you do not use an [external database](/admin/external_services).
|
||||
|
||||
### Clearing LSIF data
|
||||
|
||||
Truncate the following tables to clear all LSIF-encoded information from the database. This command can be run when clearing **all** data from the instance, in which case the associated records in the `frontend` database must also be cleared.
|
||||
|
||||
This command can also be run after a completed [migration from LSIF to SCIP](/admin/how-to/lsif_scip_migration), in which case these tables should be empty but may retain a number of blocks containing a large number of dead tuples. Truncating these _empty_ tables acts like a `VACUUM FULL` and releases the previously used disk space.
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
TRUNCATE lsif_data_metadata;
|
||||
TRUNCATE lsif_data_documents;
|
||||
TRUNCATE lsif_data_documents_schema_versions;
|
||||
TRUNCATE lsif_data_result_chunks;
|
||||
TRUNCATE lsif_data_definitions;
|
||||
TRUNCATE lsif_data_definitions_schema_versions;
|
||||
TRUNCATE lsif_data_references;
|
||||
TRUNCATE lsif_data_references_schema_versions;
|
||||
TRUNCATE lsif_data_implementations;
|
||||
TRUNCATE lsif_data_implementations_schema_versions;
|
||||
TRUNCATE codeintel_last_reconcile;
|
||||
COMMIT;
|
||||
```
|
||||
|
||||
### Clearing SCIP data
|
||||
|
||||
Truncate the following tables to clear all SCIP-encoded information from the database. Only run this command if you clearing **all** data from the instance, in which case the associated records in the `frontend` database must also be cleared.
|
||||
|
||||
```sql
|
||||
BEGIN;
|
||||
TRUNCATE codeintel_scip_metadata;
|
||||
TRUNCATE codeintel_scip_documents;
|
||||
TRUNCATE codeintel_scip_documents_schema_versions;
|
||||
TRUNCATE codeintel_scip_document_lookup;
|
||||
TRUNCATE codeintel_scip_document_lookup_schema_versions;
|
||||
TRUNCATE codeintel_scip_documents_dereference_logs;
|
||||
TRUNCATE codeintel_scip_symbols;
|
||||
TRUNCATE codeintel_scip_symbols_schema_versions;
|
||||
TRUNCATE codeintel_scip_symbol_names;
|
||||
TRUNCATE codeintel_last_reconcile;
|
||||
COMMIT;
|
||||
```
|
||||
@ -26,3 +26,4 @@
|
||||
- [How to import a set of internal repositories to Sourcegraph](internal_github_repos.md)
|
||||
- [How to identify and resolve index corruption in postgres 14](postgres14-index-corruption.md)
|
||||
- [How to use a static endpoint for updates and pings](static-ip-for-updates-and-pings.md)
|
||||
- [Migrating code intelligence data from LSIF to SCIP (Sourcegraph 4.5 -> 4.6)](lsif_scip_migration.md)
|
||||
|
||||
13
doc/admin/how-to/lsif_scip_migration.md
Normal file
13
doc/admin/how-to/lsif_scip_migration.md
Normal file
@ -0,0 +1,13 @@
|
||||
# Migrating code intelligence data from LSIF to SCIP (Sourcegraph 4.5 -> 4.6)
|
||||
|
||||
> WARNING: The migration from LSIF -> SCIP is **destructive and irreversible**. A downgrade from 4.6 to a previous version will result in the inability to access migrated code intelligence data (powering precise code navigation features).
|
||||
|
||||
Sourcegraph 4.5 introduced an [out-of-band migration](/admin/how-to/unfinished_migration#checking-progress) that re-encodes LSIF code intelligence data as SCIP in the `codeintel-db`. This migration is required to complete prior to a subsequent upgrade to 4.6, as support for reading LSIF-encoded data has been removed as of this version.
|
||||
|
||||
As of [src-cli 4.5](https://github.com/sourcegraph/src-cli/releases/tag/4.5.0), LSIF indexes will be converted to SCIP prior to upload. This ensures that only _existing_ data needs to be migrated in the background. Using an older version of src-cli to upload code intelligence to your index may continue to feed additional LSIF data that needs to be subsequently migrated. This will ultimately block the ability to upgrade to the next version as the migration will never reach 100% (and remain there).
|
||||
|
||||
Once the migration has completed, Postgres may continue to hold on to disk space that was previously occupied by LSIF data. Future versions of Sourcegraph will drop these tables completely, freeing this space. Heavier users of precise code intelligence may wish to reclaim this disk space earlier. Once the migration is complete, you can [truncate the LSIF data tables](clear_codeintel_data.md#clearing-lsif-data) to immediately reclaim this space.
|
||||
|
||||
---
|
||||
|
||||
If you wish to take the scorched earth route and clear all existing code intelligence data from your instance and start fresh, follow the entire guide on [clearing code intelligence data](clear_codeintel_data.md).
|
||||
@ -20,33 +20,27 @@ This page lists the changes that are relevant for [upgrading Sourcegraph on Dock
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details.
|
||||
|
||||
## v4.3 ➔ v4.4
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
_No notes._
|
||||
|
||||
## v4.2 ➔ v4.3.1
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
_No notes._
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
## v4.2 ➔ v4.3.1
|
||||
|
||||
_No notes._
|
||||
|
||||
## v4.1 ➔ v4.2.1
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
|
||||
This upgrade adds the [node-exporter](https://github.com/prometheus/node_exporter) deployment, which collects crucial machine-level metrics that help Sourcegraph scale your deployment.
|
||||
|
||||
## v4.0 ➔ v4.1.3
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
_No notes._
|
||||
|
||||
## v3.43 ➔ v4.0
|
||||
|
||||
|
||||
@ -24,19 +24,15 @@
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
- This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details.
|
||||
|
||||
## v4.3 ➔ v4.4
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
_No notes._
|
||||
|
||||
## v4.2 ➔ v4.3.1
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
_No notes._
|
||||
|
||||
## v4.1 ➔ v4.2.1
|
||||
|
||||
|
||||
@ -12,11 +12,7 @@ Each section comprehensively describes the changes needed in Docker images, envi
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
|
||||
## v4.3 ➔ v4.4
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
- This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details.
|
||||
|
||||
## v4.3.1 ➔ v4.4
|
||||
|
||||
@ -25,8 +21,6 @@ As a template, perform the same actions as the following diffs in your own deplo
|
||||
|
||||
## v4.2 ➔ v4.3.1
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
As a template, perform the same actions as the following diffs in your own deployment:
|
||||
- [`➔ v4.3.1`](https://github.com/sourcegraph/deploy-sourcegraph-docker/commit/1a8f9a3d71664bf31a1fae9d2ad99c9074eaebe9)
|
||||
|
||||
|
||||
@ -21,25 +21,23 @@ This document describes the exact changes needed to update a single-node Sourceg
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
- This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details.
|
||||
|
||||
## v4.3 ➔ v4.4
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
|
||||
_Upgrade notes for the next version will appear here._
|
||||
_No notes._
|
||||
|
||||
## v4.2 ➔ v4.3.1
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
_No notes._
|
||||
|
||||
## v4.1 ➔ v4.2.1
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
_No notes._
|
||||
|
||||
## v4.0 ➔ v4.1.3
|
||||
|
||||
<!-- Add changes changes to this section before release. -->
|
||||
_No notes._
|
||||
|
||||
## v3.43 ➔ v4.0
|
||||
|
||||
|
||||
@ -42,7 +42,7 @@ func NewSCIPMigrator(store, codeintelStore *basestore.Store) *scipMigrator {
|
||||
}
|
||||
}
|
||||
|
||||
func (m *scipMigrator) ID() int { return 18 }
|
||||
func (m *scipMigrator) ID() int { return 20 }
|
||||
func (m *scipMigrator) Interval() time.Duration { return time.Second }
|
||||
|
||||
// Progress returns the ratio between the number of SCIP upload records to SCIP+LSIF upload.
|
||||
|
||||
@ -98,6 +98,7 @@ func registerEnterpriseMigrators(runner *oobmigration.Runner, noDelay bool, deps
|
||||
codeintel.NewDefinitionLocationsCountMigrator(deps.codeIntelStore, 1000, 0),
|
||||
codeintel.NewReferencesLocationsCountMigrator(deps.codeIntelStore, 1000, 0),
|
||||
codeintel.NewDocumentColumnSplitMigrator(deps.codeIntelStore, 100, 0),
|
||||
codeintel.NewSCIPMigrator(deps.store, deps.codeIntelStore),
|
||||
insights.NewMigrator(deps.store, deps.insightsStore),
|
||||
insightsrecordingtimes.NewRecordingTimesMigrator(deps.insightsStore, 500),
|
||||
insightsBackfiller.NewMigrator(deps.insightsStore, glock.NewRealClock(), 10),
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
|
||||
@ -16,8 +18,9 @@ import (
|
||||
var (
|
||||
client *gqltestutil.Client
|
||||
|
||||
initSG = flag.NewFlagSet("initserver", flag.ExitOnError)
|
||||
addRepos = flag.NewFlagSet("addrepos", flag.ExitOnError)
|
||||
initSG = flag.NewFlagSet("initserver", flag.ExitOnError)
|
||||
addRepos = flag.NewFlagSet("addrepos", flag.ExitOnError)
|
||||
oobmigration = flag.NewFlagSet("oobmigration", flag.ExitOnError)
|
||||
|
||||
baseURL = initSG.String("baseurl", os.Getenv("SOURCEGRAPH_BASE_URL"), "The base URL of the Sourcegraph instance. (Required)")
|
||||
email = initSG.String("email", os.Getenv("TEST_USER_EMAIL"), "The email of the admin user. (Required)")
|
||||
@ -27,6 +30,9 @@ var (
|
||||
githubToken = addRepos.String("githubtoken", os.Getenv("GITHUB_TOKEN"), "The github access token that will be used to authenticate an external service. (Required)")
|
||||
addReposConfig = addRepos.String("config", "", "Path to the external service config. (Required)")
|
||||
|
||||
migrationID = oobmigration.String("id", "", "The target oobmigration identifier. (Required)")
|
||||
migrationDownFlag = oobmigration.Bool("down", false, "Supply to change the migration from up (default) to down.")
|
||||
|
||||
home = os.Getenv("HOME")
|
||||
profile = home + "/.sg_envrc"
|
||||
)
|
||||
@ -35,7 +41,7 @@ func main() {
|
||||
flag.Parse()
|
||||
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Println("initSG or addRepos subcommand is required")
|
||||
fmt.Println("initSG, addRepos, or oobmigration subcommand is required")
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
@ -47,6 +53,9 @@ func main() {
|
||||
case "addRepos":
|
||||
addRepos.Parse(os.Args[2:])
|
||||
addReposCommand()
|
||||
case "oobmigration":
|
||||
oobmigration.Parse(os.Args[2:])
|
||||
oobmigrationCommand()
|
||||
case "default":
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
@ -159,7 +168,6 @@ func addReposCommand() {
|
||||
jsoniter.Unmarshal(byteValue, &externalsvcs)
|
||||
|
||||
for i := range externalsvcs {
|
||||
|
||||
// Set up external service
|
||||
esID, err := client.AddExternalService(gqltestutil.AddExternalServiceInput{
|
||||
Kind: externalsvcs[i].Kind,
|
||||
@ -191,3 +199,38 @@ func addReposCommand() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const MigrationTimeout = time.Minute * 5
|
||||
|
||||
func oobmigrationCommand() {
|
||||
if *migrationID == "" {
|
||||
log.Fatal("migration identifier (-id) is not supplied")
|
||||
}
|
||||
id := *migrationID
|
||||
up := !*migrationDownFlag
|
||||
|
||||
client, err := gqltestutil.SignIn(*baseURL, *email, *password)
|
||||
if err != nil {
|
||||
log.Fatal("Failed to sign in:", err)
|
||||
}
|
||||
log.Println("Site admin authenticated:", *username)
|
||||
|
||||
if err := client.SetMigrationDirection(id, up); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), MigrationTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := client.PollMigration(ctx, id, func(progress float64) bool {
|
||||
if up {
|
||||
log.Printf("Waiting for migration %s to complete (%.2f%% done).", id, progress*100)
|
||||
} else {
|
||||
log.Printf("Waiting for migration %s to rollback (%.2f%% done).", id, (1-progress)*100)
|
||||
}
|
||||
|
||||
return (up && progress == 1) || (!up && progress == 0)
|
||||
}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
82
internal/gqltestutil/migration.go
Normal file
82
internal/gqltestutil/migration.go
Normal file
@ -0,0 +1,82 @@
|
||||
package gqltestutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/lib/errors"
|
||||
)
|
||||
|
||||
var MigrationPollInterval = time.Second
|
||||
|
||||
// PollMigration will invoke the given function periodically with the current progress of the
|
||||
// given migration. The loop will break once the function returns true or the given context
|
||||
// is canceled.
|
||||
func (c *Client) PollMigration(ctx context.Context, id string, f func(float64) bool) error {
|
||||
for {
|
||||
progress, err := c.GetMigrationProgress(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f(progress) {
|
||||
return nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-time.After(MigrationPollInterval):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) GetMigrationProgress(id string) (float64, error) {
|
||||
const query = `
|
||||
query GetMigrationStatus {
|
||||
outOfBandMigrations {
|
||||
id
|
||||
progress
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
var envelope struct {
|
||||
Data struct {
|
||||
OutOfBandMigrations []struct {
|
||||
ID string
|
||||
Progress float64
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := c.GraphQL("", query, nil, &envelope); err != nil {
|
||||
return 0, errors.Wrap(err, "request GraphQL")
|
||||
}
|
||||
|
||||
for _, migration := range envelope.Data.OutOfBandMigrations {
|
||||
if migration.ID == id {
|
||||
return migration.Progress, nil
|
||||
}
|
||||
}
|
||||
|
||||
return 0, errors.Newf("unknown oobmigration %q", id)
|
||||
}
|
||||
|
||||
func (c *Client) SetMigrationDirection(id string, up bool) error {
|
||||
const query = `
|
||||
mutation SetMigrationDirection($id: ID!, $applyReverse: Boolean!) {
|
||||
setMigrationDirection(id: $id, applyReverse: $applyReverse) {
|
||||
alwaysNil
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
variables := map[string]any{
|
||||
"id": id,
|
||||
"applyReverse": !up,
|
||||
}
|
||||
if err := c.GraphQL("", query, variables, nil); err != nil {
|
||||
return errors.Wrap(err, "request GraphQL")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -116,3 +116,13 @@
|
||||
is_enterprise: false
|
||||
introduced_version_major: 4
|
||||
introduced_version_minor: 5
|
||||
- id: 20
|
||||
team: code-intelligence
|
||||
component: codeintel-db.lsif_data_*
|
||||
description: Convert LSIF data to SCIP.
|
||||
non_destructive: false
|
||||
is_enterprise: true
|
||||
introduced_version_major: 4
|
||||
introduced_version_minor: 5
|
||||
deprecated_version_major: 4
|
||||
deprecated_version_minor: 6
|
||||
|
||||
Loading…
Reference in New Issue
Block a user