From a528304b5167c4c14ed4f70108d1e07ba683fe1e Mon Sep 17 00:00:00 2001 From: Eric Fritz Date: Mon, 23 Jan 2023 08:24:44 -0600 Subject: [PATCH] codeintel: Register SCIP oobmigration (#45106) --- CHANGELOG.md | 1 + dev/ci/integration/code-intel/install-src.sh | 25 ++++ dev/ci/integration/code-intel/preprod-run.sh | 21 ++- dev/ci/integration/code-intel/repos.json | 6 +- dev/ci/integration/code-intel/test.sh | 54 +++++--- dev/codeintel-qa/README.md | 8 +- dev/codeintel-qa/cmd/clone-and-index/main.go | 65 ++++++++- dev/codeintel-qa/cmd/download/main.go | 4 +- dev/codeintel-qa/cmd/query/state.go | 9 +- dev/codeintel-qa/cmd/query/test_cases.go | 124 +++++++++++++++++- dev/codeintel-qa/cmd/upload/main.go | 8 +- dev/codeintel-qa/cmd/upload/upload.go | 14 +- dev/codeintel-qa/internal/indexes.go | 20 ++- doc/admin/how-to/clear_codeintel_data.md | 69 ++++++++++ doc/admin/how-to/index.md | 1 + doc/admin/how-to/lsif_scip_migration.md | 13 ++ doc/admin/updates/docker_compose.md | 20 +-- doc/admin/updates/kubernetes.md | 10 +- doc/admin/updates/pure_docker.md | 8 +- doc/admin/updates/server.md | 12 +- .../migrations/codeintel/scip_migrator.go | 2 +- .../oobmigration/migrations/register.go | 1 + internal/cmd/init-sg/main.go | 51 ++++++- internal/gqltestutil/migration.go | 82 ++++++++++++ internal/oobmigration/oobmigrations.yaml | 10 ++ 25 files changed, 549 insertions(+), 89 deletions(-) create mode 100755 dev/ci/integration/code-intel/install-src.sh create mode 100644 doc/admin/how-to/clear_codeintel_data.md create mode 100644 doc/admin/how-to/lsif_scip_migration.md create mode 100644 internal/gqltestutil/migration.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d457ec65bea..f948398db38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ All notable changes to Sourcegraph are documented in this file. - The default author and email for changesets will now be pulled from user account details when possible. [#46385](https://github.com/sourcegraph/sourcegraph/pull/46385) - Code Insights has a new display option: "Max number of series points to display". This setting controls the number of data points you see per series on an insight. [#46653](https://github.com/sourcegraph/sourcegraph/pull/46653) +- Added out-of-band migration that will migrate all existing data from LSIF to SCIP (see additional [migration documentation](https://docs.sourcegraph.com/admin/how-to/lsif_scip_migration)). [#45106](https://github.com/sourcegraph/sourcegraph/pull/45106) ### Changed diff --git a/dev/ci/integration/code-intel/install-src.sh b/dev/ci/integration/code-intel/install-src.sh new file mode 100755 index 00000000000..3efbfe9a06b --- /dev/null +++ b/dev/ci/integration/code-intel/install-src.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This script is called by test.sh and preprod-run.sh to install an up-to-date +# version of src-cli as required by the codeintel-qa pipeline. The target binary +# is installed to {REPO_ROOT}/.bin/src. + +set -eux +cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." +root_dir="$(pwd)" + +# By default, the commit that added handleSCIP support +VERSION=${1:-'1c70d536b4ab3187b5aed41af8f259f1b8ceba6b'} + +TEMP=$(mktemp -d -t sgdockerbuild_XXXXXXX) +cleanup() { + rm -rf "${TEMP}" +} +trap cleanup EXIT + +git clone git@github.com:sourcegraph/src-cli.git "${TEMP}" --depth 1 +pushd "${TEMP}" +git checkout "${VERSION}" +mkdir -p "${root_dir}/.bin" +go build -o "${root_dir}/.bin" ./cmd/src +popd diff --git a/dev/ci/integration/code-intel/preprod-run.sh b/dev/ci/integration/code-intel/preprod-run.sh index b1bc1071011..5766cc0f46b 100755 --- a/dev/ci/integration/code-intel/preprod-run.sh +++ b/dev/ci/integration/code-intel/preprod-run.sh @@ -4,8 +4,8 @@ # against the preprod. set -eux - cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." +root_dir=$(pwd) SOURCEGRAPH_BASE_URL="https://preview.sgdev.dev" TEST_USER_EMAIL="testadmin@preview.sgdev.dev" @@ -18,21 +18,30 @@ export SOURCEGRAPH_SUDO_USER export TEST_USER_PASSWORD export SOURCEGRAPH_SUDO_TOKEN -echo "--- :go: Building init-sg" +echo '--- :go: Building init-sg' go build -o init-sg ./internal/cmd/init-sg/... -echo "--- :horse: Running init-sg addRepos" +echo '--- :horse: Running init-sg addRepos' ./init-sg addRepos -config ./dev/ci/integration/code-intel/repos.json +echo '--- Installing local src-cli' +./dev/ci/integration/code-intel/install-src.sh +which src +src version + +echo '--- :brain: Running the test suite' pushd dev/codeintel-qa -echo "--- :brain: Running the test suite" echo '--- :zero: downloading test data from GCS' go run ./cmd/download + echo '--- :one: clearing existing state' go run ./cmd/clear + echo '--- :two: integration test ./dev/codeintel-qa/cmd/upload' -go run ./cmd/upload --timeout=5m +env PATH="${root_dir}/.bin:${PATH}" go run ./cmd/upload --timeout=5m + +# make queries but do not assert against expected locations echo '--- :three: integration test ./dev/codeintel-qa/cmd/query' -go run ./cmd/query -check-query-result=false # make queries but do not assert against expected locations +go run ./cmd/query -check-query-result=false popd diff --git a/dev/ci/integration/code-intel/repos.json b/dev/ci/integration/code-intel/repos.json index d0746adfb1f..0597d73748e 100644 --- a/dev/ci/integration/code-intel/repos.json +++ b/dev/ci/integration/code-intel/repos.json @@ -8,7 +8,11 @@ "sourcegraph-testing/etcd", "sourcegraph-testing/tidb", "sourcegraph-testing/titan", - "sourcegraph-testing/zap" + "sourcegraph-testing/zap", + "sourcegraph-testing/nacelle", + "sourcegraph-testing/nacelle-config", + "sourcegraph-testing/nacelle-service", + "sourcegraph/code-intel-extensions" ] } } diff --git a/dev/ci/integration/code-intel/test.sh b/dev/ci/integration/code-intel/test.sh index cb75d83d8e4..ecb5d076094 100755 --- a/dev/ci/integration/code-intel/test.sh +++ b/dev/ci/integration/code-intel/test.sh @@ -3,36 +3,56 @@ # This script runs the codeintel-qa tests against a running server. # This script is invoked by ./dev/ci/integration/run-integration.sh after running an instance. +set -eux cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." root_dir=$(pwd) -set -e -export SOURCEGRAPH_BASE_URL="${1:-"http://localhost:7080"}" +SOURCEGRAPH_BASE_URL="${1:-"http://localhost:7080"}" +export SOURCEGRAPH_BASE_URL -echo '--- initializing Sourcegraph instance' +echo '--- :go: Building init-sg' +go build -o init-sg ./internal/cmd/init-sg/... -pushd internal/cmd/init-sg -go build -o "${root_dir}/init-sg" -popd - -pushd dev/ci/integration/code-intel +echo '--- Initializing instance' "${root_dir}/init-sg" initSG -# Disable `-x` to avoid printing secrets -set +x + +echo '--- Loading secrets' +set +x # Avoid printing secrets # shellcheck disable=SC1091 source /root/.sg_envrc -"${root_dir}/init-sg" addRepos -config repos.json -popd +set -x +echo '--- :horse: Running init-sg addRepos' +"${root_dir}/init-sg" addRepos -config ./dev/ci/integration/code-intel/repos.json + +echo '--- Installing local src-cli' +./dev/ci/integration/code-intel/install-src.sh +which src +src version + +echo '--- :brain: Running the test suite' pushd dev/codeintel-qa -echo "--- :brain: Running the test suite" echo '--- :zero: downloading test data from GCS' go run ./cmd/download + echo '--- :one: clearing existing state' go run ./cmd/clear -echo '--- :two: integration test ./dev/codeintel-qa/cmd/upload' -go run ./cmd/upload --timeout=5m -verbose -echo '--- :three: integration test ./dev/codeintel-qa/cmd/query' -go run ./cmd/query -verbose + +# Disable migration #20 (LSIF -> SCIP) +echo '--- :two: Disabling LSIF -> SCIP migration' +"${root_dir}/init-sg" oobmigration -id T3V0T2ZCYW5kTWlncmF0aW9uOjIw -down + +echo '--- :three: integration test ./dev/codeintel-qa/cmd/upload' +env PATH="${root_dir}/.bin:${PATH}" go run ./cmd/upload --timeout=5m + +echo '--- :four: integration test ./dev/codeintel-qa/cmd/query' +go run ./cmd/query + +# Enable migration #20 (LSIF -> SCIP) and wait for it to complete +echo '--- :five: Running LSIF -> SCIP migration' +"${root_dir}/init-sg" oobmigration -id T3V0T2ZCYW5kTWlncmF0aW9uOjIw + +echo '--- :six: integration test ./dev/codeintel-qa/cmd/query' +go run ./cmd/query popd diff --git a/dev/codeintel-qa/README.md b/dev/codeintel-qa/README.md index 81c5a476896..993b104369e 100644 --- a/dev/codeintel-qa/README.md +++ b/dev/codeintel-qa/README.md @@ -8,6 +8,8 @@ Ensure that the following tools are available on your path: - [`src`](https://github.com/sourcegraph/src-cli) - [`lsif-go`](https://github.com/sourcegraph/lsif-go) +- [`scip-go`](https://github.com/sourcegraph/scip-go) +- [`scip-typescript`](https://github.com/sourcegraph/scip-typescript) You should have enviornment variables that authenticate you to the `sourcegraph-dev` GCS project if you plan to upload or download index files (as we do in CI). @@ -27,7 +29,11 @@ SOURCEGRAPH_SUDO_TOKEN= "sourcegraph-testing/etcd", "sourcegraph-testing/tidb", "sourcegraph-testing/titan", - "sourcegraph-testing/zap" + "sourcegraph-testing/zap", + "sourcegraph-testing/nacelle", + "sourcegraph-testing/nacelle-config", + "sourcegraph-testing/nacelle-service", + "sourcegraph/code-intel-extensions" ], ``` diff --git a/dev/codeintel-qa/cmd/clone-and-index/main.go b/dev/codeintel-qa/cmd/clone-and-index/main.go index e5d4984c490..3a4d89a8fd3 100644 --- a/dev/codeintel-qa/cmd/clone-and-index/main.go +++ b/dev/codeintel-qa/cmd/clone-and-index/main.go @@ -80,6 +80,42 @@ var repositoryMeta = []struct { "f8307e394c512b4263fc0cd67ccf9fd46f1ad9a5", }, }, + + // These repositories have their module names modified and new tags created to refer to each other + { + org: "sourcegraph-testing", + name: "nacelle", + indexer: "scip-go", + revisions: []string{ + "68d3125fb03d4aec540714577401f9f01adffa8a", + }, + }, + { + org: "sourcegraph-testing", + name: "nacelle-config", + indexer: "scip-go", + revisions: []string{ + "4d4864d3b5b046fe12154f3aae7a86a04690c4ae", + }, + }, + { + org: "sourcegraph-testing", + name: "nacelle-service", + indexer: "scip-go", + revisions: []string{ + "0652f3023c1bc7e7466a487f20bbe4b5e28fdcc7", + }, + }, + + // This repository is archived in-practice and as a good candidate for a low-effort scip-typescript test + { + org: "sourcegraph", + name: "code-intel-extensions", + indexer: "scip-typescript", + revisions: []string{ + "c66e756d3d68a1e19048c3f7515ba42a7e793767", + }, + }, } func mainErr(ctx context.Context) error { @@ -175,7 +211,9 @@ type IndexerPair struct { } var indexFunMap = map[string]IndexerPair{ - "lsif-go": {"dump", indexGoWithLSIF}, + "lsif-go": {"dump", indexGoWithLSIF}, + "scip-go": {"scip", indexGoWithSCIP}, + "scip-typescript": {"scip", indexTypeScriptWithSCIP}, } func indexGoWithLSIF(ctx context.Context, reposDir, targetFile, name, revision string) error { @@ -196,6 +234,31 @@ func indexGoWithLSIF(ctx context.Context, reposDir, targetFile, name, revision s }) } +func indexGoWithSCIP(ctx context.Context, reposDir, targetFile, name, revision string) error { + return indexGeneric(ctx, reposDir, targetFile, name, revision, func(repoCopyDir string) error { + // --repository-root=. is necessary here as the temp dir might be within a strange + // nest of symlinks on MacOS, which confuses the repository root detection in scip-go. + if err := run.Bash(ctx, "scip-go", "--repository-root=.", "-o", targetFile).Dir(repoCopyDir).Run().Wait(); err != nil { + return err + } + + return nil + }) +} + +func indexTypeScriptWithSCIP(ctx context.Context, reposDir, targetFile, name, revision string) error { + return indexGeneric(ctx, reposDir, targetFile, name, revision, func(repoCopyDir string) error { + if err := run.Bash(ctx, "yarn").Dir(repoCopyDir).Run().Wait(); err != nil { + return err + } + if err := run.Bash(ctx, "scip-typescript", "index", "--output", targetFile).Dir(repoCopyDir).Run().Wait(); err != nil { + return err + } + + return nil + }) +} + func indexGeneric(ctx context.Context, reposDir, targetFile, name, revision string, index func(repoCopyDir string) error) error { if ok, err := internal.FileExists(targetFile); err != nil { return err diff --git a/dev/codeintel-qa/cmd/download/main.go b/dev/codeintel-qa/cmd/download/main.go index df8cb3538be..e32abcf4ee6 100644 --- a/dev/codeintel-qa/cmd/download/main.go +++ b/dev/codeintel-qa/cmd/download/main.go @@ -26,7 +26,7 @@ func main() { } const ( - bucketName = "precise-code-intel-integration-testdata" + bucketName = "codeintel-qa-indexes" relativeIndexesDir = "dev/codeintel-qa/testdata/indexes" ) @@ -89,7 +89,7 @@ func downloadAll(ctx context.Context, bucket *storage.BucketHandle, paths []stri } func downloadIndex(ctx context.Context, bucket *storage.BucketHandle, indexesDir, name string) (err error) { - targetFile := filepath.Join(indexesDir, "sourcegraph-testing."+strings.TrimSuffix(name, ".gz")) + targetFile := filepath.Join(indexesDir, strings.TrimSuffix(name, ".gz")) if ok, err := internal.FileExists(targetFile); err != nil { return err diff --git a/dev/codeintel-qa/cmd/query/state.go b/dev/codeintel-qa/cmd/query/state.go index ed004da4a26..40b2835b2af 100644 --- a/dev/codeintel-qa/cmd/query/state.go +++ b/dev/codeintel-qa/cmd/query/state.go @@ -21,12 +21,17 @@ func checkInstanceState(ctx context.Context) error { } func instanceStateDiff(ctx context.Context) (string, error) { - commitsByRepo, err := internal.CommitsByRepo(indexDir) + extensionAndCommitsByRepo, err := internal.ExtensionAndCommitsByRepo(indexDir) if err != nil { return "", err } expectedCommitsByRepo := map[string][]string{} - for repoName, commits := range commitsByRepo { + for repoName, extensionAndCommits := range extensionAndCommitsByRepo { + commits := make([]string, 0, len(extensionAndCommits)) + for _, e := range extensionAndCommits { + commits = append(commits, e.Commit) + } + sort.Strings(commits) expectedCommitsByRepo[internal.MakeTestRepoName(repoName)] = commits } diff --git a/dev/codeintel-qa/cmd/query/test_cases.go b/dev/codeintel-qa/cmd/query/test_cases.go index 191346cee57..81c84565e0e 100644 --- a/dev/codeintel-qa/cmd/query/test_cases.go +++ b/dev/codeintel-qa/cmd/query/test_cases.go @@ -10,15 +10,21 @@ type Location struct { } const ( - repo_etcd = "github.com/sourcegraph-testing/etcd" - repo_tidb = "github.com/sourcegraph-testing/tidb" - repo_titan = "github.com/sourcegraph-testing/titan" - repo_zap = "github.com/sourcegraph-testing/zap" + repo_etcd = "github.com/sourcegraph-testing/etcd" + repo_tidb = "github.com/sourcegraph-testing/tidb" + repo_titan = "github.com/sourcegraph-testing/titan" + repo_zap = "github.com/sourcegraph-testing/zap" + repo_nacelle = "github.com/sourcegraph-testing/nacelle" + repo_nacelle_config = "github.com/sourcegraph-testing/nacelle-config" + repo_nacelle_service = "github.com/sourcegraph-testing/nacelle-service" + repo_extensions = "github.com/sourcegraph/code-intel-extensions" commit_2aa9fa2 = "2aa9fa25da83bdfff756c36a91442edc9a84576c" + commit_4d4864d = "4d4864d3b5b046fe12154f3aae7a86a04690c4ae" commit_9aab491 = "9aab49176993f9dc0ed2fcb9ef7e5125518e8b98" commit_a6015e1 = "a6015e13fab9b744d96085308ce4e8f11bad1996" commit_ad78480 = "ad7848014a051dbe3fcd6a4cff2c7befdd16d5a8" + commit_c66e756 = "c66e756d3d68a1e19048c3f7515ba42a7e793767" commit_f8307e3 = "f8307e394c512b4263fc0cd67ccf9fd46f1ad9a5" ) @@ -36,6 +42,10 @@ const ( // All the test cases for those repos find definitions and references for // the symbol `zap.String`, a global function defined in fields.go in both // versions. +// +// go-nacelle release tags: +// - `v5.0.0` for `nacelle-config at commit `4d4864d`, and +// - `v5.0.0` for `nacelle-service` at commit `0652f30`. var testCases = []struct { Definition Location // Symbol definition @@ -1679,4 +1689,110 @@ var testCases = []struct { {Repo: repo_zap, Rev: commit_a6015e1, Path: "zaptest/observer/observer_test.go", Line: 170, Character: 34}, }, }, + { + Definition: Location{ + Repo: repo_nacelle_config, + Rev: commit_4d4864d, + Path: "sourcer.go", + Line: 6, + Character: 5, + }, + References: []Location{ + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "config.go", Line: 31, Character: 12}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "config.go", Line: 37, Character: 23}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 11, Character: 10}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 14, Character: 6}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 19, Character: 107}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 33, Character: 99}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 55, Character: 41}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "directory_sourcer.go", Line: 72, Character: 15}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "env_sourcer.go", Line: 13, Character: 6}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "env_sourcer.go", Line: 23, Character: 34}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 20, Character: 6}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 33, Character: 98}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 47, Character: 90}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 113, Character: 41}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "file_sourcer.go", Line: 118, Character: 41}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "flag_sourcer.go", Line: 12, Character: 6}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "flag_sourcer.go", Line: 16, Character: 54}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "glob_sourcer.go", Line: 6, Character: 89}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "glob_sourcer.go", Line: 9, Character: 15}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 7, Character: 12}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 11, Character: 6}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 15, Character: 33}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "multi_sourcer.go", Line: 15, Character: 42}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "test_env_sourcer.go", Line: 10, Character: 6}, + {Repo: repo_nacelle_config, Rev: commit_4d4864d, Path: "test_env_sourcer.go", Line: 14, Character: 49}, + }, + }, + { + Definition: Location{ + Repo: repo_extensions, + Rev: commit_c66e756, + Path: "template/src/util/graphql.ts", + Line: 16, + Character: 12, + }, + References: []Location{ + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/api.ts", Character: 9}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/api.ts", Line: 23, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 5, Character: 9}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 13, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 46, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 68, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.test.ts", Line: 91, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.ts", Line: 4, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/definition-hover.ts", Line: 81, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/implementations.ts", Line: 3, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/implementations.ts", Line: 75, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/implementations.ts", Line: 92, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/locations.ts", Line: 2, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/locations.ts", Line: 43, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/locations.ts", Line: 54, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 5, Character: 9}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 33, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 76, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 113, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 126, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 175, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 205, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 221, Character: 27}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 222, Character: 27}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 224, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 270, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 303, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 334, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.test.ts", Line: 358, Character: 45}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 7, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 46, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 108, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 152, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 207, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/providers.ts", Line: 262, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 5, Character: 9}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 33, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 67, Character: 42}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 78, Character: 42}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 189, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.test.ts", Line: 249, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 3, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 63, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 139, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 268, Character: 44}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/ranges.ts", Line: 365, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 5, Character: 9}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 24, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 47, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 56, Character: 23}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 57, Character: 23}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 59, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.test.ts", Line: 102, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.ts", Line: 3, Character: 41}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.ts", Line: 75, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/references.ts", Line: 87, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/stencil.ts", Line: 4, Character: 9}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/stencil.ts", Line: 11, Character: 18}, + {Repo: repo_extensions, Rev: commit_c66e756, Path: "template/src/lsif/stencil.ts", Line: 47, Character: 18}, + }, + }, } diff --git a/dev/codeintel-qa/cmd/upload/main.go b/dev/codeintel-qa/cmd/upload/main.go index 2405c23c210..3cadadbd774 100644 --- a/dev/codeintel-qa/cmd/upload/main.go +++ b/dev/codeintel-qa/cmd/upload/main.go @@ -54,13 +54,13 @@ func mainErr(ctx context.Context) error { return err } - commitsByRepo, err := internal.CommitsByRepo(indexDir) + extensionAndCommitsByRepo, err := internal.ExtensionAndCommitsByRepo(indexDir) if err != nil { return err } - repoNames := make([]string, 0, len(commitsByRepo)) - for name := range commitsByRepo { + repoNames := make([]string, 0, len(extensionAndCommitsByRepo)) + for name := range extensionAndCommitsByRepo { repoNames = append(repoNames, name) } sort.Strings(repoNames) @@ -68,7 +68,7 @@ func mainErr(ctx context.Context) error { limiter := internal.NewLimiter(numConcurrentUploads) defer limiter.Close() - uploads, err := uploadAll(ctx, commitsByRepo, limiter) + uploads, err := uploadAll(ctx, extensionAndCommitsByRepo, limiter) if err != nil { return err } diff --git a/dev/codeintel-qa/cmd/upload/upload.go b/dev/codeintel-qa/cmd/upload/upload.go index 8af1c36bea7..df5abc0e2e2 100644 --- a/dev/codeintel-qa/cmd/upload/upload.go +++ b/dev/codeintel-qa/cmd/upload/upload.go @@ -25,9 +25,9 @@ type uploadMeta struct { // Uploads are performed concurrently given the limiter instance as well as the set of // flags supplied by the user. This function returns a slice of uploadMeta containing // the graphql identifier of the uploaded resources. -func uploadAll(ctx context.Context, commitsByRepo map[string][]string, limiter *internal.Limiter) ([]uploadMeta, error) { +func uploadAll(ctx context.Context, extensionAndCommitsByRepo map[string][]internal.ExtensionAndCommit, limiter *internal.Limiter) ([]uploadMeta, error) { n := 0 - for _, commits := range commitsByRepo { + for _, commits := range extensionAndCommitsByRepo { n += len(commits) } @@ -35,8 +35,11 @@ func uploadAll(ctx context.Context, commitsByRepo map[string][]string, limiter * errCh := make(chan error, n) uploadCh := make(chan uploadMeta, n) - for repoName, commits := range commitsByRepo { - for i, commit := range commits { + for repoName, extensionAndCommits := range extensionAndCommitsByRepo { + for i, extensionAndCommit := range extensionAndCommits { + commit := extensionAndCommit.Commit + extension := extensionAndCommit.Extension + wg.Add(1) go func(repoName, commit, file string) { @@ -63,7 +66,7 @@ func uploadAll(ctx context.Context, commitsByRepo map[string][]string, limiter * repoName: repoName, commit: commit, } - }(repoName, commit, fmt.Sprintf("%s.%d.%s.dump", strings.Replace(repoName, "/", ".", 1), i, commit)) + }(repoName, commit, fmt.Sprintf("%s.%d.%s.%s", strings.Replace(repoName, "/", ".", 1), i, commit, extension)) } } @@ -99,6 +102,7 @@ func upload(ctx context.Context, repoName, commit, file string) (string, error) for k, v := range argMap { args = append(args, fmt.Sprintf("-%s=%s", k, v)) } + args = append(args, "--skip-scip") tempDir, err := os.MkdirTemp("", "codeintel-qa") if err != nil { diff --git a/dev/codeintel-qa/internal/indexes.go b/dev/codeintel-qa/internal/indexes.go index c6969c7089f..626d281c978 100644 --- a/dev/codeintel-qa/internal/indexes.go +++ b/dev/codeintel-qa/internal/indexes.go @@ -7,22 +7,28 @@ import ( "github.com/grafana/regexp" ) -var indexFilenamePattern = regexp.MustCompile(`^([^.]+)\.([^.]+)\.\d+\.([0-9A-Fa-f]{40})\.dump$`) +var indexFilenamePattern = regexp.MustCompile(`^([^.]+)\.([^.]+)\.\d+\.([0-9A-Fa-f]{40})\.(scip|dump)$`) -// CommitsByRepo returns a map from org+repository name to a slice of commits for that repository. The -// repositories and commits are read from the filesystem state of the index directory supplied by the user. -// This method assumes that index files have been downloaded or generated locally. -func CommitsByRepo(indexDir string) (map[string][]string, error) { +type ExtensionAndCommit struct { + Extension string + Commit string +} + +// ExtensionAndCommitsByRepo returns a map from org+repository name to a slice of commit and extension +// pairs for that repository. The repositories and commits are read from the filesystem state of the +// index directory supplied by the user. This method assumes that index files have been downloaded or +// generated locally. +func ExtensionAndCommitsByRepo(indexDir string) (map[string][]ExtensionAndCommit, error) { infos, err := os.ReadDir(indexDir) if err != nil { return nil, err } - commitsByRepo := map[string][]string{} + commitsByRepo := map[string][]ExtensionAndCommit{} for _, info := range infos { if matches := indexFilenamePattern.FindStringSubmatch(info.Name()); len(matches) > 0 { orgRepo := fmt.Sprintf("%s/%s", matches[1], matches[2]) - commitsByRepo[orgRepo] = append(commitsByRepo[orgRepo], matches[3]) + commitsByRepo[orgRepo] = append(commitsByRepo[orgRepo], ExtensionAndCommit{Extension: matches[4], Commit: matches[3]}) } } diff --git a/doc/admin/how-to/clear_codeintel_data.md b/doc/admin/how-to/clear_codeintel_data.md new file mode 100644 index 00000000000..407093373fd --- /dev/null +++ b/doc/admin/how-to/clear_codeintel_data.md @@ -0,0 +1,69 @@ +# Clearing code intelligence data + +Clear all precise code intelligence data from your instance and start fresh. + +## Clearing the `frontend` database + +The following commands assume a connection to the `frontend` database. Refer to our guide on [`psql`](/admin/how-to/run-psql) if you do not use an [external database](/admin/external_services). + +If you are clearing **all** code intelligence data, then you will need to clear the metadata information in the frontend database. This will clear the high-level information about code intelligence indexes, but the raw data will remain in the `codeintel-db` database, which will also need to be cleared (see the next section). + +```sql +BEGIN; +TRUNCATE lsif_uploads; +TRUNCATE lsif_uploads_audit_logs; +TRUNCATE lsif_uploads_reference_counts; +TRUNCATE codeintel_commit_dates; +TRUNCATE lsif_packages; +TRUNCATE lsif_references; +TRUNCATE lsif_nearest_uploads; +TRUNCATE lsif_nearest_uploads_links; +TRUNCATE lsif_uploads_visible_at_tip; +TRUNCATE lsif_dirty_repositories; +COMMIT; +``` + +## Clearing the `codeintel-db` database + +The following commands assume a connection to the `codeintel-db` database. Refer to our guide on [`psql`](/admin/how-to/run-psql) if you do not use an [external database](/admin/external_services). + +### Clearing LSIF data + +Truncate the following tables to clear all LSIF-encoded information from the database. This command can be run when clearing **all** data from the instance, in which case the associated records in the `frontend` database must also be cleared. + +This command can also be run after a completed [migration from LSIF to SCIP](/admin/how-to/lsif_scip_migration), in which case these tables should be empty but may retain a number of blocks containing a large number of dead tuples. Truncating these _empty_ tables acts like a `VACUUM FULL` and releases the previously used disk space. + +```sql +BEGIN; +TRUNCATE lsif_data_metadata; +TRUNCATE lsif_data_documents; +TRUNCATE lsif_data_documents_schema_versions; +TRUNCATE lsif_data_result_chunks; +TRUNCATE lsif_data_definitions; +TRUNCATE lsif_data_definitions_schema_versions; +TRUNCATE lsif_data_references; +TRUNCATE lsif_data_references_schema_versions; +TRUNCATE lsif_data_implementations; +TRUNCATE lsif_data_implementations_schema_versions; +TRUNCATE codeintel_last_reconcile; +COMMIT; +``` + +### Clearing SCIP data + +Truncate the following tables to clear all SCIP-encoded information from the database. Only run this command if you clearing **all** data from the instance, in which case the associated records in the `frontend` database must also be cleared. + +```sql +BEGIN; +TRUNCATE codeintel_scip_metadata; +TRUNCATE codeintel_scip_documents; +TRUNCATE codeintel_scip_documents_schema_versions; +TRUNCATE codeintel_scip_document_lookup; +TRUNCATE codeintel_scip_document_lookup_schema_versions; +TRUNCATE codeintel_scip_documents_dereference_logs; +TRUNCATE codeintel_scip_symbols; +TRUNCATE codeintel_scip_symbols_schema_versions; +TRUNCATE codeintel_scip_symbol_names; +TRUNCATE codeintel_last_reconcile; +COMMIT; +``` diff --git a/doc/admin/how-to/index.md b/doc/admin/how-to/index.md index c2ab4fe03bf..7b86e8e5d57 100644 --- a/doc/admin/how-to/index.md +++ b/doc/admin/how-to/index.md @@ -26,3 +26,4 @@ - [How to import a set of internal repositories to Sourcegraph](internal_github_repos.md) - [How to identify and resolve index corruption in postgres 14](postgres14-index-corruption.md) - [How to use a static endpoint for updates and pings](static-ip-for-updates-and-pings.md) +- [Migrating code intelligence data from LSIF to SCIP (Sourcegraph 4.5 -> 4.6)](lsif_scip_migration.md) diff --git a/doc/admin/how-to/lsif_scip_migration.md b/doc/admin/how-to/lsif_scip_migration.md new file mode 100644 index 00000000000..995cb8b7a5d --- /dev/null +++ b/doc/admin/how-to/lsif_scip_migration.md @@ -0,0 +1,13 @@ +# Migrating code intelligence data from LSIF to SCIP (Sourcegraph 4.5 -> 4.6) + +> WARNING: The migration from LSIF -> SCIP is **destructive and irreversible**. A downgrade from 4.6 to a previous version will result in the inability to access migrated code intelligence data (powering precise code navigation features). + +Sourcegraph 4.5 introduced an [out-of-band migration](/admin/how-to/unfinished_migration#checking-progress) that re-encodes LSIF code intelligence data as SCIP in the `codeintel-db`. This migration is required to complete prior to a subsequent upgrade to 4.6, as support for reading LSIF-encoded data has been removed as of this version. + +As of [src-cli 4.5](https://github.com/sourcegraph/src-cli/releases/tag/4.5.0), LSIF indexes will be converted to SCIP prior to upload. This ensures that only _existing_ data needs to be migrated in the background. Using an older version of src-cli to upload code intelligence to your index may continue to feed additional LSIF data that needs to be subsequently migrated. This will ultimately block the ability to upgrade to the next version as the migration will never reach 100% (and remain there). + +Once the migration has completed, Postgres may continue to hold on to disk space that was previously occupied by LSIF data. Future versions of Sourcegraph will drop these tables completely, freeing this space. Heavier users of precise code intelligence may wish to reclaim this disk space earlier. Once the migration is complete, you can [truncate the LSIF data tables](clear_codeintel_data.md#clearing-lsif-data) to immediately reclaim this space. + +--- + +If you wish to take the scorched earth route and clear all existing code intelligence data from your instance and start fresh, follow the entire guide on [clearing code intelligence data](clear_codeintel_data.md). diff --git a/doc/admin/updates/docker_compose.md b/doc/admin/updates/docker_compose.md index 1a58ea19ee1..ec23546af94 100644 --- a/doc/admin/updates/docker_compose.md +++ b/doc/admin/updates/docker_compose.md @@ -20,33 +20,27 @@ This page lists the changes that are relevant for [upgrading Sourcegraph on Dock -_Upgrade notes for the next version will appear here._ +This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details. ## v4.3 ➔ v4.4 - - -_Upgrade notes for the next version will appear here._ +_No notes._ ## v4.2 ➔ v4.3.1 - +_No notes._ -_Upgrade notes for the next version will appear here._ +## v4.2 ➔ v4.3.1 + +_No notes._ ## v4.1 ➔ v4.2.1 - - -_Upgrade notes for the next version will appear here._ - This upgrade adds the [node-exporter](https://github.com/prometheus/node_exporter) deployment, which collects crucial machine-level metrics that help Sourcegraph scale your deployment. ## v4.0 ➔ v4.1.3 - - -_Upgrade notes for the next version will appear here._ +_No notes._ ## v3.43 ➔ v4.0 diff --git a/doc/admin/updates/kubernetes.md b/doc/admin/updates/kubernetes.md index 60bcb1394f8..3dceebe47e8 100644 --- a/doc/admin/updates/kubernetes.md +++ b/doc/admin/updates/kubernetes.md @@ -24,19 +24,15 @@ -_Upgrade notes for the next version will appear here._ +- This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details. ## v4.3 ➔ v4.4 - - -_Upgrade notes for the next version will appear here._ +_No notes._ ## v4.2 ➔ v4.3.1 - - -_Upgrade notes for the next version will appear here._ +_No notes._ ## v4.1 ➔ v4.2.1 diff --git a/doc/admin/updates/pure_docker.md b/doc/admin/updates/pure_docker.md index f913d8e1c3f..5b84937c05b 100644 --- a/doc/admin/updates/pure_docker.md +++ b/doc/admin/updates/pure_docker.md @@ -12,11 +12,7 @@ Each section comprehensively describes the changes needed in Docker images, envi -_Upgrade notes for the next version will appear here._ - -## v4.3 ➔ v4.4 - - +- This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details. ## v4.3.1 ➔ v4.4 @@ -25,8 +21,6 @@ As a template, perform the same actions as the following diffs in your own deplo ## v4.2 ➔ v4.3.1 - - As a template, perform the same actions as the following diffs in your own deployment: - [`➔ v4.3.1`](https://github.com/sourcegraph/deploy-sourcegraph-docker/commit/1a8f9a3d71664bf31a1fae9d2ad99c9074eaebe9) diff --git a/doc/admin/updates/server.md b/doc/admin/updates/server.md index f6efcf0c5f2..7019fb22453 100644 --- a/doc/admin/updates/server.md +++ b/doc/admin/updates/server.md @@ -21,25 +21,23 @@ This document describes the exact changes needed to update a single-node Sourceg -_Upgrade notes for the next version will appear here._ +- This release introduces a background job that will convert all LSIF data into SCIP. **This migration is irreversible** and a rollback from this version may result in loss of precise code intelligence data. Please see the [migration notes](/admin/how-to/lsif_scip_migration) for more details. ## v4.3 ➔ v4.4 - - -_Upgrade notes for the next version will appear here._ +_No notes._ ## v4.2 ➔ v4.3.1 - +_No notes._ ## v4.1 ➔ v4.2.1 - +_No notes._ ## v4.0 ➔ v4.1.3 - +_No notes._ ## v3.43 ➔ v4.0 diff --git a/enterprise/internal/oobmigration/migrations/codeintel/scip_migrator.go b/enterprise/internal/oobmigration/migrations/codeintel/scip_migrator.go index 1ebb76a2761..d310d8013d0 100644 --- a/enterprise/internal/oobmigration/migrations/codeintel/scip_migrator.go +++ b/enterprise/internal/oobmigration/migrations/codeintel/scip_migrator.go @@ -42,7 +42,7 @@ func NewSCIPMigrator(store, codeintelStore *basestore.Store) *scipMigrator { } } -func (m *scipMigrator) ID() int { return 18 } +func (m *scipMigrator) ID() int { return 20 } func (m *scipMigrator) Interval() time.Duration { return time.Second } // Progress returns the ratio between the number of SCIP upload records to SCIP+LSIF upload. diff --git a/enterprise/internal/oobmigration/migrations/register.go b/enterprise/internal/oobmigration/migrations/register.go index 83f035fcb41..e6af34e0071 100644 --- a/enterprise/internal/oobmigration/migrations/register.go +++ b/enterprise/internal/oobmigration/migrations/register.go @@ -98,6 +98,7 @@ func registerEnterpriseMigrators(runner *oobmigration.Runner, noDelay bool, deps codeintel.NewDefinitionLocationsCountMigrator(deps.codeIntelStore, 1000, 0), codeintel.NewReferencesLocationsCountMigrator(deps.codeIntelStore, 1000, 0), codeintel.NewDocumentColumnSplitMigrator(deps.codeIntelStore, 100, 0), + codeintel.NewSCIPMigrator(deps.store, deps.codeIntelStore), insights.NewMigrator(deps.store, deps.insightsStore), insightsrecordingtimes.NewRecordingTimesMigrator(deps.insightsStore, 500), insightsBackfiller.NewMigrator(deps.insightsStore, glock.NewRealClock(), 10), diff --git a/internal/cmd/init-sg/main.go b/internal/cmd/init-sg/main.go index 3c7d4beca91..b1e4d0b028d 100644 --- a/internal/cmd/init-sg/main.go +++ b/internal/cmd/init-sg/main.go @@ -1,12 +1,14 @@ package main import ( + "context" "flag" "fmt" "io" "log" "os" "strings" + "time" jsoniter "github.com/json-iterator/go" @@ -16,8 +18,9 @@ import ( var ( client *gqltestutil.Client - initSG = flag.NewFlagSet("initserver", flag.ExitOnError) - addRepos = flag.NewFlagSet("addrepos", flag.ExitOnError) + initSG = flag.NewFlagSet("initserver", flag.ExitOnError) + addRepos = flag.NewFlagSet("addrepos", flag.ExitOnError) + oobmigration = flag.NewFlagSet("oobmigration", flag.ExitOnError) baseURL = initSG.String("baseurl", os.Getenv("SOURCEGRAPH_BASE_URL"), "The base URL of the Sourcegraph instance. (Required)") email = initSG.String("email", os.Getenv("TEST_USER_EMAIL"), "The email of the admin user. (Required)") @@ -27,6 +30,9 @@ var ( githubToken = addRepos.String("githubtoken", os.Getenv("GITHUB_TOKEN"), "The github access token that will be used to authenticate an external service. (Required)") addReposConfig = addRepos.String("config", "", "Path to the external service config. (Required)") + migrationID = oobmigration.String("id", "", "The target oobmigration identifier. (Required)") + migrationDownFlag = oobmigration.Bool("down", false, "Supply to change the migration from up (default) to down.") + home = os.Getenv("HOME") profile = home + "/.sg_envrc" ) @@ -35,7 +41,7 @@ func main() { flag.Parse() if len(os.Args) < 2 { - fmt.Println("initSG or addRepos subcommand is required") + fmt.Println("initSG, addRepos, or oobmigration subcommand is required") flag.PrintDefaults() os.Exit(1) } @@ -47,6 +53,9 @@ func main() { case "addRepos": addRepos.Parse(os.Args[2:]) addReposCommand() + case "oobmigration": + oobmigration.Parse(os.Args[2:]) + oobmigrationCommand() case "default": flag.PrintDefaults() os.Exit(1) @@ -159,7 +168,6 @@ func addReposCommand() { jsoniter.Unmarshal(byteValue, &externalsvcs) for i := range externalsvcs { - // Set up external service esID, err := client.AddExternalService(gqltestutil.AddExternalServiceInput{ Kind: externalsvcs[i].Kind, @@ -191,3 +199,38 @@ func addReposCommand() { } } } + +const MigrationTimeout = time.Minute * 5 + +func oobmigrationCommand() { + if *migrationID == "" { + log.Fatal("migration identifier (-id) is not supplied") + } + id := *migrationID + up := !*migrationDownFlag + + client, err := gqltestutil.SignIn(*baseURL, *email, *password) + if err != nil { + log.Fatal("Failed to sign in:", err) + } + log.Println("Site admin authenticated:", *username) + + if err := client.SetMigrationDirection(id, up); err != nil { + log.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), MigrationTimeout) + defer cancel() + + if err := client.PollMigration(ctx, id, func(progress float64) bool { + if up { + log.Printf("Waiting for migration %s to complete (%.2f%% done).", id, progress*100) + } else { + log.Printf("Waiting for migration %s to rollback (%.2f%% done).", id, (1-progress)*100) + } + + return (up && progress == 1) || (!up && progress == 0) + }); err != nil { + log.Fatal(err) + } +} diff --git a/internal/gqltestutil/migration.go b/internal/gqltestutil/migration.go new file mode 100644 index 00000000000..ce834b53737 --- /dev/null +++ b/internal/gqltestutil/migration.go @@ -0,0 +1,82 @@ +package gqltestutil + +import ( + "context" + "time" + + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +var MigrationPollInterval = time.Second + +// PollMigration will invoke the given function periodically with the current progress of the +// given migration. The loop will break once the function returns true or the given context +// is canceled. +func (c *Client) PollMigration(ctx context.Context, id string, f func(float64) bool) error { + for { + progress, err := c.GetMigrationProgress(id) + if err != nil { + return err + } + if f(progress) { + return nil + } + + select { + case <-time.After(MigrationPollInterval): + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (c *Client) GetMigrationProgress(id string) (float64, error) { + const query = ` + query GetMigrationStatus { + outOfBandMigrations { + id + progress + } + } + ` + + var envelope struct { + Data struct { + OutOfBandMigrations []struct { + ID string + Progress float64 + } + } + } + if err := c.GraphQL("", query, nil, &envelope); err != nil { + return 0, errors.Wrap(err, "request GraphQL") + } + + for _, migration := range envelope.Data.OutOfBandMigrations { + if migration.ID == id { + return migration.Progress, nil + } + } + + return 0, errors.Newf("unknown oobmigration %q", id) +} + +func (c *Client) SetMigrationDirection(id string, up bool) error { + const query = ` + mutation SetMigrationDirection($id: ID!, $applyReverse: Boolean!) { + setMigrationDirection(id: $id, applyReverse: $applyReverse) { + alwaysNil + } + } + ` + + variables := map[string]any{ + "id": id, + "applyReverse": !up, + } + if err := c.GraphQL("", query, variables, nil); err != nil { + return errors.Wrap(err, "request GraphQL") + } + + return nil +} diff --git a/internal/oobmigration/oobmigrations.yaml b/internal/oobmigration/oobmigrations.yaml index 9873908c2b6..4a9690746bf 100644 --- a/internal/oobmigration/oobmigrations.yaml +++ b/internal/oobmigration/oobmigrations.yaml @@ -116,3 +116,13 @@ is_enterprise: false introduced_version_major: 4 introduced_version_minor: 5 +- id: 20 + team: code-intelligence + component: codeintel-db.lsif_data_* + description: Convert LSIF data to SCIP. + non_destructive: false + is_enterprise: true + introduced_version_major: 4 + introduced_version_minor: 5 + deprecated_version_major: 4 + deprecated_version_minor: 6