artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 18:11:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: Completely sunset qdrant (#62018)

Browse Source 
This removes qdrant from this codebase entirely.
All the docker images, dependencies, (dead) usage in code.

My understanding is that we don't use this feature and never properly rolled it out.

Test plan:

CI passes and code review from owners.
This commit is contained in:
Erik Seliger 2024-04-22 18:00:57 +02:00 committed by GitHub
parent a309948d11
commit 24e8505019
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
48 changed files with 53 additions and 2580 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/frontend/graphqlbackend/embeddings.go
View File

@ -14,7 +14,6 @@ type EmbeddingsResolver interface {
EmbeddingsMultiSearch(ctx context.Context, args EmbeddingsMultiSearchInputArgs) (EmbeddingsSearchResultsResolver, error)
IsContextRequiredForChatQuery(ctx context.Context, args IsContextRequiredForChatQueryInputArgs) (bool, error)
RepoEmbeddingJobs(ctx context.Context, args ListRepoEmbeddingJobsArgs) (*graphqlutil.ConnectionResolver[RepoEmbeddingJobResolver], error)
MigrateToQdrant(ctx context.Context) (*EmptyResponse, error)
ScheduleRepositoriesForEmbedding(ctx context.Context, args ScheduleRepositoriesForEmbeddingArgs) (*EmptyResponse, error)
CancelRepoEmbeddingJob(ctx context.Context, args CancelRepoEmbeddingJobArgs) (*EmptyResponse, error)

5
cmd/frontend/graphqlbackend/embeddings.graphql
View File

@ -99,11 +99,6 @@ extend type Mutation {
Experimental: Cancels the embedding job with the given ID. The job must exist and be in either 'processing' or 'queued' state.
"""
cancelRepoEmbeddingJob(job: ID!): EmptyResponse!
"""
TEMPORARY: creates a new embedding job for all completed embeddings
jobs so that they will be moved from blobstore to qdrant
"""
migrateToQdrant: EmptyResponse!
}
"""

3
cmd/frontend/internal/cli/config.go
View File

@ -598,7 +598,6 @@ func serviceConnections(logger log.Logger) conftypes.ServiceConnections {
Searchers: searcherAddrs,
Symbols: symbolsAddrs,
Embeddings: embeddingsAddrs,
Qdrant: qdrantAddr,
Zoekts: zoektAddrs,
ZoektListTTL: indexedListTTL,
}
@ -617,8 +616,6 @@ var (
embeddingsURLsOnce sync.Once
embeddingsURLs *endpoint.Map
qdrantAddr = os.Getenv("QDRANT_ENDPOINT")
indexedListTTL = func() time.Duration {
ttl, _ := time.ParseDuration(env.Get("SRC_INDEXED_SEARCH_LIST_CACHE_TTL", "", "Indexed search list cache TTL"))
if ttl == 0 {

1
cmd/frontend/internal/codycontext/BUILD.bazel
View File

@ -18,7 +18,6 @@ go_library(
"//internal/database",
"//internal/dotcom",
"//internal/embeddings",
"//internal/embeddings/db",
"//internal/embeddings/embed",
"//internal/gitserver",
"//internal/metrics",

21
cmd/frontend/internal/codycontext/context.go
View File

@ -16,7 +16,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/embeddings"
vdb "github.com/sourcegraph/sourcegraph/internal/embeddings/db"
"github.com/sourcegraph/sourcegraph/internal/embeddings/embed"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/metrics"
@ -40,7 +39,7 @@ type FileChunkContext struct {
EndLine int
}
func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embeddingsClient embeddings.Client, searchClient client.SearchClient, gitserverClient gitserver.Client, getQdrantSearcher func() (vdb.VectorSearcher, error)) *CodyContextClient {
func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embeddingsClient embeddings.Client, searchClient client.SearchClient, gitserverClient gitserver.Client) *CodyContextClient {
redMetrics := metrics.NewREDMetrics(
obsCtx.Registerer,
"codycontext_client",
@ -59,11 +58,10 @@ func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embedding
}
return &CodyContextClient{
db: db,
embeddingsClient: embeddingsClient,
searchClient: searchClient,
getQdrantSearcher: getQdrantSearcher,
contentFilter: newRepoContentFilter(obsCtx.Logger, gitserverClient),
db: db,
embeddingsClient: embeddingsClient,
searchClient: searchClient,
contentFilter: newRepoContentFilter(obsCtx.Logger, gitserverClient),
obsCtx: obsCtx,
getCodyContextOp: op("getCodyContext"),
@ -73,11 +71,10 @@ func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embedding
}
type CodyContextClient struct {
db database.DB
embeddingsClient embeddings.Client
searchClient client.SearchClient
contentFilter RepoContentFilter
getQdrantSearcher func() (vdb.VectorSearcher, error)
db database.DB
embeddingsClient embeddings.Client
searchClient client.SearchClient
contentFilter RepoContentFilter
obsCtx *observation.Context
getCodyContextOp *observation.Operation

1
cmd/frontend/internal/context/BUILD.bazel
View File

@ -13,7 +13,6 @@ go_library(
"//internal/conf/conftypes",
"//internal/database",
"//internal/embeddings",
"//internal/embeddings/db",
"//internal/gitserver",
"//internal/observation",
"//internal/search/client",

4
cmd/frontend/internal/context/init.go
View File

@ -10,7 +10,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/embeddings"
vdb "github.com/sourcegraph/sourcegraph/internal/embeddings/db"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/search/client"
@ -29,8 +28,6 @@ func Init(
embeddingsClient := embeddings.NewDefaultClient()
searchClient := client.New(observationCtx.Logger, db, gitserver.NewClient("graphql.context.search"))
getQdrantDB := vdb.NewDBFromConfFunc(observationCtx.Logger, vdb.NewDisabledDB())
getQdrantSearcher := func() (vdb.VectorSearcher, error) { return getQdrantDB() }
contextClient := codycontext.NewCodyContextClient(
observationCtx,
@ -38,7 +35,6 @@ func Init(
embeddingsClient,
searchClient,
services.GitserverClient.Scoped("codycontext.client"),
getQdrantSearcher,
)
enterpriseServices.CodyContextResolver = resolvers.NewResolver(
db,

1
cmd/frontend/internal/context/resolvers/context_test.go
View File

@ -245,7 +245,6 @@ func TestContextResolver(t *testing.T) {
mockEmbeddingsClient,
mockSearchClient,
mockGitserver,
nil,
)
resolver := NewResolver(

13
cmd/frontend/internal/embeddings/resolvers/resolvers.go
View File

@ -161,19 +161,6 @@ func (r *Resolver) ScheduleRepositoriesForEmbedding(ctx context.Context, args gr
return &graphqlbackend.EmptyResponse{}, nil
}
func (r *Resolver) MigrateToQdrant(ctx context.Context) (*graphqlbackend.EmptyResponse, error) {
if !conf.EmbeddingsEnabled() {
return nil, errors.New("embeddings are not configured or disabled")
}
ec := conf.GetEmbeddingsConfig(conf.Get().SiteConfig())
if ec == nil || !ec.Qdrant.Enabled {
return nil, errors.New("qdrant is not enabled")
}
return &graphqlbackend.EmptyResponse{}, r.repoEmbeddingJobsStore.RescheduleAllRepos(ctx)
}
func (r *Resolver) CancelRepoEmbeddingJob(ctx context.Context, args graphqlbackend.CancelRepoEmbeddingJobArgs) (*graphqlbackend.EmptyResponse, error) {
// 🚨 SECURITY: check whether user is site-admin
if err := auth.CheckCurrentUserIsSiteAdmin(ctx, r.db); err != nil {

1
cmd/worker/internal/embeddings/repo/BUILD.bazel
View File

@ -25,7 +25,6 @@ go_library(
"//internal/database",
"//internal/embeddings",
"//internal/embeddings/background/repo",
"//internal/embeddings/db",
"//internal/embeddings/embed",
"//internal/env",
"//internal/featureflag",

83
cmd/worker/internal/embeddings/repo/handler.go
View File

@ -16,7 +16,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/embeddings"
bgrepo "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo"
"github.com/sourcegraph/sourcegraph/internal/embeddings/db"
"github.com/sourcegraph/sourcegraph/internal/embeddings/embed"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/featureflag"
@ -31,7 +30,6 @@ type handler struct {
db database.DB
uploadStore uploadstore.Store
gitserverClient gitserver.Client
getQdrantInserter func() (db.VectorInserter, error)
contextService embed.ContextService
repoEmbeddingJobsStore bgrepo.RepoEmbeddingJobsStore
rankingService *ranking.Service
@ -104,22 +102,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.
return err
}
modelID := embeddingsClient.GetModelIdentifier()
modelDims, err := embeddingsClient.GetDimensions()
if err != nil {
return err
}
qdrantInserter, err := h.getQdrantInserter()
if err != nil {
return err
}
err = qdrantInserter.PrepareUpdate(ctx, modelID, uint64(modelDims))
if err != nil {
return err
}
var previousIndex *embeddings.RepoEmbeddingIndex
if embeddingsConfig.Incremental {
previousIndex, err = embeddings.DownloadRepoEmbeddingIndex(ctx, h.uploadStore, repo.ID, repo.Name)
@ -151,18 +133,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.
if previousIndex != nil {
logger.Info("found previous embeddings index. Attempting incremental update", log.String("old_revision", string(previousIndex.Revision)))
opts.IndexedRevision = previousIndex.Revision
hasPreviousIndex, err := qdrantInserter.HasIndex(ctx, modelID, repo.ID, previousIndex.Revision)
if err != nil {
return err
}
if !hasPreviousIndex {
err = uploadPreviousIndex(ctx, modelID, qdrantInserter, repo.ID, previousIndex)
if err != nil {
return err
}
}
}
ranks, err := h.rankingService.GetDocumentRanks(ctx, repo.Name)
@ -179,7 +149,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.
repoEmbeddingIndex, toRemove, stats, err := embed.EmbedRepo(
ctx,
embeddingsClient,
qdrantInserter,
h.contextService,
fetcher,
repo.IDName(),
@ -192,16 +161,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.
return err
}
err = qdrantInserter.FinalizeUpdate(ctx, db.FinalizeUpdateParams{
ModelID: modelID,
RepoID: repo.ID,
Revision: record.Revision,
FilesToRemove: toRemove,
})
if err != nil {
return err
}
reportStats(stats) // final, complete report
logger.Info(
@ -324,45 +283,3 @@ func (r *revisionFetcher) validateRevision(ctx context.Context) error {
}
return nil
}
func uploadPreviousIndex(ctx context.Context, modelID string, inserter db.VectorInserter, repoID api.RepoID, previousIndex *embeddings.RepoEmbeddingIndex) error {
const batchSize = 128
batch := make([]db.ChunkPoint, batchSize)
for indexNum, index := range []embeddings.EmbeddingIndex{previousIndex.CodeIndex, previousIndex.TextIndex} {
isCode := indexNum == 0
// returns the ith row in the index as a ChunkPoint
getChunkPoint := func(i int) db.ChunkPoint {
payload := db.ChunkPayload{
RepoName: previousIndex.RepoName,
RepoID: repoID,
Revision: previousIndex.Revision,
FilePath: index.RowMetadata[i].FileName,
StartLine: uint32(index.RowMetadata[i].StartLine),
EndLine: uint32(index.RowMetadata[i].EndLine),
IsCode: isCode,
}
return db.NewChunkPoint(payload, embeddings.Dequantize(index.Row(i)))
}
for batchStart := 0; batchStart < len(index.RowMetadata); batchStart += batchSize {
// Build a batch
batch = batch[:0] // reset batch
for i := batchStart; i < batchStart+batchSize && i < len(index.RowMetadata); i++ {
batch = append(batch, getChunkPoint(i))
}
// Insert the batch
err := inserter.InsertChunks(ctx, db.InsertParams{
ModelID: modelID,
ChunkPoints: batch,
})
if err != nil {
return err
}
}
}
return nil
}

8
cmd/worker/internal/embeddings/repo/worker.go
View File

@ -12,7 +12,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/embeddings"
repoembeddingsbg "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo"
vdb "github.com/sourcegraph/sourcegraph/internal/embeddings/db"
"github.com/sourcegraph/sourcegraph/internal/embeddings/embed"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
@ -54,10 +53,6 @@ func (s *repoEmbeddingJob) Routines(_ context.Context, observationCtx *observati
return nil, err
}
// qdrant is going to be removed. For now we only ever set the noop db.
qdrantDB := vdb.NewNoopDB()
getQdrantInserter := func() (vdb.VectorInserter, error) { return qdrantDB, nil }
workCtx := actor.WithInternalActor(context.Background())
return []goroutine.BackgroundRoutine{
newRepoEmbeddingJobWorker(
@ -67,7 +62,6 @@ func (s *repoEmbeddingJob) Routines(_ context.Context, observationCtx *observati
db,
uploadStore,
gitserver.NewClient("embeddings.worker"),
getQdrantInserter,
services.ContextService,
repoembeddingsbg.NewRepoEmbeddingJobsStore(db),
services.RankingService,
@ -82,7 +76,6 @@ func newRepoEmbeddingJobWorker(
db database.DB,
uploadStore uploadstore.Store,
gitserverClient gitserver.Client,
getQdrantInserter func() (vdb.VectorInserter, error),
contextService embed.ContextService,
repoEmbeddingJobsStore repoembeddingsbg.RepoEmbeddingJobsStore,
rankingService *ranking.Service,
@ -91,7 +84,6 @@ func newRepoEmbeddingJobWorker(
db: db,
uploadStore: uploadStore,
gitserverClient: gitserverClient,
getQdrantInserter: getQdrantInserter,
contextService: contextService,
repoEmbeddingJobsStore: repoEmbeddingJobsStore,
rankingService: rankingService,

7
deps.bzl
View File

@ -4823,13 +4823,6 @@ def go_dependencies():
sum = "h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=",
version = "v0.0.0-20170810143723-de5bf2ad4578",
)
go_repository(
name = "com_github_qdrant_go_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/qdrant/go-client",
sum = "h1:kh5B4yWjrd5BcynJoA4014mZlI/j6++inCMMQoKUOtI=",
version = "v1.4.1",
)
go_repository(
name = "com_github_qustavo_sqlhooks_v2",
build_file_proto_mode = "disable_global",

1
dev/ci/images/images.go
View File

@ -125,7 +125,6 @@ var DeploySourcegraphDockerImages = []string{
"postgres_exporter",
"precise-code-intel-worker",
"prometheus",
"qdrant",
"redis-cache",
"redis-store",
"redis_exporter",

6
dev/oci_deps.bzl
View File

@ -196,12 +196,6 @@ def oci_deps():
image = "index.docker.io/sourcegraph/wolfi-blobstore-base",
)
oci_pull(
name = "wolfi_qdrant_base",
digest = "sha256:6b0210c48107719494a6b59c9542817bb4222c47038ae3bf1cc29fed563770b3",
image = "index.docker.io/sourcegraph/wolfi-qdrant-base",
)
oci_pull(
name = "scip-java",
digest = "sha256:808b063b7376cfc0a4937d89ddc3d4dd9652d10609865fae3f3b34302132737a",

2
doc/dev/background-information/sg/reference.md
View File

@ -52,7 +52,6 @@ Available commandsets in `sg.config.yaml`:
* monitoring
* monitoring-alerts
* otel
* qdrant
* single-program
* web-standalone
* web-standalone-prod
@ -129,7 +128,6 @@ Available commands in `sg.config.yaml`:
* pings
* postgres_exporter
* prometheus
* qdrant
* redis-postgres: Dockerized version of redis and postgres
* repo-updater
* searcher

42
docker-images/qdrant/BUILD.bazel
View File

@ -1,42 +0,0 @@
load("@container_structure_test//:defs.bzl", "container_structure_test")
load("//dev:oci_defs.bzl", "image_repository", "oci_image", "oci_push", "oci_tarball")
load("//wolfi-images:defs.bzl", "wolfi_base")
oci_image(
name = "qdrant_image",
base = ":base_image",
entrypoint = [
"/sbin/tini",
"--",
"/usr/local/bin/qdrant",
"--config-path=/etc/qdrant/config.yaml",
],
env = {},
user = "sourcegraph",
)
container_structure_test(
name = "qdrant_image_test",
timeout = "short",
configs = ["qdrant_image_test.yaml"],
driver = "docker",
image = ":qdrant_image",
tags = [
"exclusive",
"requires-network",
],
)
oci_tarball(
name = "qdrant_image_tarball",
image = ":qdrant_image",
repo_tags = ["qdrant:candidate"],
)
oci_push(
name = "qdrant_candidate_push",
image = ":qdrant_image",
repository = image_repository("qdrant"),
)
wolfi_base()

20
docker-images/qdrant/qdrant_image_test.yaml
View File

@ -1,20 +0,0 @@
schemaVersion: "2.0.0"
commandTests:
- name: "not running as root"
command: "/usr/bin/id"
args:
- -u
excludedOutput: ["^0"]
exitCode: 0
- name: "qdrant is runnable"
command: "/usr/local/bin/qdrant"
args:
- --version
fileExistenceTests:
- name: 'Test for qdrant'
path: '/usr/local/bin/qdrant'
shouldExist: true
uid: 0
gid: 0

1
go.mod
View File

@ -273,7 +273,6 @@ require (
github.com/oschwald/maxminddb-golang v1.12.0
github.com/pkoukk/tiktoken-go v0.1.6
github.com/prometheus/statsd_exporter v0.22.7
github.com/qdrant/go-client v1.4.1
github.com/sourcegraph/cloud-api v0.0.0-20231205211631-907f2d5f11b7
github.com/sourcegraph/managed-services-platform-cdktf/gen/cloudflare v0.0.0-20230822024612-edb48c530722
github.com/sourcegraph/managed-services-platform-cdktf/gen/google v0.0.0-20240325114905-87053fe51a82

2
go.sum
View File

@ -1553,8 +1553,6 @@ github.com/pseudomuto/protoc-gen-doc v1.5.1 h1:Ah259kcrio7Ix1Rhb6u8FCaOkzf9qRBqX
github.com/pseudomuto/protoc-gen-doc v1.5.1/go.mod h1:XpMKYg6zkcpgfpCfQ8GcWBDRtRxOmMR5w7pz4Xo+dYM=
github.com/pseudomuto/protokit v0.2.1 h1:kCYpE3thoR6Esm0CUvd5xbrDTOZPvQPTDeyXpZfrJdk=
github.com/pseudomuto/protokit v0.2.1/go.mod h1:gt7N5Rz2flBzYafvaxyIxMZC0TTF5jDZfRnw25hAAyo=
github.com/qdrant/go-client v1.4.1 h1:kh5B4yWjrd5BcynJoA4014mZlI/j6++inCMMQoKUOtI=
github.com/qdrant/go-client v1.4.1/go.mod h1:680gkxNAsVtre0Z8hAQmtPzJtz1xFAyCu2TUxULtnoE=
github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU=
github.com/rafaeljusto/redigomock/v3 v3.1.2 h1:B4Y0XJQiPjpwYmkH55aratKX1VfR+JRqzmDKyZbC99o=

47
internal/api/internalapi/v1/internalapi.pb.go generated
View File

@ -199,8 +199,6 @@ type ServiceConnections struct {
// Embeddings is the addresses of embeddings instances that should be talked
// to.
Embeddings []string `protobuf:"bytes,7,rep,name=embeddings,proto3" json:"embeddings,omitempty"`
// Qdrant is the address of the Qdrant instance (or empty if disabled)
Qdrant string `protobuf:"bytes,8,opt,name=qdrant,proto3" json:"qdrant,omitempty"`
// Zoekts is the addresses of Zoekt instances to talk to.
Zoekts []string `protobuf:"bytes,9,rep,name=zoekts,proto3" json:"zoekts,omitempty"`
// ZoektListTTL is the TTL of the internal cache that Zoekt clients use to
@ -290,13 +288,6 @@ func (x *ServiceConnections) GetEmbeddings() []string {
return nil
}
func (x *ServiceConnections) GetQdrant() string {
if x != nil {
return x.Qdrant
}
return ""
}
func (x *ServiceConnections) GetZoekts() []string {
if x != nil {
return x.Zoekts
@ -333,7 +324,7 @@ var file_internalapi_proto_rawDesc = []byte{
0x0b, 0x32, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f,
0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x84, 0x03,
0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xfa, 0x02,
0x0a, 0x12, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74,
0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x67, 0x69, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x76,
0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x69, 0x74, 0x53, 0x65,
@ -351,25 +342,25 @@ var file_internalapi_proto_rawDesc = []byte{
0x62, 0x6f, 0x6c, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x79, 0x6d, 0x62,
0x6f, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67,
0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69,
0x6e, 0x67, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x71, 0x64, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x08, 0x20,
0x01, 0x28, 0x09, 0x52, 0x06, 0x71, 0x64, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x7a,
0x6f, 0x65, 0x6b, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x7a, 0x6f, 0x65,
0x6b, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x5f, 0x6c, 0x69, 0x73,
0x74, 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x4c, 0x69, 0x73,
0x74, 0x54, 0x74, 0x6c, 0x32, 0x6e, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5d, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61,
0x6c, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69,
0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69,
0x6e, 0x67, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x73, 0x18, 0x09, 0x20,
0x03, 0x28, 0x09, 0x52, 0x06, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0e, 0x7a,
0x6f, 0x65, 0x6b, 0x74, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x0a, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c,
0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x74, 0x6c, 0x4a, 0x04, 0x08, 0x08,
0x10, 0x09, 0x52, 0x06, 0x71, 0x64, 0x72, 0x61, 0x6e, 0x74, 0x32, 0x6e, 0x0a, 0x0d, 0x43, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5d, 0x0a, 0x09, 0x47,
0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69,
0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65,
0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
0x03, 0x90, 0x02, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x73,
0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72,
0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25,
0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x61, 0x70, 0x69,
0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69,
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67,
0x72, 0x61, 0x70, 0x68, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68,
0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x69, 0x6e,
0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (

4
internal/api/internalapi/v1/internalapi.proto
View File

@ -50,8 +50,8 @@ message ServiceConnections {
// Embeddings is the addresses of embeddings instances that should be talked
// to.
repeated string embeddings = 7;
// Qdrant is the address of the Qdrant instance (or empty if disabled)
string qdrant = 8;
reserved 8;
reserved "qdrant";
// Zoekts is the addresses of Zoekt instances to talk to.
repeated string zoekts = 9;
// ZoektListTTL is the TTL of the internal cache that Zoekt clients use to

54
internal/conf/computed.go
View File

@ -1060,59 +1060,6 @@ func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.Embeddi
MaxFileSizeBytes: maxFileSizeLimit,
}
// Default values should match the documented defaults in site.schema.json.
computedQdrantConfig := conftypes.QdrantConfig{
Enabled: false,
QdrantHNSWConfig: conftypes.QdrantHNSWConfig{
EfConstruct: nil,
FullScanThreshold: nil,
M: nil,
OnDisk: true,
PayloadM: nil,
},
QdrantOptimizersConfig: conftypes.QdrantOptimizersConfig{
IndexingThreshold: 0,
MemmapThreshold: 100,
},
QdrantQuantizationConfig: conftypes.QdrantQuantizationConfig{
Enabled: true,
Quantile: 0.98,
},
}
if embeddingsConfig.Qdrant != nil {
qc := embeddingsConfig.Qdrant
computedQdrantConfig.Enabled = qc.Enabled
if qc.Hnsw != nil {
computedQdrantConfig.QdrantHNSWConfig.EfConstruct = toUint64(qc.Hnsw.EfConstruct)
computedQdrantConfig.QdrantHNSWConfig.FullScanThreshold = toUint64(qc.Hnsw.FullScanThreshold)
computedQdrantConfig.QdrantHNSWConfig.M = toUint64(qc.Hnsw.M)
computedQdrantConfig.QdrantHNSWConfig.PayloadM = toUint64(qc.Hnsw.PayloadM)
if qc.Hnsw.OnDisk != nil {
computedQdrantConfig.QdrantHNSWConfig.OnDisk = *qc.Hnsw.OnDisk
}
}
if qc.Optimizers != nil {
if qc.Optimizers.IndexingThreshold != nil {
computedQdrantConfig.QdrantOptimizersConfig.IndexingThreshold = uint64(*qc.Optimizers.IndexingThreshold)
}
if qc.Optimizers.MemmapThreshold != nil {
computedQdrantConfig.QdrantOptimizersConfig.MemmapThreshold = uint64(*qc.Optimizers.MemmapThreshold)
}
}
if qc.Quantization != nil {
if qc.Quantization.Enabled != nil {
computedQdrantConfig.QdrantQuantizationConfig.Enabled = *qc.Quantization.Enabled
}
if qc.Quantization.Quantile != nil {
computedQdrantConfig.QdrantQuantizationConfig.Quantile = float32(*qc.Quantization.Quantile)
}
}
}
computedConfig := &conftypes.EmbeddingsConfig{
Provider: conftypes.EmbeddingsProviderName(embeddingsConfig.Provider),
AccessToken: embeddingsConfig.AccessToken,
@ -1126,7 +1073,6 @@ func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.Embeddi
MaxTextEmbeddingsPerRepo: embeddingsConfig.MaxTextEmbeddingsPerRepo,
PolicyRepositoryMatchLimit: embeddingsConfig.PolicyRepositoryMatchLimit,
ExcludeChunkOnError: pointers.Deref(embeddingsConfig.ExcludeChunkOnError, true),
Qdrant: computedQdrantConfig,
PerCommunityUserEmbeddingsMonthlyLimit: embeddingsConfig.PerCommunityUserEmbeddingsMonthlyLimit,
PerProUserEmbeddingsMonthlyLimit: embeddingsConfig.PerProUserEmbeddingsMonthlyLimit,
}

20
internal/conf/computed_test.go
View File

@ -767,19 +767,6 @@ func TestGetFeaturesConfig(t *testing.T) {
func TestGetEmbeddingsConfig(t *testing.T) {
licenseKey := "theasdfkey"
licenseAccessToken := license.GenerateLicenseKeyBasedAccessToken(licenseKey)
defaultQdrantConfig := conftypes.QdrantConfig{
QdrantHNSWConfig: conftypes.QdrantHNSWConfig{
OnDisk: true,
},
QdrantOptimizersConfig: conftypes.QdrantOptimizersConfig{
IndexingThreshold: 0,
MemmapThreshold: 100,
},
QdrantQuantizationConfig: conftypes.QdrantQuantizationConfig{
Enabled: true,
Quantile: 0.98,
},
}
zeroConfigDefaultWithLicense := &conftypes.EmbeddingsConfig{
Provider: "sourcegraph",
AccessToken: licenseAccessToken,
@ -795,7 +782,6 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
}
testCases := []struct {
@ -911,7 +897,6 @@ func TestGetEmbeddingsConfig(t *testing.T) {
ExcludedFilePathPatterns: []string{"*.java"},
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -945,7 +930,6 @@ func TestGetEmbeddingsConfig(t *testing.T) {
ExcludedFilePathPatterns: []string{"*.java"},
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -981,7 +965,6 @@ func TestGetEmbeddingsConfig(t *testing.T) {
ExcludedFilePathPatterns: []string{"*.java"},
},
ExcludeChunkOnError: false,
Qdrant: defaultQdrantConfig,
},
},
{
@ -1009,7 +992,6 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -1050,7 +1032,6 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -1094,7 +1075,6 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{

4
internal/conf/conftypes/conftypes.go
View File

@ -38,8 +38,6 @@ type ServiceConnections struct {
Symbols []string `json:"symbols"`
// Embeddings is the addresses of embeddings instances that should be talked to.
Embeddings []string `json:"embeddings"`
// Qdrant is the address of the Qdrant instance (or empty if disabled)
Qdrant string `json:"qdrant"`
// Zoekts is the addresses of Zoekt instances to talk to.
Zoekts []string `json:"zoekts"`
// ZoektListTTL is the TTL of the internal cache that Zoekt clients use to
@ -57,7 +55,6 @@ func (sc *ServiceConnections) ToProto() *proto.ServiceConnections {
Searchers: sc.Searchers,
Symbols: sc.Symbols,
Embeddings: sc.Embeddings,
Qdrant: sc.Qdrant,
Zoekts: sc.Zoekts,
ZoektListTtl: durationpb.New(sc.ZoektListTTL),
}
@ -72,7 +69,6 @@ func (sc *ServiceConnections) FromProto(in *proto.ServiceConnections) {
Searchers: in.GetSearchers(),
Symbols: in.GetSymbols(),
Embeddings: in.GetEmbeddings(),
Qdrant: in.GetQdrant(),
Zoekts: in.GetZoekts(),
ZoektListTTL: in.GetZoektListTtl().AsDuration(),
}

26
internal/conf/conftypes/consts.go
View File

@ -57,36 +57,10 @@ type EmbeddingsConfig struct {
MaxTextEmbeddingsPerRepo int
PolicyRepositoryMatchLimit *int
ExcludeChunkOnError bool
Qdrant QdrantConfig
PerCommunityUserEmbeddingsMonthlyLimit int
PerProUserEmbeddingsMonthlyLimit int
}
type QdrantConfig struct {
Enabled bool
QdrantHNSWConfig QdrantHNSWConfig
QdrantOptimizersConfig QdrantOptimizersConfig
QdrantQuantizationConfig QdrantQuantizationConfig
}
type QdrantHNSWConfig struct {
EfConstruct *uint64
FullScanThreshold *uint64
M *uint64
OnDisk bool
PayloadM *uint64
}
type QdrantOptimizersConfig struct {
IndexingThreshold uint64
MemmapThreshold uint64
}
type QdrantQuantizationConfig struct {
Enabled bool
Quantile float32
}
type EmbeddingsProviderName string
const (

122
internal/embeddings/background/repo/mocks_temp.go
View File

@ -58,9 +58,6 @@ type MockRepoEmbeddingJobsStore struct {
// ListRepoEmbeddingJobsFunc is an instance of a mock function object
// controlling the behavior of the method ListRepoEmbeddingJobs.
ListRepoEmbeddingJobsFunc *RepoEmbeddingJobsStoreListRepoEmbeddingJobsFunc
// RescheduleAllReposFunc is an instance of a mock function object
// controlling the behavior of the method RescheduleAllRepos.
RescheduleAllReposFunc *RepoEmbeddingJobsStoreRescheduleAllReposFunc
// TransactFunc is an instance of a mock function object controlling the
// behavior of the method Transact.
TransactFunc *RepoEmbeddingJobsStoreTransactFunc
@ -135,11 +132,6 @@ func NewMockRepoEmbeddingJobsStore() *MockRepoEmbeddingJobsStore {
return
},
},
RescheduleAllReposFunc: &RepoEmbeddingJobsStoreRescheduleAllReposFunc{
defaultHook: func(context.Context) (r0 error) {
return
},
},
TransactFunc: &RepoEmbeddingJobsStoreTransactFunc{
defaultHook: func(context.Context) (r0 RepoEmbeddingJobsStore, r1 error) {
return
@ -218,11 +210,6 @@ func NewStrictMockRepoEmbeddingJobsStore() *MockRepoEmbeddingJobsStore {
panic("unexpected invocation of MockRepoEmbeddingJobsStore.ListRepoEmbeddingJobs")
},
},
RescheduleAllReposFunc: &RepoEmbeddingJobsStoreRescheduleAllReposFunc{
defaultHook: func(context.Context) error {
panic("unexpected invocation of MockRepoEmbeddingJobsStore.RescheduleAllRepos")
},
},
TransactFunc: &RepoEmbeddingJobsStoreTransactFunc{
defaultHook: func(context.Context) (RepoEmbeddingJobsStore, error) {
panic("unexpected invocation of MockRepoEmbeddingJobsStore.Transact")
@ -277,9 +264,6 @@ func NewMockRepoEmbeddingJobsStoreFrom(i RepoEmbeddingJobsStore) *MockRepoEmbedd
ListRepoEmbeddingJobsFunc: &RepoEmbeddingJobsStoreListRepoEmbeddingJobsFunc{
defaultHook: i.ListRepoEmbeddingJobs,
},
RescheduleAllReposFunc: &RepoEmbeddingJobsStoreRescheduleAllReposFunc{
defaultHook: i.RescheduleAllRepos,
},
TransactFunc: &RepoEmbeddingJobsStoreTransactFunc{
defaultHook: i.Transact,
},
@ -1607,112 +1591,6 @@ func (c RepoEmbeddingJobsStoreListRepoEmbeddingJobsFuncCall) Results() []interfa
return []interface{}{c.Result0, c.Result1}
}
// RepoEmbeddingJobsStoreRescheduleAllReposFunc describes the behavior when
// the RescheduleAllRepos method of the parent MockRepoEmbeddingJobsStore
// instance is invoked.
type RepoEmbeddingJobsStoreRescheduleAllReposFunc struct {
defaultHook func(context.Context) error
hooks []func(context.Context) error
history []RepoEmbeddingJobsStoreRescheduleAllReposFuncCall
mutex sync.Mutex
}
// RescheduleAllRepos delegates to the next hook function in the queue and
// stores the parameter and result values of this invocation.
func (m *MockRepoEmbeddingJobsStore) RescheduleAllRepos(v0 context.Context) error {
r0 := m.RescheduleAllReposFunc.nextHook()(v0)
m.RescheduleAllReposFunc.appendCall(RepoEmbeddingJobsStoreRescheduleAllReposFuncCall{v0, r0})
return r0
}
// SetDefaultHook sets function that is called when the RescheduleAllRepos
// method of the parent MockRepoEmbeddingJobsStore instance is invoked and
// the hook queue is empty.
func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) SetDefaultHook(hook func(context.Context) error) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// RescheduleAllRepos method of the parent MockRepoEmbeddingJobsStore
// instance invokes the hook at the front of the queue and discards it.
// After the queue is empty, the default hook function is invoked for any
// future action.
func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) PushHook(hook func(context.Context) error) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) SetDefaultReturn(r0 error) {
f.SetDefaultHook(func(context.Context) error {
return r0
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) PushReturn(r0 error) {
f.PushHook(func(context.Context) error {
return r0
})
}
func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) nextHook() func(context.Context) error {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) appendCall(r0 RepoEmbeddingJobsStoreRescheduleAllReposFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of
// RepoEmbeddingJobsStoreRescheduleAllReposFuncCall objects describing the
// invocations of this function.
func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) History() []RepoEmbeddingJobsStoreRescheduleAllReposFuncCall {
f.mutex.Lock()
history := make([]RepoEmbeddingJobsStoreRescheduleAllReposFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// RepoEmbeddingJobsStoreRescheduleAllReposFuncCall is an object that
// describes an invocation of method RescheduleAllRepos on an instance of
// MockRepoEmbeddingJobsStore.
type RepoEmbeddingJobsStoreRescheduleAllReposFuncCall struct {
// Arg0 is the value of the 1st argument passed to this method
// invocation.
Arg0 context.Context
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c RepoEmbeddingJobsStoreRescheduleAllReposFuncCall) Args() []interface{} {
return []interface{}{c.Arg0}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c RepoEmbeddingJobsStoreRescheduleAllReposFuncCall) Results() []interface{} {
return []interface{}{c.Result0}
}
// RepoEmbeddingJobsStoreTransactFunc describes the behavior when the
// Transact method of the parent MockRepoEmbeddingJobsStore instance is
// invoked.

22
internal/embeddings/background/repo/store.go
View File

@ -106,7 +106,6 @@ type RepoEmbeddingJobsStore interface {
CountRepoEmbeddingJobs(ctx context.Context, args ListOpts) (int, error)
GetEmbeddableRepos(ctx context.Context, opts EmbeddableRepoOpts) ([]EmbeddableRepo, error)
CancelRepoEmbeddingJob(ctx context.Context, job int) error
RescheduleAllRepos(ctx context.Context) error
UpdateRepoEmbeddingJobStats(ctx context.Context, jobID int, stats *EmbedRepoStats) error
GetRepoEmbeddingJobStats(ctx context.Context, jobID int) (EmbedRepoStats, error)
@ -256,27 +255,6 @@ func (s *repoEmbeddingJobsStore) CreateRepoEmbeddingJob(ctx context.Context, rep
return id, err
}
func (s *repoEmbeddingJobsStore) RescheduleAllRepos(ctx context.Context) error {
const rescheduleAllQuery = `
INSERT INTO repo_embedding_jobs (repo_id, revision)
SELECT * FROM (
SELECT
DISTINCT repo_id,
(
SELECT r2.revision
FROM repo_embedding_jobs r2
WHERE r2.repo_id = r1.repo_id
AND state = 'completed'
ORDER BY finished_at DESC
LIMIT 1
) latest_successful_revision
FROM repo_embedding_jobs r1
) subquery
WHERE latest_successful_revision IS NOT NULL
`
return s.Store.Exec(ctx, sqlf.Sprintf(rescheduleAllQuery))
}
var repoEmbeddingJobStatsColumns = []*sqlf.Query{
sqlf.Sprintf("repo_embedding_job_stats.job_id"),
sqlf.Sprintf("repo_embedding_job_stats.is_incremental"),

42
internal/embeddings/background/repo/store_test.go
View File

@ -142,48 +142,6 @@ func TestRepoEmbeddingJobsStore(t *testing.T) {
})
}
func TestRescheduleAll(t *testing.T) {
logger := logtest.Scoped(t)
db := database.NewDB(logger, dbtest.NewDB(t))
repoStore := db.Repos()
ctx := context.Background()
repo1 := &types.Repo{Name: "github.com/sourcegraph/sourcegraph", URI: "github.com/sourcegraph/sourcegraph", ExternalRepo: api.ExternalRepoSpec{}}
err := repoStore.Create(ctx, repo1)
require.NoError(t, err)
repo2 := &types.Repo{Name: "github.com/sourcegraph/sourcegraph2", URI: "github.com/sourcegraph/sourcegraph2", ExternalRepo: api.ExternalRepoSpec{}}
err = repoStore.Create(ctx, repo2)
require.NoError(t, err)
repo3 := &types.Repo{Name: "github.com/sourcegraph/sourcegraph3", URI: "github.com/sourcegraph/sourcegraph3", ExternalRepo: api.ExternalRepoSpec{}}
err = repoStore.Create(ctx, repo3)
require.NoError(t, err)
// Insert three completed jobs from two repos
_, err = db.Handle().ExecContext(ctx, fmt.Sprintf(
`insert into repo_embedding_jobs (repo_id, revision, state) values
(%d, 'rev1', 'completed'),
(%d, 'rev2', 'completed'),
(%d, 'rev3', 'completed'),
(%d, 'rev4', 'failed')
`,
repo1.ID,
repo1.ID,
repo2.ID,
repo3.ID,
))
require.NoError(t, err)
store := NewRepoEmbeddingJobsStore(db)
err = store.RescheduleAllRepos(ctx)
require.NoError(t, err)
jobs, err := store.ListRepoEmbeddingJobs(ctx, ListOpts{PaginationArgs: &database.PaginationArgs{}})
require.NoError(t, err)
require.Len(t, jobs, 6) // 4 jobs to start, added 2
}
func TestCancelRepoEmbeddingJob(t *testing.T) {
logger := logtest.Scoped(t)
db := database.NewDB(logger, dbtest.NewDB(t))

46
internal/embeddings/db/BUILD.bazel
View File

@ -1,46 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//dev:go_defs.bzl", "go_test")
go_library(
name = "db",
srcs = [
"chunk_point.go",
"conf.go",
"db.go",
"migrate.go",
"noop.go",
"qdrant.go",
],
importpath = "github.com/sourcegraph/sourcegraph/internal/embeddings/db",
visibility = ["//:__subpackages__"],
deps = [
"//internal/api",
"//internal/conf",
"//internal/conf/conftypes",
"//internal/grpc/defaults",
"//lib/errors",
"//lib/pointers",
"@com_github_google_uuid//:uuid",
"@com_github_qdrant_go_client//qdrant",
"@com_github_sourcegraph_log//:log",
"@org_golang_google_grpc//:go_default_library",
],
)
go_test(
name = "db_test",
srcs = [
"chunk_point_test.go",
"conf_test.go",
],
embed = [":db"],
deps = [
"//internal/conf",
"//internal/conf/conftypes",
"//internal/dotcom",
"//lib/pointers",
"//schema",
"@com_github_sourcegraph_log//logtest",
"@com_github_stretchr_testify//require",
],
)

151
internal/embeddings/db/chunk_point.go
View File

@ -1,151 +0,0 @@
package db
import (
"encoding/binary"
"hash/fnv"
"github.com/google/uuid"
qdrant "github.com/qdrant/go-client/qdrant"
"github.com/sourcegraph/sourcegraph/internal/api"
)
// ChunkResult is a point along with its search score.
type ChunkResult struct {
Point ChunkPoint
Score float32
}
func (c *ChunkResult) FromQdrantResult(res *qdrant.ScoredPoint) error {
u, err := uuid.Parse(res.GetId().GetUuid())
if err != nil {
return err
}
var payload ChunkPayload
payload.FromQdrantPayload(res.GetPayload())
*c = ChunkResult{
Point: ChunkPoint{
ID: u,
Payload: payload,
Vector: res.GetVectors().GetVector().GetData(),
},
Score: res.GetScore(),
}
return nil
}
func NewChunkPoint(payload ChunkPayload, vector []float32) ChunkPoint {
return ChunkPoint{
ID: chunkUUID(
payload.RepoID,
payload.Revision,
payload.FilePath,
payload.StartLine,
payload.EndLine,
),
Payload: payload,
Vector: vector,
}
}
type ChunkPoint struct {
ID uuid.UUID
Payload ChunkPayload
Vector []float32
}
func (c *ChunkPoint) ToQdrantPoint() *qdrant.PointStruct {
return &qdrant.PointStruct{
Id: &qdrant.PointId{
PointIdOptions: &qdrant.PointId_Uuid{
Uuid: c.ID.String(),
},
},
Payload: c.Payload.ToQdrantPayload(),
Vectors: &qdrant.Vectors{
VectorsOptions: &qdrant.Vectors_Vector{
Vector: &qdrant.Vector{
Data: c.Vector,
},
},
},
}
}
type ChunkPoints []ChunkPoint
func (ps ChunkPoints) ToQdrantPoints() []*qdrant.PointStruct {
res := make([]*qdrant.PointStruct, len(ps))
for i, p := range ps {
res[i] = p.ToQdrantPoint()
}
return res
}
type PayloadField = string
const (
fieldRepoID PayloadField = "repoID"
fieldRepoName PayloadField = "repoName"
fieldRevision PayloadField = "revision"
fieldFilePath PayloadField = "filePath"
fieldStartLine PayloadField = "startLine"
fieldEndLine PayloadField = "endLine"
fieldIsCode PayloadField = "isCode"
)
// ChunkPayload is a well-typed representation of the payload we store in the vector DB.
// Changes to the contents of this struct may require a migration of the data in the DB.
type ChunkPayload struct {
RepoName api.RepoName
RepoID api.RepoID
Revision api.CommitID
FilePath string
StartLine, EndLine uint32
IsCode bool
}
func (p *ChunkPayload) ToQdrantPayload() map[string]*qdrant.Value {
return map[string]*qdrant.Value{
fieldRepoID: {Kind: &qdrant.Value_IntegerValue{IntegerValue: int64(p.RepoID)}},
fieldRepoName: {Kind: &qdrant.Value_StringValue{StringValue: string(p.RepoName)}},
fieldRevision: {Kind: &qdrant.Value_StringValue{StringValue: string(p.Revision)}},
fieldFilePath: {Kind: &qdrant.Value_StringValue{StringValue: p.FilePath}},
fieldStartLine: {Kind: &qdrant.Value_IntegerValue{IntegerValue: int64(p.StartLine)}},
fieldEndLine: {Kind: &qdrant.Value_IntegerValue{IntegerValue: int64(p.EndLine)}},
fieldIsCode: {Kind: &qdrant.Value_BoolValue{BoolValue: p.IsCode}},
}
}
func (p *ChunkPayload) FromQdrantPayload(payload map[string]*qdrant.Value) {
*p = ChunkPayload{
RepoName: api.RepoName(payload[fieldRepoName].GetStringValue()),
RepoID: api.RepoID(payload[fieldRepoID].GetIntegerValue()),
Revision: api.CommitID(payload[fieldRevision].GetStringValue()),
FilePath: payload[fieldFilePath].GetStringValue(),
StartLine: uint32(payload[fieldStartLine].GetIntegerValue()),
EndLine: uint32(payload[fieldEndLine].GetIntegerValue()),
IsCode: payload[fieldIsCode].GetBoolValue(),
}
}
// chunkUUID generates a stable UUID for a file chunk. It is not strictly necessary to have a stable ID,
// but it does make it easier to reason about idempotent updates.
func chunkUUID(repoID api.RepoID, revision api.CommitID, filePath string, startLine, endLine uint32) uuid.UUID {
hasher := fnv.New128()
var buf [4]byte
binary.LittleEndian.PutUint32(buf[:], uint32(repoID))
hasher.Write(buf[:])
hasher.Write([]byte(revision))
hasher.Write([]byte(filePath))
binary.LittleEndian.PutUint32(buf[:], startLine)
binary.LittleEndian.PutUint32(buf[:], endLine)
hasher.Write(buf[:])
var u uuid.UUID
sum := hasher.Sum(nil)
copy(u[:], sum)
return u
}

26
internal/embeddings/db/chunk_point_test.go
View File

@ -1,26 +0,0 @@
package db
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestPayload(t *testing.T) {
t.Run("roundtrip", func(t *testing.T) {
pp := ChunkPayload{
RepoName: "a",
RepoID: 2,
Revision: "c",
FilePath: "d",
StartLine: 5,
EndLine: 6,
IsCode: false,
}
qp := pp.ToQdrantPayload()
var newPP ChunkPayload
newPP.FromQdrantPayload(qp)
require.Equal(t, pp, newPP)
})
}

166
internal/embeddings/db/conf.go
View File

@ -1,166 +0,0 @@
package db
import (
"sync/atomic"
qdrant "github.com/qdrant/go-client/qdrant"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"google.golang.org/grpc"
)
// NewDBFromConfFunc returns a function that can be called to get an
// VectorDB instance based on the connection info from the conf package.
// It will watch conf and update the connection if there are any changes.
//
// If Qdrant is disabled, it will instead return the provided default VectorDB.
func NewDBFromConfFunc(logger log.Logger, def VectorDB) func() (VectorDB, error) {
type connAndErr struct {
conn *grpc.ClientConn
err error
}
var (
oldAddr string
ptr atomic.Pointer[connAndErr]
)
conf.Watch(func() {
c := conf.Get()
qc := conf.GetEmbeddingsConfig(c.SiteConfiguration)
if qc == nil || !qc.Qdrant.Enabled {
// Embeddings is disabled. Clear any errors and close any previous connection.
old := ptr.Swap(&connAndErr{nil, nil})
if old != nil && old.conn != nil {
old.conn.Close()
}
return
}
if newAddr := c.ServiceConnections().Qdrant; newAddr != oldAddr {
// The address has changed or this is running for the first time.
// Attempt to open dial Qdrant.
newConn, newErr := defaults.Dial(newAddr, logger)
oldAddr = newAddr
old := ptr.Swap(&connAndErr{newConn, newErr})
if old != nil && old.conn != nil {
old.conn.Close()
}
}
})
return func() (VectorDB, error) {
curr := ptr.Load()
if curr == nil {
return def, nil
}
if curr.err != nil {
return nil, curr.err
}
if curr.conn == nil {
return def, nil
}
return NewQdrantDBFromConn(curr.conn), nil
}
}
func createCollectionParams(name string, dims uint64, qc conftypes.QdrantConfig) *qdrant.CreateCollection {
return &qdrant.CreateCollection{
CollectionName: name,
HnswConfig: getHnswConfigDiff(qc.QdrantHNSWConfig),
OptimizersConfig: getOptimizersConfigDiff(qc.QdrantOptimizersConfig),
QuantizationConfig: getQuantizationConfig(qc.QdrantQuantizationConfig),
OnDiskPayload: pointers.Ptr(true),
VectorsConfig: getVectorsConfig(dims),
ShardNumber: nil, // default
WalConfig: nil, // default
ReplicationFactor: nil, // default
WriteConsistencyFactor: nil, // default
InitFromCollection: nil, // default
}
}
func updateCollectionParams(name string, qc conftypes.QdrantConfig) *qdrant.UpdateCollection {
return &qdrant.UpdateCollection{
CollectionName: name,
HnswConfig: getHnswConfigDiff(qc.QdrantHNSWConfig),
OptimizersConfig: getOptimizersConfigDiff(qc.QdrantOptimizersConfig),
QuantizationConfig: getQuantizationConfigDiff(qc.QdrantQuantizationConfig),
Params: nil, // do not update collection params
// Do not update vectors config.
// TODO(camdencheek): consider making OnDisk configurable
VectorsConfig: nil,
}
}
func getVectorsConfig(dims uint64) *qdrant.VectorsConfig {
return &qdrant.VectorsConfig{
Config: &qdrant.VectorsConfig_Params{
Params: &qdrant.VectorParams{
Size: dims,
Distance: qdrant.Distance_Cosine,
OnDisk: pointers.Ptr(true),
HnswConfig: nil, // use collection default
QuantizationConfig: nil, // use collection default
},
},
}
}
func getHnswConfigDiff(qhc conftypes.QdrantHNSWConfig) *qdrant.HnswConfigDiff {
return &qdrant.HnswConfigDiff{
M: qhc.M,
PayloadM: qhc.PayloadM,
EfConstruct: qhc.EfConstruct,
FullScanThreshold: qhc.FullScanThreshold,
OnDisk: &qhc.OnDisk,
MaxIndexingThreads: nil, // default
}
}
func getOptimizersConfigDiff(qoc conftypes.QdrantOptimizersConfig) *qdrant.OptimizersConfigDiff {
// Default values should match the documented defaults in site.schema.json.
return &qdrant.OptimizersConfigDiff{
IndexingThreshold: &qoc.IndexingThreshold,
MemmapThreshold: &qoc.MemmapThreshold,
DefaultSegmentNumber: nil,
VacuumMinVectorNumber: nil,
MaxOptimizationThreads: nil,
}
}
func getQuantizationConfigDiff(qqc conftypes.QdrantQuantizationConfig) *qdrant.QuantizationConfigDiff {
if !qqc.Enabled {
return &qdrant.QuantizationConfigDiff{
Quantization: &qdrant.QuantizationConfigDiff_Disabled{},
}
}
return &qdrant.QuantizationConfigDiff{
Quantization: &qdrant.QuantizationConfigDiff_Scalar{
Scalar: &qdrant.ScalarQuantization{
Type: qdrant.QuantizationType_Int8,
Quantile: &qqc.Quantile,
AlwaysRam: pointers.Ptr(false),
},
},
}
}
func getQuantizationConfig(qqc conftypes.QdrantQuantizationConfig) *qdrant.QuantizationConfig {
if !qqc.Enabled {
return nil
}
return &qdrant.QuantizationConfig{
Quantization: &qdrant.QuantizationConfig_Scalar{
Scalar: &qdrant.ScalarQuantization{
Type: qdrant.QuantizationType_Int8,
Quantile: &qqc.Quantile,
AlwaysRam: pointers.Ptr(false),
},
},
}
}

52
internal/embeddings/db/conf_test.go
View File

@ -1,52 +0,0 @@
package db
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/log/logtest"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/dotcom"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"github.com/sourcegraph/sourcegraph/schema"
)
func TestNewDBFromConfFunc(t *testing.T) {
dotcom.MockSourcegraphDotComMode(t, true)
t.Run("default nil", func(t *testing.T) {
conf.Mock(&conf.Unified{
ServiceConnectionConfig: conftypes.ServiceConnections{
Qdrant: "",
},
})
getDB := NewDBFromConfFunc(logtest.Scoped(t), nil)
got, err := getDB()
require.NoError(t, err)
require.Nil(t, got)
})
t.Run("fake addr", func(t *testing.T) {
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Provider: "sourcegraph",
AccessToken: "fake",
Enabled: pointers.Ptr(true),
Qdrant: &schema.Qdrant{
Enabled: true,
},
},
CodyEnabled: pointers.Ptr(true),
},
ServiceConnectionConfig: conftypes.ServiceConnections{Qdrant: "fake_address_but_it_does_not_matter_because_grpc_dialing_is_lazy"},
})
getDB := NewDBFromConfFunc(logtest.Scoped(t), nil)
got, err := getDB()
require.NoError(t, err)
require.NotNil(t, got)
})
}

29
internal/embeddings/db/db.go
View File

@ -1,29 +0,0 @@
package db
import (
"context"
"fmt"
"strings"
"github.com/sourcegraph/sourcegraph/internal/api"
)
type VectorDB interface {
VectorSearcher
VectorInserter
}
type VectorSearcher interface {
Search(context.Context, SearchParams) ([]ChunkResult, error)
}
type VectorInserter interface {
PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error
HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error)
InsertChunks(context.Context, InsertParams) error
FinalizeUpdate(context.Context, FinalizeUpdateParams) error
}
func CollectionName(modelID string) string {
return fmt.Sprintf("repos.%s", strings.ReplaceAll(modelID, "/", "."))
}

79
internal/embeddings/db/migrate.go
View File

@ -1,79 +0,0 @@
package db
import (
"context"
qdrant "github.com/qdrant/go-client/qdrant"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/pointers"
)
func ensureModelCollection(ctx context.Context, db *qdrantDB, modelID string, modelDims uint64) error {
// Make the actual collection end with `.default` so we can switch between
// configurations with aliases.
name := CollectionName(modelID)
realName := name + ".default"
err := ensureCollectionWithConfig(ctx, db.collectionsClient, realName, modelDims, conf.GetEmbeddingsConfig(conf.Get().SiteConfiguration).Qdrant)
if err != nil {
return err
}
// Update the alias atomically to point to the new collection
_, err = db.collectionsClient.UpdateAliases(ctx, &qdrant.ChangeAliases{
Actions: []*qdrant.AliasOperations{{
Action: &qdrant.AliasOperations_CreateAlias{
CreateAlias: &qdrant.CreateAlias{
CollectionName: realName,
AliasName: name,
},
},
}},
})
if err != nil {
return errors.Wrap(err, "update aliases")
}
err = ensureRepoIDIndex(ctx, db.pointsClient, realName)
if err != nil {
return errors.Wrap(err, "add repo index")
}
return nil
}
func ensureCollectionWithConfig(ctx context.Context, cc qdrant.CollectionsClient, name string, dims uint64, qc conftypes.QdrantConfig) error {
resp, err := cc.List(ctx, &qdrant.ListCollectionsRequest{})
if err != nil {
return err
}
for _, collection := range resp.GetCollections() {
if collection.GetName() == name {
_, err = cc.Update(ctx, updateCollectionParams(name, qc))
return err
}
}
_, err = cc.Create(ctx, createCollectionParams(name, dims, qc))
return err
}
func ensureRepoIDIndex(ctx context.Context, cc qdrant.PointsClient, name string) error {
// This is idempotent, so no need to check if it exists first
_, err := cc.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{
CollectionName: name,
Wait: pointers.Ptr(true),
FieldName: fieldRepoID,
FieldType: pointers.Ptr(qdrant.FieldType_FieldTypeInteger),
FieldIndexParams: nil,
Ordering: nil,
})
if err != nil {
return err
}
return nil
}

58
internal/embeddings/db/noop.go
View File

@ -1,58 +0,0 @@
package db
import (
"context"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
func NewNoopDB() VectorDB {
return noopDB{}
}
var _ VectorDB = noopDB{}
type noopDB struct{}
func (noopDB) Search(context.Context, SearchParams) ([]ChunkResult, error) {
return nil, nil
}
func (noopDB) PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error {
return nil
}
func (noopDB) HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) {
return false, nil
}
func (noopDB) InsertChunks(context.Context, InsertParams) error {
return nil
}
func (noopDB) FinalizeUpdate(context.Context, FinalizeUpdateParams) error {
return nil
}
var ErrDisabled = errors.New("Qdrant is disabled. Enable by setting QDRANT_ENDPOINT")
func NewDisabledDB() VectorDB {
return disabledDB{}
}
var _ VectorDB = disabledDB{}
type disabledDB struct{}
func (disabledDB) Search(context.Context, SearchParams) ([]ChunkResult, error) {
return nil, ErrDisabled
}
func (disabledDB) PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error {
return ErrDisabled
}
func (disabledDB) HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) {
return false, ErrDisabled
}
func (disabledDB) InsertChunks(context.Context, InsertParams) error {
return ErrDisabled
}
func (disabledDB) FinalizeUpdate(context.Context, FinalizeUpdateParams) error {
return ErrDisabled
}

284
internal/embeddings/db/qdrant.go
View File

@ -1,284 +0,0 @@
package db
import (
"context"
qdrant "github.com/qdrant/go-client/qdrant"
"google.golang.org/grpc"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/lib/pointers"
)
func NewQdrantDBFromConn(conn *grpc.ClientConn) VectorDB {
return NewQdrantDB(qdrant.NewPointsClient(conn), qdrant.NewCollectionsClient(conn))
}
func NewQdrantDB(pointsClient qdrant.PointsClient, collectionsClient qdrant.CollectionsClient) VectorDB {
return &qdrantDB{
pointsClient: pointsClient,
collectionsClient: collectionsClient,
}
}
type qdrantDB struct {
pointsClient qdrant.PointsClient
collectionsClient qdrant.CollectionsClient
}
var _ VectorDB = (*qdrantDB)(nil)
type SearchParams struct {
// RepoIDs is the set of repos to search.
// If empty, all repos are searched.
RepoIDs []api.RepoID
// The ID of the model that the query was embedded with.
// Embeddings for other models will not be searched.
ModelID string
// Query is the embedding for the search query.
// Its dimensions must match the model dimensions.
Query []float32
// The maximum number of code results to return
CodeLimit int
// The maximum number of text results to return
TextLimit int
}
func (db *qdrantDB) Search(ctx context.Context, params SearchParams) ([]ChunkResult, error) {
collectionName := CollectionName(params.ModelID)
getSearchPoints := func(isCode bool) *qdrant.SearchPoints {
var limit uint64
if isCode {
limit = uint64(params.CodeLimit)
} else {
limit = uint64(params.TextLimit)
}
return &qdrant.SearchPoints{
CollectionName: collectionName,
Vector: params.Query,
WithPayload: fullPayloadSelector,
Filter: &qdrant.Filter{
Should: repoIDsConditions(params.RepoIDs),
Must: []*qdrant.Condition{isCodeCondition(isCode)},
},
Limit: limit,
}
}
codeSearch := getSearchPoints(true)
textSearch := getSearchPoints(false)
resp, err := db.pointsClient.SearchBatch(ctx, &qdrant.SearchBatchPoints{
CollectionName: collectionName,
SearchPoints: []*qdrant.SearchPoints{codeSearch, textSearch},
})
if err != nil {
return nil, err
}
results := make([]ChunkResult, 0, params.CodeLimit+params.TextLimit)
for _, group := range resp.GetResult() {
for _, res := range group.GetResult() {
var cr ChunkResult
if err := cr.FromQdrantResult(res); err != nil {
return nil, err
}
results = append(results, cr)
}
}
return results, nil
}
func (db *qdrantDB) PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error {
return ensureModelCollection(ctx, db, modelID, modelDims)
}
func (db *qdrantDB) HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) {
resp, err := db.pointsClient.Scroll(ctx, &qdrant.ScrollPoints{
CollectionName: CollectionName(modelID),
Filter: &qdrant.Filter{
Must: []*qdrant.Condition{
repoIDCondition(repoID),
revisionCondition(revision),
},
},
Limit: pointers.Ptr(uint32(1)),
})
if err != nil {
return false, err
}
return len(resp.GetResult()) > 0, nil
}
type InsertParams struct {
ModelID string
ChunkPoints ChunkPoints
}
func (db *qdrantDB) InsertChunks(ctx context.Context, params InsertParams) error {
_, err := db.pointsClient.Upsert(ctx, &qdrant.UpsertPoints{
CollectionName: CollectionName(params.ModelID),
// Wait to avoid overloading the server
Wait: pointers.Ptr(true),
Points: params.ChunkPoints.ToQdrantPoints(),
Ordering: nil,
})
return err
}
type FinalizeUpdateParams struct {
ModelID string
RepoID api.RepoID
Revision api.CommitID
FilesToRemove []string
}
// TODO: document that this is idempotent and why it's important
func (db *qdrantDB) FinalizeUpdate(ctx context.Context, params FinalizeUpdateParams) error {
// First, delete the old files
err := db.deleteFiles(ctx, params)
if err != nil {
return err
}
// Then, update all the unchanged chunks to use the latest revision
err = db.updateRevisions(ctx, params)
if err != nil {
return err
}
return nil
}
func (db *qdrantDB) deleteFiles(ctx context.Context, params FinalizeUpdateParams) error {
// TODO: batch the deletes in case the file list is extremely large
filePathConditions := make([]*qdrant.Condition, len(params.FilesToRemove))
for i, path := range params.FilesToRemove {
filePathConditions[i] = filePathCondition(path)
}
_, err := db.pointsClient.Delete(ctx, &qdrant.DeletePoints{
CollectionName: CollectionName(params.ModelID),
Wait: pointers.Ptr(true), // wait until deleted before sending update
Ordering: &qdrant.WriteOrdering{Type: qdrant.WriteOrderingType_Strong},
Points: &qdrant.PointsSelector{
PointsSelectorOneOf: &qdrant.PointsSelector_Filter{
Filter: &qdrant.Filter{
// Only chunks for this repo
Must: []*qdrant.Condition{repoIDCondition(params.RepoID)},
// No chunks that are from the newest revision
MustNot: []*qdrant.Condition{revisionCondition(params.Revision)},
// Chunks that match at least one of the "to remove" filenames
Should: filePathConditions,
},
},
},
})
return err
}
func (db *qdrantDB) updateRevisions(ctx context.Context, params FinalizeUpdateParams) error {
_, err := db.pointsClient.SetPayload(ctx, &qdrant.SetPayloadPoints{
CollectionName: CollectionName(params.ModelID),
Wait: pointers.Ptr(true), // wait until deleted before sending update
Ordering: &qdrant.WriteOrdering{Type: qdrant.WriteOrderingType_Strong},
Payload: map[string]*qdrant.Value{
fieldRevision: {
Kind: &qdrant.Value_StringValue{
StringValue: string(params.Revision),
},
},
},
PointsSelector: &qdrant.PointsSelector{
PointsSelectorOneOf: &qdrant.PointsSelector_Filter{
Filter: &qdrant.Filter{
// Only chunks in this repo
Must: []*qdrant.Condition{repoIDCondition(params.RepoID)},
// Only chunks that are not already marked as part of this revision
MustNot: []*qdrant.Condition{revisionCondition(params.Revision)},
},
},
},
})
return err
}
func filePathCondition(path string) *qdrant.Condition {
return &qdrant.Condition{
ConditionOneOf: &qdrant.Condition_Field{
Field: &qdrant.FieldCondition{
Key: fieldFilePath,
Match: &qdrant.Match{
MatchValue: &qdrant.Match_Keyword{
Keyword: string(path),
},
},
},
},
}
}
func revisionCondition(revision api.CommitID) *qdrant.Condition {
return &qdrant.Condition{
ConditionOneOf: &qdrant.Condition_Field{
Field: &qdrant.FieldCondition{
Key: fieldRevision,
Match: &qdrant.Match{
MatchValue: &qdrant.Match_Keyword{
Keyword: string(revision),
},
},
},
},
}
}
func isCodeCondition(isCode bool) *qdrant.Condition {
return &qdrant.Condition{
ConditionOneOf: &qdrant.Condition_Field{
Field: &qdrant.FieldCondition{
Key: fieldIsCode,
Match: &qdrant.Match{
MatchValue: &qdrant.Match_Boolean{
Boolean: isCode,
},
},
},
},
}
}
func repoIDsConditions(ids []api.RepoID) []*qdrant.Condition {
conds := make([]*qdrant.Condition, len(ids))
for i, id := range ids {
conds[i] = repoIDCondition(id)
}
return conds
}
func repoIDCondition(repoID api.RepoID) *qdrant.Condition {
return &qdrant.Condition{
ConditionOneOf: &qdrant.Condition_Field{
Field: &qdrant.FieldCondition{
Key: fieldRepoID,
Match: &qdrant.Match{
MatchValue: &qdrant.Match_Integer{
Integer: int64(repoID),
},
},
},
},
}
}
// Select the full payload
var fullPayloadSelector = &qdrant.WithPayloadSelector{
SelectorOptions: &qdrant.WithPayloadSelector_Enable{
Enable: true,
},
}

2
internal/embeddings/embed/BUILD.bazel
View File

@ -19,7 +19,6 @@ go_library(
"//internal/conf/conftypes",
"//internal/embeddings",
"//internal/embeddings/background/repo",
"//internal/embeddings/db",
"//internal/embeddings/embed/client",
"//internal/embeddings/embed/client/azureopenai",
"//internal/embeddings/embed/client/openai",
@ -47,7 +46,6 @@ go_test(
"//internal/codeintel/types",
"//internal/embeddings",
"//internal/embeddings/background/repo",
"//internal/embeddings/db",
"//internal/embeddings/embed/client",
"//internal/paths",
"//internal/types",

44
internal/embeddings/embed/embed.go
View File

@ -11,7 +11,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/embeddings"
bgrepo "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo"
"github.com/sourcegraph/sourcegraph/internal/embeddings/db"
"github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client"
"github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client/azureopenai"
"github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client/openai"
@ -41,7 +40,6 @@ func NewEmbeddingsClient(config *conftypes.EmbeddingsConfig) (client.EmbeddingsC
func EmbedRepo(
ctx context.Context,
client client.EmbeddingsClient,
inserter db.VectorInserter,
contextService ContextService,
readLister FileReadLister,
repo types.RepoIDName,
@ -106,13 +104,6 @@ func EmbedRepo(
IsIncremental: isIncremental,
}
insertDB := func(batch []embeddings.RepoEmbeddingRowMetadata, embeddings []float32, isCode bool) error {
return inserter.InsertChunks(ctx, db.InsertParams{
ModelID: client.GetModelIdentifier(),
ChunkPoints: batchToChunkPoints(repo, opts.Revision, batch, embeddings, isCode),
})
}
insertIndex := func(index *embeddings.EmbeddingIndex, metadata []embeddings.RepoEmbeddingRowMetadata, vectors []float32) {
index.RowMetadata = append(index.RowMetadata, metadata...)
index.Embeddings = append(index.Embeddings, embeddings.Quantize(vectors, nil)...)
@ -125,9 +116,8 @@ func EmbedRepo(
}
codeIndex := newIndex(len(codeFileNames))
insertCode := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) error {
insertCode := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) {
insertIndex(&codeIndex, md, embeddings)
return insertDB(md, embeddings, true)
}
reportCodeProgress := func(codeIndexStats bgrepo.EmbedFilesStats) {
@ -150,9 +140,8 @@ func EmbedRepo(
stats.CodeIndexStats = codeIndexStats
textIndex := newIndex(len(textFileNames))
insertText := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) error {
insertText := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) {
insertIndex(&textIndex, md, embeddings)
return insertDB(md, embeddings, false)
}
reportTextProgress := func(textIndexStats bgrepo.EmbedFilesStats) {
@ -214,7 +203,7 @@ type FileFilters struct {
MaxFileSizeBytes int
}
type batchInserter func(metadata []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) error
type batchInserter func(metadata []embeddings.RepoEmbeddingRowMetadata, embeddings []float32)
type FlushResults struct {
size int
@ -335,9 +324,7 @@ func embedFiles(
cursor++
}
if err := insert(metadata, batchEmbeddings.Embeddings[:cursor*dimensions]); err != nil {
return nil, err
}
insert(metadata, batchEmbeddings.Embeddings[:cursor*dimensions])
batch = batch[:0] // reset batch
reportProgress(stats)
@ -418,29 +405,6 @@ func embedFiles(
return stats, nil
}
func batchToChunkPoints(repo types.RepoIDName, revision api.CommitID, batch []embeddings.RepoEmbeddingRowMetadata, embeddings []float32, isCode bool) []db.ChunkPoint {
if len(batch) == 0 {
return nil
}
dimensions := len(embeddings) / len(batch)
points := make([]db.ChunkPoint, 0, len(batch))
for i, chunk := range batch {
payload := db.ChunkPayload{
RepoName: repo.Name,
RepoID: repo.ID,
Revision: revision,
FilePath: chunk.FileName,
StartLine: uint32(chunk.StartLine),
EndLine: uint32(chunk.EndLine),
IsCode: isCode,
}
point := db.NewChunkPoint(payload, embeddings[i*dimensions:(i+1)*dimensions])
points = append(points, point)
}
return points
}
type FileReadLister interface {
FileReader
FileLister

41
internal/embeddings/embed/embed_test.go
View File

@ -17,7 +17,6 @@ import (
citypes "github.com/sourcegraph/sourcegraph/internal/codeintel/types"
"github.com/sourcegraph/sourcegraph/internal/embeddings"
bgrepo "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo"
"github.com/sourcegraph/sourcegraph/internal/embeddings/db"
"github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -39,7 +38,6 @@ func TestEmbedRepo(t *testing.T) {
}
revision := api.CommitID("deadbeef")
embeddingsClient := NewMockEmbeddingsClient()
inserter := db.NewNoopDB()
contextService := NewMockContextService()
contextService.SplitIntoEmbeddableChunksFunc.SetDefaultHook(defaultSplitter)
splitOptions := codeintelContext.SplitOptions{ChunkTokensThreshold: 8}
@ -150,7 +148,7 @@ func TestEmbedRepo(t *testing.T) {
noopReport := func(*bgrepo.EmbedRepoStats) {}
t.Run("no files", func(t *testing.T) {
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, newReadLister(), repoIDName, mockRepoPathRanks, opts, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, newReadLister(), repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
require.Len(t, index.CodeIndex.Embeddings, 0)
require.Len(t, index.TextIndex.Embeddings, 0)
@ -167,7 +165,7 @@ func TestEmbedRepo(t *testing.T) {
})
t.Run("code files only", func(t *testing.T) {
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, newReadLister("a.go"), repoIDName, mockRepoPathRanks, opts, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, newReadLister("a.go"), repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
require.Len(t, index.TextIndex.Embeddings, 0)
require.Len(t, index.CodeIndex.Embeddings, 6)
@ -191,7 +189,7 @@ func TestEmbedRepo(t *testing.T) {
})
t.Run("text files only", func(t *testing.T) {
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, newReadLister("b.md"), repoIDName, mockRepoPathRanks, opts, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, newReadLister("b.md"), repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
require.Len(t, index.CodeIndex.Embeddings, 0)
require.Len(t, index.TextIndex.Embeddings, 6)
@ -216,7 +214,7 @@ func TestEmbedRepo(t *testing.T) {
t.Run("mixed code and text files", func(t *testing.T) {
rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin")
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
require.Len(t, index.CodeIndex.Embeddings, 15)
require.Len(t, index.CodeIndex.RowMetadata, 5)
@ -252,7 +250,7 @@ func TestEmbedRepo(t *testing.T) {
t.Run("not included files", func(t *testing.T) {
rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin", "not_included.jl")
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
require.Len(t, index.CodeIndex.Embeddings, 15)
require.Len(t, index.CodeIndex.RowMetadata, 5)
@ -297,7 +295,7 @@ func TestEmbedRepo(t *testing.T) {
countingReporter := func(*bgrepo.EmbedRepoStats) {
statReports++
}
_, _, _, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, countingReporter)
_, _, _, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, countingReporter)
require.NoError(t, err)
require.Equal(t, 2, statReports, `
Expected one update for flush. This is subject to change if the
@ -312,7 +310,7 @@ func TestEmbedRepo(t *testing.T) {
optsCopy.MaxTextEmbeddings = 1
rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin")
index, _, _, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
index, _, _, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.NoError(t, err)
// a.md has 2 chunks, c.java has 3 chunks
@ -332,19 +330,19 @@ func TestEmbedRepo(t *testing.T) {
rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin")
misbehavingClient := &misbehavingEmbeddingsClient{embeddingsClient, 32} // too many dimensions
_, _, _, err := EmbedRepo(ctx, misbehavingClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
_, _, _, err := EmbedRepo(ctx, misbehavingClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.ErrorContains(t, err, "expected embeddings for batch to have length")
misbehavingClient = &misbehavingEmbeddingsClient{embeddingsClient, 32} // too few dimensions
_, _, _, err = EmbedRepo(ctx, misbehavingClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
_, _, _, err = EmbedRepo(ctx, misbehavingClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.ErrorContains(t, err, "expected embeddings for batch to have length")
misbehavingClient = &misbehavingEmbeddingsClient{embeddingsClient, 0} // empty return
_, _, _, err = EmbedRepo(ctx, misbehavingClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
_, _, _, err = EmbedRepo(ctx, misbehavingClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.ErrorContains(t, err, "expected embeddings for batch to have length")
erroringClient := &erroringEmbeddingsClient{embeddingsClient, errors.New("whoops")} // normal error
_, _, _, err = EmbedRepo(ctx, erroringClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
_, _, _, err = EmbedRepo(ctx, erroringClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.ErrorContains(t, err, "whoops")
})
@ -358,7 +356,7 @@ func TestEmbedRepo(t *testing.T) {
failed[1] = struct{}{}
partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed}
_, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
_, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.ErrorContains(t, err, "batch failed on file")
require.ErrorContains(t, err, "a.go", "for a chunk error, the error message should contain the file name")
@ -374,7 +372,7 @@ func TestEmbedRepo(t *testing.T) {
failed[3] = struct{}{}
partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed}
_, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
_, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.ErrorContains(t, err, "batch failed on file")
require.ErrorContains(t, err, "b.md", "for a chunk error, the error message should contain the file name")
@ -388,7 +386,6 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) {
repoIDName := types.RepoIDName{Name: repoName}
embeddingsClient := NewMockEmbeddingsClient()
contextService := NewMockContextService()
inserter := db.NewNoopDB()
contextService.SplitIntoEmbeddableChunksFunc.SetDefaultHook(defaultSplitter)
splitOptions := codeintelContext.SplitOptions{ChunkTokensThreshold: 8}
mockFiles := map[string][]byte{
@ -478,7 +475,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) {
remainingFailures: 1,
err: errors.New("FAIL"),
}
_, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
_, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
require.True(t, stats.CodeIndexStats.ChunksEmbedded > 0)
})
@ -493,7 +490,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) {
remainingFailures: 100,
err: errors.New("FAIL"),
}
_, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
_, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.Error(t, err)
})
@ -507,7 +504,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) {
remainingFailures: 1,
err: client.NewRateLimitExceededError(time.Now().Add(time.Minute)),
}
_, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
_, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.Error(t, err)
})
@ -522,7 +519,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) {
failed[7] = struct{}{}
partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed}
index, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
@ -572,7 +569,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) {
failed[8] = struct{}{}
partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed}
index, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport)
require.NoError(t, err)
@ -624,7 +621,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) {
failed[8] = struct{}{}
partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed}
index, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
index, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport)
require.NoError(t, err)

40
schema/schema.go
View File

@ -795,8 +795,6 @@ type Embeddings struct {
PolicyRepositoryMatchLimit *int `json:"policyRepositoryMatchLimit,omitempty"`
// Provider description: The provider to use for generating embeddings. Defaults to sourcegraph.
Provider string `json:"provider,omitempty"`
// Qdrant description: Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team.
Qdrant *Qdrant `json:"qdrant,omitempty"`
// Url description: The url to the external embedding API service. Deprecated, use endpoint instead.
Url string `json:"url,omitempty"`
}
@ -1534,20 +1532,6 @@ type Header struct {
Value string `json:"value"`
}
// Hnsw description: Overrides for the HNSW index config.
type Hnsw struct {
// EfConstruct description: Number of neighbours to consider during the index building. Larger the value, more accurate the search, more time required to build the index.
EfConstruct *int `json:"efConstruct,omitempty"`
// FullScanThreshold description: Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
FullScanThreshold *int `json:"fullScanThreshold,omitempty"`
// M description: Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.
M *int `json:"m,omitempty"`
// OnDisk description: Store HNSW index on disk.
OnDisk *bool `json:"onDisk,omitempty"`
// PayloadM description: Number of edges per node in the index graph. Larger the value, more accurate the search, more space required.
PayloadM *int `json:"payloadM,omitempty"`
}
// IdentityProvider description: The source of identity to use when computing permissions. This defines how to compute the GitLab identity to use for a given Sourcegraph user.
type IdentityProvider struct {
Oauth *OAuthIdentity
@ -1947,12 +1931,6 @@ type OpenTelemetry struct {
// Endpoint description: OpenTelemetry tracing collector endpoint. By default, Sourcegraph's "/-/debug/otlp" endpoint forwards data to the configured collector backend.
Endpoint string `json:"endpoint,omitempty"`
}
type Optimizers struct {
// IndexingThreshold description: Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing. Set to 0 to disable indexing
IndexingThreshold *int `json:"indexingThreshold,omitempty"`
// MemmapThreshold description: Maximum size (in kilobytes) of vectors to store in-memory per segment.
MemmapThreshold *int `json:"memmapThreshold,omitempty"`
}
// OrganizationInvitations description: Configuration for organization invitations.
type OrganizationInvitations struct {
@ -2113,24 +2091,6 @@ type PythonRateLimit struct {
// RequestsPerHour description: Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.
RequestsPerHour float64 `json:"requestsPerHour"`
}
// Qdrant description: Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team.
type Qdrant struct {
Enabled bool `json:"enabled,omitempty"`
// Hnsw description: Overrides for the HNSW index config.
Hnsw *Hnsw `json:"hnsw,omitempty"`
Optimizers *Optimizers `json:"optimizers,omitempty"`
// Quantization description: Overrides for quantization config.
Quantization *Quantization `json:"quantization,omitempty"`
}
// Quantization description: Overrides for quantization config.
type Quantization struct {
// Enabled description: Whether to enable int8 scalar quantization
Enabled *bool `json:"enabled,omitempty"`
// Quantile description: Any values that lie outside the quantile range will be truncated
Quantile *float64 `json:"quantile,omitempty"`
}
type QuickLink struct {
// Description description: A description for this quick link
Description string `json:"description,omitempty"`

110
schema/site.schema.json
View File

@ -2762,116 +2762,6 @@
"pointer": true
},
"default": true
},
"qdrant": {
"description": "Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team.",
"type": "object",
"$comment": "Skipped fields from HnswConfigDiff: MaxIndexingThreads",
"properties": {
"enabled": {
"type": "boolean",
"default": false
},
"hnsw": {
"description": "Overrides for the HNSW index config.",
"type": "object",
"properties": {
"m": {
"description": "Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 8
},
"efConstruct": {
"description": "Number of neighbours to consider during the index building. Larger the value, more accurate the search, more time required to build the index.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 100
},
"fullScanThreshold": {
"description": "Minimal size (in KiloBytes) of vectors for additional payload-based indexing.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 1000
},
"onDisk": {
"description": "Store HNSW index on disk.",
"type": "boolean",
"!go": {
"pointer": true
},
"default": true
},
"payloadM": {
"description": "Number of edges per node in the index graph. Larger the value, more accurate the search, more space required.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 8
}
}
},
"quantization": {
"description": "Overrides for quantization config.",
"type": "object",
"$comment": "Skipped fields from ScalarQuantization: AlwaysRam",
"properties": {
"enabled": {
"description": "Whether to enable int8 scalar quantization",
"type": "boolean",
"!go": {
"pointer": true
},
"default": true
},
"quantile": {
"description": "Any values that lie outside the quantile range will be truncated",
"type": "number",
"minimum": 0,
"maximum": 1,
"!go": {
"pointer": true
},
"default": 0.98
}
}
},
"optimizers": {
"type": "object",
"$comment": "Skipped fields from OptimizersConfigDiff: DeletedThreshold, VacuumMinVectorNumber, DefaultSegmentNumber, MaxSegmentSize, FlushIntervalSec, MaxOptimizationThreads",
"properties": {
"indexingThreshold": {
"description": "Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing. Set to 0 to disable indexing",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 0
},
"memmapThreshold": {
"description": "Maximum size (in kilobytes) of vectors to store in-memory per segment.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 100
}
}
}
}
}
},
"examples": [

51
sg.config.yaml
View File

@ -273,18 +273,6 @@ commands:
- cmd/embeddings
- internal/embeddings
qdrant:
cmd: |
docker run -p 6333:6333 -p 6334:6334 \
-v $HOME/.sourcegraph-dev/data/qdrant_data:/data \
-e QDRANT__SERVICE__GRPC_PORT="6334" \
-e QDRANT__LOG_LEVEL=INFO \
-e QDRANT__STORAGE__STORAGE_PATH=/data \
-e QDRANT__STORAGE__SNAPSHOTS_PATH=/data \
-e QDRANT_INIT_FILE_PATH=/data/.qdrant-initialized \
--entrypoint /usr/local/bin/qdrant \
sourcegraph/qdrant:insiders
worker:
cmd: |
export SOURCEGRAPH_LICENSE_GENERATION_KEY=$(cat ../dev-private/enterprise/dev/test-license-generation-key.pem)
@ -1250,22 +1238,6 @@ dockerCommands:
WORKERS: 1
ROCKET_ADDRESS: 0.0.0.0
qdrant:
docker:
image: sourcegraph/qdrant:insiders
ports:
- 6333
- 6334
volumes:
- from: $HOME/.sourcegraph-dev/data/qdrant_data
to: /data
env:
QDRANT__SERVICE__GRPC_PORT: 6334
QDRANT__LOG_LEVEL: INFO
QDRANT__STORAGE__STORAGE_PATH: /data
QDRANT__STORAGE__SNAPSHOTS_PATH: /data
QDRANT_INIT_FILE_PATH: /data/.qdrant-initialized
#
# CommandSets ################################################################
#
@ -1748,29 +1720,6 @@ commandsets:
bazelCommands:
- cody-gateway
qdrant:
commands:
- qdrant
- frontend
- worker
- repo-updater
- web
- gitserver-0
- gitserver-1
- searcher
- caddy
- symbols
- docsite
- syntax-highlighter
- zoekt-index-0
- zoekt-index-1
- zoekt-web-0
- zoekt-web-1
- blobstore
- embeddings
env:
QDRANT_ENDPOINT: 'localhost:6334'
enterprise-bazel-sveltekit:
<<: *enterprise_bazel_set
env:

1
third-party-licenses/ThirdPartyLicenses.csv
View File

@ -1514,7 +1514,6 @@ Go,github.com/prometheus/common,v0.32.1,"Apache 2.0,New BSD",github.com/promethe
Go,github.com/prometheus/common/sigv4,v0.1.0,Apache 2.0,github.com/prometheus/common,Approved
Go,github.com/prometheus/procfs,v0.11.1,Apache 2.0,github.com/prometheus/procfs,Approved
Go,github.com/prometheus/prometheus,v0.40.5,Apache 2.0,github.com/prometheus/prometheus,Approved
Go,github.com/qdrant/go-client,v1.4.1,Apache 2.0,github.com/qdrant/go-client,Approved
Go,github.com/qustavo/sqlhooks/v2,v2.1.0,MIT,github.com/qustavo/sqlhooks,Approved
Go,github.com/rickb777/date,v1.14.3,New BSD,github.com/rickb777/date,Approved
Go,github.com/rickb777/plural,v1.2.2,New BSD,github.com/rickb777/plural,Approved

1 package_manager name version licenses homepage approved
1514 Go github.com/prometheus/common/sigv4 v0.1.0 Apache 2.0 github.com/prometheus/common Approved
1515 Go github.com/prometheus/procfs v0.11.1 Apache 2.0 github.com/prometheus/procfs Approved
1516 Go github.com/prometheus/prometheus v0.40.5 Apache 2.0 github.com/prometheus/prometheus Approved
Go github.com/qdrant/go-client v1.4.1 Apache 2.0 github.com/qdrant/go-client Approved
1517 Go github.com/qustavo/sqlhooks/v2 v2.1.0 MIT github.com/qustavo/sqlhooks Approved
1518 Go github.com/rickb777/date v1.14.3 New BSD github.com/rickb777/date Approved
1519 Go github.com/rickb777/plural v1.2.2 New BSD github.com/rickb777/plural Approved

828
wolfi-images/qdrant.lock.json
View File

@ -1,828 +0,0 @@
{
"configHash": "f5dfaea80b9aa5f34f3aa22bfa12474efd99bdde5587dbff22187b5731e4b5ae",
"contents": {
"keyring": [
{
"name": "packages.wolfi.dev/os/wolfi-signing.rsa.pub",
"url": "https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"
},
{
"name": "packages.sgdev.org/sourcegraph-melange-prod.rsa.pub",
"url": "https://packages.sgdev.org/sourcegraph-melange-prod.rsa.pub"
}
],
"packages": [
{
"architecture": "x86_64",
"checksum": "Q1YQmPfQ1Ym4tfjrCMChbESrrRg/o=",
"control": {
"checksum": "sha1-YQmPfQ1Ym4tfjrCMChbESrrRg/o=",
"range": "bytes=696-1032"
},
"data": {
"checksum": "sha256-5hhCQURRrKVfPk8TOZVxfjceIUkVE0fh3/vEJBk88Ps=",
"range": "bytes=1033-256258"
},
"name": "ca-certificates-bundle",
"signature": {
"checksum": "sha1-E1NIpx8yCH6x5GcSqB4MzKQxuq4=",
"range": "bytes=0-695"
},
"url": "https://packages.wolfi.dev/os/x86_64/ca-certificates-bundle-20240315-r0.apk",
"version": "20240315-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1OHhyuiUviNHTg939DA0lyeRee18=",
"control": {
"checksum": "sha1-OHhyuiUviNHTg939DA0lyeRee18=",
"range": "bytes=702-1052"
},
"data": {
"checksum": "sha256-om3EZzEM+3dD9a77sOB2uOuAKBlf7XoUW/ORnDHQvZY=",
"range": "bytes=1053-125427"
},
"name": "wolfi-baselayout",
"signature": {
"checksum": "sha1-1CcRiULOFhX8ldA/Ae2qCMUGNmQ=",
"range": "bytes=0-701"
},
"url": "https://packages.wolfi.dev/os/x86_64/wolfi-baselayout-20230201-r7.apk",
"version": "20230201-r7"
},
{
"architecture": "x86_64",
"checksum": "Q1uuAznXPkDNBMP/eILVO7UDGj1pU=",
"control": {
"checksum": "sha1-uuAznXPkDNBMP/eILVO7UDGj1pU=",
"range": "bytes=702-1112"
},
"data": {
"checksum": "sha256-Ath31YTTsiakoEktGl5txvNQpnr/grvFQHlmXa5E7fI=",
"range": "bytes=1113-267815"
},
"name": "ld-linux",
"signature": {
"checksum": "sha1-oh4vr00LXL4ArbAOR9LS1VzzfYc=",
"range": "bytes=0-701"
},
"url": "https://packages.wolfi.dev/os/x86_64/ld-linux-2.39-r2.apk",
"version": "2.39-r2"
},
{
"architecture": "x86_64",
"checksum": "Q1TXuyiDJBsTNDwvPq7HFxaeBQ33w=",
"control": {
"checksum": "sha1-TXuyiDJBsTNDwvPq7HFxaeBQ33w=",
"range": "bytes=703-1059"
},
"data": {
"checksum": "sha256-3s2Fygt+ZuHj/StxP7YffswmhHE7EJ4TxZ7EWlRQ4NA=",
"range": "bytes=1060-408275"
},
"name": "glibc-locale-posix",
"signature": {
"checksum": "sha1-FbeSNtuEgFmzNamDx5Dj1LGVgsQ=",
"range": "bytes=0-702"
},
"url": "https://packages.wolfi.dev/os/x86_64/glibc-locale-posix-2.39-r2.apk",
"version": "2.39-r2"
},
{
"architecture": "x86_64",
"checksum": "Q1SD8az8RMNr5tnb8/mPZdR+A6V5k=",
"control": {
"checksum": "sha1-SD8az8RMNr5tnb8/mPZdR+A6V5k=",
"range": "bytes=701-1330"
},
"data": {
"checksum": "sha256-QvhWs/fGBaX5calu0uES9fcTMx6+R1xKZHdxkxxSxsg=",
"range": "bytes=1331-5861481"
},
"name": "glibc",
"signature": {
"checksum": "sha1-v4LRB2eP5VLe623v8sJ3f4wDNFM=",
"range": "bytes=0-700"
},
"url": "https://packages.wolfi.dev/os/x86_64/glibc-2.39-r2.apk",
"version": "2.39-r2"
},
{
"architecture": "x86_64",
"checksum": "Q1mVgCtcYDHEkUa+8x41i7w9cQ4Qg=",
"control": {
"checksum": "sha1-mVgCtcYDHEkUa+8x41i7w9cQ4Qg=",
"range": "bytes=704-1079"
},
"data": {
"checksum": "sha256-xE4spLqr7qIgImPfMC1kMY1a7Xu7xu1/eLkBMgOzSSc=",
"range": "bytes=1080-77936"
},
"name": "protobuf-c",
"signature": {
"checksum": "sha1-NGL0ELlBK8mhhAzuOkm17d2LbRo=",
"range": "bytes=0-703"
},
"url": "https://packages.wolfi.dev/os/x86_64/protobuf-c-1.5.0-r3.apk",
"version": "1.5.0-r3"
},
{
"architecture": "x86_64",
"checksum": "Q135v8eEv8ZI/s/HXKkE96INoEoJk=",
"control": {
"checksum": "sha1-35v8eEv8ZI/s/HXKkE96INoEoJk=",
"range": "bytes=704-1040"
},
"data": {
"checksum": "sha256-3y4Tb3jy8M/ZXhnY6jxwj2YQFhdjLr/kjXhqJX7I+is=",
"range": "bytes=1041-27155"
},
"name": "krb5-conf",
"signature": {
"checksum": "sha1-WWjewHF5gekYrUPqdUHQEPIc97M=",
"range": "bytes=0-703"
},
"url": "https://packages.wolfi.dev/os/x86_64/krb5-conf-1.0-r1.apk",
"version": "1.0-r1"
},
{
"architecture": "x86_64",
"checksum": "Q1TAnNLCVTaCylmbN84TlliyM47qM=",
"control": {
"checksum": "sha1-TAnNLCVTaCylmbN84TlliyM47qM=",
"range": "bytes=704-1069"
},
"data": {
"checksum": "sha256-+6lzaltti6IVf7RNynwcL9LmP9cJKMOjONPFUhHtnsA=",
"range": "bytes=1070-57492"
},
"name": "keyutils-libs",
"signature": {
"checksum": "sha1-MY+r6+HKyYcrgJtVqyG50KdP+QY=",
"range": "bytes=0-703"
},
"url": "https://packages.wolfi.dev/os/x86_64/keyutils-libs-1.6.3-r1.apk",
"version": "1.6.3-r1"
},
{
"architecture": "x86_64",
"checksum": "Q1baVeVcWh84uv5G9/ZuxJCTg06uQ=",
"control": {
"checksum": "sha1-baVeVcWh84uv5G9/ZuxJCTg06uQ=",
"range": "bytes=700-1058"
},
"data": {
"checksum": "sha256-9BtoieN8gK8o4nzCDUyWkp6RmEsXrOwdu/XeTdZtDSE=",
"range": "bytes=1059-61938"
},
"name": "libverto",
"signature": {
"checksum": "sha1-TVkKg7JApxa7nvaj9DNvOsTv3Io=",
"range": "bytes=0-699"
},
"url": "https://packages.wolfi.dev/os/x86_64/libverto-0.3.2-r1.apk",
"version": "0.3.2-r1"
},
{
"architecture": "x86_64",
"checksum": "Q1M+CfES+G7micWuVpGjyxcTOj9zQ=",
"control": {
"checksum": "sha1-M+CfES+G7micWuVpGjyxcTOj9zQ=",
"range": "bytes=702-1120"
},
"data": {
"checksum": "sha256-UqwHAn95sL7AsIRMN0DM1TtSJ2Q5KD1WN4+uU8+Knqg=",
"range": "bytes=1121-52564"
},
"name": "libcom_err",
"signature": {
"checksum": "sha1-qciIi1MZRLclhn8K+GnrOJlNNfE=",
"range": "bytes=0-701"
},
"url": "https://packages.wolfi.dev/os/x86_64/libcom_err-1.47.0-r1.apk",
"version": "1.47.0-r1"
},
{
"architecture": "x86_64",
"checksum": "Q1+rBEmKEIywe0o/a65ZPjWoUECRQ=",
"control": {
"checksum": "sha1-+rBEmKEIywe0o/a65ZPjWoUECRQ=",
"range": "bytes=703-1054"
},
"data": {
"checksum": "sha256-fcShKjtj6wR6kgjwToU1J8hU+AGRf1OGZt9jeXJdM0Q=",
"range": "bytes=1055-82406"
},
"name": "openssl-config",
"signature": {
"checksum": "sha1-RT/o4gi2oC/l+6IhikP9/PH5awg=",
"range": "bytes=0-702"
},
"url": "https://packages.wolfi.dev/os/x86_64/openssl-config-3.3.0-r5.apk",
"version": "3.3.0-r5"
},
{
"architecture": "x86_64",
"checksum": "Q11DkfNRzraQMj7ue8LFH6VN6d7Bo=",
"control": {
"checksum": "sha1-1DkfNRzraQMj7ue8LFH6VN6d7Bo=",
"range": "bytes=707-1092"
},
"data": {
"checksum": "sha256-c95GJNThldUhfxVSGbchUI0pS4j6h45Yx6JDPXHaAI0=",
"range": "bytes=1093-5915048"
},
"name": "libcrypto3",
"signature": {
"checksum": "sha1-EmjggcTaE4QTKI1FzSom0rXam1c=",
"range": "bytes=0-706"
},
"url": "https://packages.wolfi.dev/os/x86_64/libcrypto3-3.3.0-r5.apk",
"version": "3.3.0-r5"
},
{
"architecture": "x86_64",
"checksum": "Q15sT5bGF6kgE7DepzZIHCpPpuP70=",
"control": {
"checksum": "sha1-5sT5bGF6kgE7DepzZIHCpPpuP70=",
"range": "bytes=702-1085"
},
"data": {
"checksum": "sha256-fWOAfZYgXO6+yM4Dpo/y7dvV3uxSEhhfO4SUX79TQDw=",
"range": "bytes=1086-1196497"
},
"name": "libssl3",
"signature": {
"checksum": "sha1-4+7eITv9weTmoYBAAyd92WsXS24=",
"range": "bytes=0-701"
},
"url": "https://packages.wolfi.dev/os/x86_64/libssl3-3.3.0-r5.apk",
"version": "3.3.0-r5"
},
{
"architecture": "x86_64",
"checksum": "Q1myu18Yt+0lLtpxOyrQ3LKnV6SoI=",
"control": {
"checksum": "sha1-myu18Yt+0lLtpxOyrQ3LKnV6SoI=",
"range": "bytes=698-1216"
},
"data": {
"checksum": "sha256-elcsQDiEgNifO11g4MoJLGo5XJu8wQZaytjIKYxzVjw=",
"range": "bytes=1217-2564165"
},
"name": "krb5-libs",
"signature": {
"checksum": "sha1-NdbRDYVgwMQYQQrqt9GSGuBkjuA=",
"range": "bytes=0-697"
},
"url": "https://packages.wolfi.dev/os/x86_64/krb5-libs-1.21.2-r1.apk",
"version": "1.21.2-r1"
},
{
"architecture": "x86_64",
"checksum": "Q1BMcnbxIFejKxlfYLJaX4uo/+X0M=",
"control": {
"checksum": "sha1-BMcnbxIFejKxlfYLJaX4uo/+X0M=",
"range": "bytes=699-1059"
},
"data": {
"checksum": "sha256-Rp339LV5T4+zr98USK12eoyKGF84NviKlq3jh1AvbQQ=",
"range": "bytes=1060-96440"
},
"name": "fstrm",
"signature": {
"checksum": "sha1-tOY0x5olyX6V9uRtlgxtf4WEhR0=",
"range": "bytes=0-698"
},
"url": "https://packages.wolfi.dev/os/x86_64/fstrm-0.6.1-r1.apk",
"version": "0.6.1-r1"
},
{
"architecture": "x86_64",
"checksum": "Q126C/3voUst+sFakQOGVOKucs6v8=",
"control": {
"checksum": "sha1-26C/3voUst+sFakQOGVOKucs6v8=",
"range": "bytes=698-1076"
},
"data": {
"checksum": "sha256-S2VfC729Glys7iRmR7sX4RoS4LLyNi9KmSzDc8wkKrQ=",
"range": "bytes=1077-277291"
},
"name": "libuv",
"signature": {
"checksum": "sha1-23pSGkhyhlBtbVxNtDXYlWnt+vk=",
"range": "bytes=0-697"
},
"url": "https://packages.wolfi.dev/os/x86_64/libuv-1.48.0-r0.apk",
"version": "1.48.0-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1ybud27/W+hJ0asiUj3hfrXVjcMs=",
"control": {
"checksum": "sha1-ybud27/W+hJ0asiUj3hfrXVjcMs=",
"range": "bytes=698-1081"
},
"data": {
"checksum": "sha256-3uFtBCAT2Gu/AplcYbKYMCuMMC94JLJDNyoWnSMLqjc=",
"range": "bytes=1082-156680"
},
"name": "zlib",
"signature": {
"checksum": "sha1-JYfhgb71ZjFG0VIAKLwOQSlHmlg=",
"range": "bytes=0-697"
},
"url": "https://packages.wolfi.dev/os/x86_64/zlib-1.3.1-r0.apk",
"version": "1.3.1-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1sGD1MEAazAXpfNixX74EpwafGo0=",
"control": {
"checksum": "sha1-sGD1MEAazAXpfNixX74EpwafGo0=",
"range": "bytes=698-1059"
},
"data": {
"checksum": "sha256-s+9m0l/R7hVKQ5YLbU90D2E6DPC2QVTxdjhToMi5iYU=",
"range": "bytes=1060-251831"
},
"name": "libnghttp2-14",
"signature": {
"checksum": "sha1-wOIYtvU72n76UGvbargK2ccZx0E=",
"range": "bytes=0-697"
},
"url": "https://packages.wolfi.dev/os/x86_64/libnghttp2-14-1.61.0-r0.apk",
"version": "1.61.0-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1cCNlvL7pVXPEgMJQz7uLXVW0w5g=",
"control": {
"checksum": "sha1-cCNlvL7pVXPEgMJQz7uLXVW0w5g=",
"range": "bytes=697-1073"
},
"data": {
"checksum": "sha256-7m+RnrAOS8ZCnFW/j2P2NcQJxOzfwSjOLYhEZXIwBG8=",
"range": "bytes=1074-114096"
},
"name": "libev",
"signature": {
"checksum": "sha1-PUsxiEtEuEoDpgKP2L36G/X55s0=",
"range": "bytes=0-696"
},
"url": "https://packages.wolfi.dev/os/x86_64/libev-4.33-r4.apk",
"version": "4.33-r4"
},
{
"architecture": "x86_64",
"checksum": "Q1jyz//Wx+59L1JtSRpS5LPxe5iaM=",
"control": {
"checksum": "sha1-jyz//Wx+59L1JtSRpS5LPxe5iaM=",
"range": "bytes=708-1084"
},
"data": {
"checksum": "sha256-v6jAIoRu7n3hQJ8nwWCLtvRsszuMGCAEyZm4zUF1cVI=",
"range": "bytes=1085-185893"
},
"name": "libgcc",
"signature": {
"checksum": "sha1-3fVn7jRkfOtxSgpmdey4iJJqQPM=",
"range": "bytes=0-707"
},
"url": "https://packages.wolfi.dev/os/x86_64/libgcc-13.2.0-r5.apk",
"version": "13.2.0-r5"
},
{
"architecture": "x86_64",
"checksum": "Q1NsUsznaiP7XyU6U/5pssXQgGJgU=",
"control": {
"checksum": "sha1-NsUsznaiP7XyU6U/5pssXQgGJgU=",
"range": "bytes=701-1093"
},
"data": {
"checksum": "sha256-0XrQ++geRWYRUazF23zdsuGVLFkiIDM1FKmAbbz9ojQ=",
"range": "bytes=1094-3156830"
},
"name": "libstdc++",
"signature": {
"checksum": "sha1-Zu2LUNkKQt3BnFU9+4PX0ud8D6Q=",
"range": "bytes=0-700"
},
"url": "https://packages.wolfi.dev/os/x86_64/libstdc++-13.2.0-r5.apk",
"version": "13.2.0-r5"
},
{
"architecture": "x86_64",
"checksum": "Q1EHyaNLx1UadaqJXEC86gtIjZGMM=",
"control": {
"checksum": "sha1-EHyaNLx1UadaqJXEC86gtIjZGMM=",
"range": "bytes=696-1076"
},
"data": {
"checksum": "sha256-+ValBcpIsLeYUFo73SZrUM2vpLGefts7Qx95EPMATaY=",
"range": "bytes=1077-232308"
},
"name": "c-ares",
"signature": {
"checksum": "sha1-OtwgYMiySwl+l6divnnSgQu+QUg=",
"range": "bytes=0-695"
},
"url": "https://packages.wolfi.dev/os/x86_64/c-ares-1.28.1-r0.apk",
"version": "1.28.1-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1NZCvjmGBZ7soi6T4endRPtDztk4=",
"control": {
"checksum": "sha1-NZCvjmGBZ7soi6T4endRPtDztk4=",
"range": "bytes=698-1162"
},
"data": {
"checksum": "sha256-q79pSkGvnDzz1dHsQceLPqyVC+cFCyWWJB5L2EfkZRc=",
"range": "bytes=1163-2556930"
},
"name": "nghttp2",
"signature": {
"checksum": "sha1-UuOeekThV8UC737772jFTRj0Y50=",
"range": "bytes=0-697"
},
"url": "https://packages.wolfi.dev/os/x86_64/nghttp2-1.61.0-r0.apk",
"version": "1.61.0-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1O/sL8Kjkxzh39Ffh9gJl+0olkF4=",
"control": {
"checksum": "sha1-O/sL8Kjkxzh39Ffh9gJl+0olkF4=",
"range": "bytes=700-1061"
},
"data": {
"checksum": "sha256-6qS8u1yYrzO+4Ypa+g4zfz3m16/7O/U4wjiRrKl8/vw=",
"range": "bytes=1062-631650"
},
"name": "nghttp2-dev",
"signature": {
"checksum": "sha1-avAz/6snsG86C9jh8Gxa59gs29U=",
"range": "bytes=0-699"
},
"url": "https://packages.wolfi.dev/os/x86_64/nghttp2-dev-1.61.0-r0.apk",
"version": "1.61.0-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1sJOz3InYXKj1qNN3oPm3pb30VO0=",
"control": {
"checksum": "sha1-sJOz3InYXKj1qNN3oPm3pb30VO0=",
"range": "bytes=697-1149"
},
"data": {
"checksum": "sha256-fahwYvY6Lv9AhtEHBsSaoFAoKFWlvbbVC8jYGuRmTcg=",
"range": "bytes=1150-2380016"
},
"name": "xz",
"signature": {
"checksum": "sha1-aQxFIS7BG8u/jlY2JU8oQIEloYw=",
"range": "bytes=0-696"
},
"url": "https://packages.wolfi.dev/os/x86_64/xz-5.4.6-r0.apk",
"version": "5.4.6-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1aNAiDiOAPLkPMJFYq6mpzKq4V18=",
"control": {
"checksum": "sha1-aNAiDiOAPLkPMJFYq6mpzKq4V18=",
"range": "bytes=699-1084"
},
"data": {
"checksum": "sha256-Wc0u/JyHwxC7ubVFz4UlXEc5URwWiBZm4jNKqVv9LF0=",
"range": "bytes=1085-4698210"
},
"name": "libxml2",
"signature": {
"checksum": "sha1-v7pbNfh/TdC3LzRewdC3GeA9rec=",
"range": "bytes=0-698"
},
"url": "https://packages.wolfi.dev/os/x86_64/libxml2-2.12.6-r0.apk",
"version": "2.12.6-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1wJPtj5XZtnPiVWGM5dsT+nyfWXc=",
"control": {
"checksum": "sha1-wJPtj5XZtnPiVWGM5dsT+nyfWXc=",
"range": "bytes=705-1234"
},
"data": {
"checksum": "sha256-sw+CjlpsjO01q6lqzXgHnniiJKn7bqOSNO6vNqdGc7I=",
"range": "bytes=1235-3873775"
},
"name": "bind-libs",
"signature": {
"checksum": "sha1-9zDzmHAeu7mWpidXAatqrLZVzUs=",
"range": "bytes=0-704"
},
"url": "https://packages.wolfi.dev/os/x86_64/bind-libs-9.18.26-r0.apk",
"version": "9.18.26-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1AxEQdi8fVcYtGc3zHTCW3OZdY2Q=",
"control": {
"checksum": "sha1-AxEQdi8fVcYtGc3zHTCW3OZdY2Q=",
"range": "bytes=694-1208"
},
"data": {
"checksum": "sha256-/dt3qJw5D9iSJ/mKkRQkhWD8R52selhpcwXyw7KLMhg=",
"range": "bytes=1209-879444"
},
"name": "bind-tools",
"signature": {
"checksum": "sha1-VTb0YdUzPMk8Po0p4nj0y5Jey6k=",
"range": "bytes=0-693"
},
"url": "https://packages.wolfi.dev/os/x86_64/bind-tools-9.18.26-r0.apk",
"version": "9.18.26-r0"
},
{
"architecture": "x86_64",
"checksum": "Q11cs1/Vkyp8KEwqtqZvPrB+Mfb8A=",
"control": {
"checksum": "sha1-1cs1/Vkyp8KEwqtqZvPrB+Mfb8A=",
"range": "bytes=698-1093"
},
"data": {
"checksum": "sha256-t284K9/cZQaQMy4y4nYXMIjKUTbyaBa/QnUj0cYmTNk=",
"range": "bytes=1094-234977"
},
"name": "libxcrypt",
"signature": {
"checksum": "sha1-hhR4Puw7nMj2H9OzUMTKhK1/7N0=",
"range": "bytes=0-697"
},
"url": "https://packages.wolfi.dev/os/x86_64/libxcrypt-4.4.36-r4.apk",
"version": "4.4.36-r4"
},
{
"architecture": "x86_64",
"checksum": "Q1drCieAMJpJBP1KWvJk6Vhq7Zj20=",
"control": {
"checksum": "sha1-drCieAMJpJBP1KWvJk6Vhq7Zj20=",
"range": "bytes=701-1108"
},
"data": {
"checksum": "sha256-LcGaT+nLHN7YtUab5sY0EoXErbOj/S58FXtf1JVxWbM=",
"range": "bytes=1109-21605"
},
"name": "libcrypt1",
"signature": {
"checksum": "sha1-wwLYQc1joetP6IOipSVSCQsZHsM=",
"range": "bytes=0-700"
},
"url": "https://packages.wolfi.dev/os/x86_64/libcrypt1-2.39-r2.apk",
"version": "2.39-r2"
},
{
"architecture": "x86_64",
"checksum": "Q17FDk2/BvxV3n5UBi4rz7m8aR1Wc=",
"control": {
"checksum": "sha1-7FDk2/BvxV3n5UBi4rz7m8aR1Wc=",
"range": "bytes=701-1208"
},
"data": {
"checksum": "sha256-67DYE+o9zQIS2KUyXkBN8SAEkWVh2+isnGTn78VdLMg=",
"range": "bytes=1209-636015"
},
"name": "busybox",
"signature": {
"checksum": "sha1-70uMRez2BMN2clrT3wFBWsR5Gew=",
"range": "bytes=0-700"
},
"url": "https://packages.wolfi.dev/os/x86_64/busybox-1.36.1-r7.apk",
"version": "1.36.1-r7"
},
{
"architecture": "x86_64",
"checksum": "Q1f9ldLw5Jdbm9CkZfsDWXP2YaQWE=",
"control": {
"checksum": "sha1-f9ldLw5Jdbm9CkZfsDWXP2YaQWE=",
"range": "bytes=706-1103"
},
"data": {
"checksum": "sha256-ttIYvsu5Vp2YYKv/C7vK1hFUI8kNsOJxD65/SsOtWvQ=",
"range": "bytes=1104-2862070"
},
"name": "libunistring",
"signature": {
"checksum": "sha1-IwR2lnVv+Ixi8qtznw+ruuV9OVw=",
"range": "bytes=0-705"
},
"url": "https://packages.wolfi.dev/os/x86_64/libunistring-1.2-r0.apk",
"version": "1.2-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1tBy70+JqCVQbecHMDxN0U9PKn8k=",
"control": {
"checksum": "sha1-tBy70+JqCVQbecHMDxN0U9PKn8k=",
"range": "bytes=697-1102"
},
"data": {
"checksum": "sha256-xFjkADRrilqBaMgedKfRkaUGOqAlLDunZnmM/nggwDo=",
"range": "bytes=1103-411419"
},
"name": "libidn2",
"signature": {
"checksum": "sha1-AkpnAB73nCuJM4BJIXJsWn2/urk=",
"range": "bytes=0-696"
},
"url": "https://packages.wolfi.dev/os/x86_64/libidn2-2.3.7-r0.apk",
"version": "2.3.7-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1QobDOHNHcnrYAlkCuVI1Te8I5V0=",
"control": {
"checksum": "sha1-QobDOHNHcnrYAlkCuVI1Te8I5V0=",
"range": "bytes=702-1082"
},
"data": {
"checksum": "sha256-cEuUD1tNXYeUHPC7dK9BSHOx8vt7IIRBGd181Sd0RXA=",
"range": "bytes=1083-114314"
},
"name": "libpsl",
"signature": {
"checksum": "sha1-H2Bp5J4UMCXPQlfvEiFxLXG5rEY=",
"range": "bytes=0-701"
},
"url": "https://packages.wolfi.dev/os/x86_64/libpsl-0.21.5-r0.apk",
"version": "0.21.5-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1LcDEPPjJrWwhueUvesRr4kLyGPI=",
"control": {
"checksum": "sha1-LcDEPPjJrWwhueUvesRr4kLyGPI=",
"range": "bytes=707-1052"
},
"data": {
"checksum": "sha256-xtIiQnSjJGdZnAzaBoJD6q3TbCpnAwTfD9SvbafkaP0=",
"range": "bytes=1053-174068"
},
"name": "libbrotlicommon1",
"signature": {
"checksum": "sha1-kcyKhs8jgdpGEEUhbyXHPJQpvyM=",
"range": "bytes=0-706"
},
"url": "https://packages.wolfi.dev/os/x86_64/libbrotlicommon1-1.1.0-r1.apk",
"version": "1.1.0-r1"
},
{
"architecture": "x86_64",
"checksum": "Q10jZlLouHAzMUvZYphGGNjOoZ1ug=",
"control": {
"checksum": "sha1-0jZlLouHAzMUvZYphGGNjOoZ1ug=",
"range": "bytes=701-1051"
},
"data": {
"checksum": "sha256-nx7lapEgf5fd4ZrtBxqXtAWE0tEVndBBEwXOUbhY2OA=",
"range": "bytes=1052-81979"
},
"name": "libbrotlidec1",
"signature": {
"checksum": "sha1-iUdAt3okR6P8rJ4sNv4yjC0I8Ys=",
"range": "bytes=0-700"
},
"url": "https://packages.wolfi.dev/os/x86_64/libbrotlidec1-1.1.0-r1.apk",
"version": "1.1.0-r1"
},
{
"architecture": "x86_64",
"checksum": "Q1hgEBBo1tLreSINI2qiCUpUrIE9k=",
"control": {
"checksum": "sha1-hgEBBo1tLreSINI2qiCUpUrIE9k=",
"range": "bytes=700-1139"
},
"data": {
"checksum": "sha256-OFDWkK8l015fs0YvTCXVKLQmOxq+7W5RgJHIqKYZsTE=",
"range": "bytes=1140-808582"
},
"name": "libcurl-openssl4",
"signature": {
"checksum": "sha1-sygIdNZXB/W8fyOrdelY5RXh3aA=",
"range": "bytes=0-699"
},
"url": "https://packages.wolfi.dev/os/x86_64/libcurl-openssl4-8.6.0-r0.apk",
"version": "8.6.0-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1eQzLI9XN4FM69fm9B4LWd9frcwM=",
"control": {
"checksum": "sha1-eQzLI9XN4FM69fm9B4LWd9frcwM=",
"range": "bytes=703-1098"
},
"data": {
"checksum": "sha256-zkew7+IbyyyENezhlStwO062zaWd+yP/lpbI+CCJVZQ=",
"range": "bytes=1099-281166"
},
"name": "curl",
"signature": {
"checksum": "sha1-igXspQSxo3Eu3o8DeNrYt3YxSbY=",
"range": "bytes=0-702"
},
"url": "https://packages.wolfi.dev/os/x86_64/curl-8.6.0-r0.apk",
"version": "8.6.0-r0"
},
{
"architecture": "x86_64",
"checksum": "Q19yTFZBeYlCCMWWWtfHBXivqZKfc=",
"control": {
"checksum": "sha1-9yTFZBeYlCCMWWWtfHBXivqZKfc=",
"range": "bytes=657-991"
},
"data": {
"checksum": "sha256-2FUyZ+/OJ9o/PTteTNS/jIlOBJuw7eJrhAbxiFIbU9Q=",
"range": "bytes=992-42513766"
},
"name": "qdrant",
"signature": {
"checksum": "sha1-ZfLA+OrzXH+JAysAXgT9IoFbiRA=",
"range": "bytes=0-656"
},
"url": "https://packages.sgdev.org/main/x86_64/qdrant-1.5.1-r4.apk",
"version": "1.5.1-r4"
},
{
"architecture": "x86_64",
"checksum": "Q1m7SsrH+XnrPIapzhnK0vkoxMen4=",
"control": {
"checksum": "sha1-m7SsrH+XnrPIapzhnK0vkoxMen4=",
"range": "bytes=698-1077"
},
"data": {
"checksum": "sha256-0WaULqZyE0zNNC6D7YeaXVN4shAVEouyqAEiRlB09C0=",
"range": "bytes=1078-54479"
},
"name": "tini",
"signature": {
"checksum": "sha1-eomfhWCyCB1OA/2DHboLUzXunvA=",
"range": "bytes=0-697"
},
"url": "https://packages.wolfi.dev/os/x86_64/tini-0.19.0-r3.apk",
"version": "0.19.0-r3"
},
{
"architecture": "x86_64",
"checksum": "Q1lvoD8CoCRqulbibjC3gSGuEe8K0=",
"control": {
"checksum": "sha1-lvoD8CoCRqulbibjC3gSGuEe8K0=",
"range": "bytes=702-1035"
},
"data": {
"checksum": "sha256-YB3jK9thvtrl7FCbwBPhBgHnz6ncykVxex/dHC4wYc8=",
"range": "bytes=1036-3022935"
},
"name": "tzdata",
"signature": {
"checksum": "sha1-AJ7WqmxNNMiSUQ8WuwjXg5Y23Gw=",
"range": "bytes=0-701"
},
"url": "https://packages.wolfi.dev/os/x86_64/tzdata-2024a-r0.apk",
"version": "2024a-r0"
},
{
"architecture": "x86_64",
"checksum": "Q1JVK/s+yGDiDotAujuHcQwzrchvI=",
"control": {
"checksum": "sha1-JVK/s+yGDiDotAujuHcQwzrchvI=",
"range": "bytes=699-1099"
},
"data": {
"checksum": "sha256-6MWjAN767fVREf1xZ18eV4QLzKBUdZusnhj7ljPZuhU=",
"range": "bytes=1100-786256"
},
"name": "wget",
"signature": {
"checksum": "sha1-cJ/swsh0XVO9ISogqXDo8oBBG7M=",
"range": "bytes=0-698"
},
"url": "https://packages.wolfi.dev/os/x86_64/wget-1.24.5-r0.apk",
"version": "1.24.5-r0"
}
],
"repositories": [
{
"architecture": "x86_64",
"name": "packages.wolfi.dev/os/x86_64",
"url": "https://packages.wolfi.dev/os/x86_64/APKINDEX.tar.gz"
},
{
"architecture": "x86_64",
"name": "@sourcegraph https://packages.sgdev.org/main/x86_64",
"url": "@sourcegraph https://packages.sgdev.org/main/x86_64/APKINDEX.tar.gz"
}
]
},
"version": "v1"
}

20
wolfi-images/qdrant.yaml
View File

@ -1,20 +0,0 @@
include: ./sourcegraph-template.yaml
contents:
packages:
# Included by existing SG base image
- tini
- qdrant@sourcegraph
paths:
- path: /data
type: directory
uid: 100
gid: 101
permissions: 0o755
entrypoint:
command: /sbin/tini -- /usr/local/bin/qdrant --config-path /etc/qdrant/config.yaml
# MANUAL REBUILD:

41
wolfi-packages/qdrant.yaml
View File

@ -1,41 +0,0 @@
package:
name: qdrant
version: 1.5.1
epoch: 4
description: "Qdrant vector database"
target-architecture:
- x86_64
copyright:
- paths:
- "*"
license: 'Apache 2.0'
dependencies:
runtime:
environment:
contents:
repositories:
- https://packages.wolfi.dev/os
keyring:
- https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
packages:
- wolfi-base
- busybox
- ca-certificates-bundle
pipeline:
- uses: fetch
with:
uri: https://github.com/qdrant/qdrant/releases/download/v${{package.version}}/qdrant-x86_64-unknown-linux-gnu.tar.gz
expected-sha256: 0d8b385590c1f1145f6114cde39eb8105305c6123b9a0505606cd489f322dbaa
strip-components: 0
- runs: |
chmod +x qdrant
mkdir -p ${{targets.destdir}}/usr/local/bin
mv qdrant ${{targets.destdir}}/usr/local/bin/qdrant
update:
enabled: true
github:
identifier: qdrant/qdrant
strip-prefix: "v"