From 24e8505019f9ee825e7470cf4df7d054336ddfa1 Mon Sep 17 00:00:00 2001 From: Erik Seliger Date: Mon, 22 Apr 2024 18:00:57 +0200 Subject: [PATCH] chore: Completely sunset qdrant (#62018) This removes qdrant from this codebase entirely. All the docker images, dependencies, (dead) usage in code. My understanding is that we don't use this feature and never properly rolled it out. Test plan: CI passes and code review from owners. --- cmd/frontend/graphqlbackend/embeddings.go | 1 - .../graphqlbackend/embeddings.graphql | 5 - cmd/frontend/internal/cli/config.go | 3 - cmd/frontend/internal/codycontext/BUILD.bazel | 1 - cmd/frontend/internal/codycontext/context.go | 21 +- cmd/frontend/internal/context/BUILD.bazel | 1 - cmd/frontend/internal/context/init.go | 4 - .../context/resolvers/context_test.go | 1 - .../embeddings/resolvers/resolvers.go | 13 - .../internal/embeddings/repo/BUILD.bazel | 1 - .../internal/embeddings/repo/handler.go | 83 -- cmd/worker/internal/embeddings/repo/worker.go | 8 - deps.bzl | 7 - dev/ci/images/images.go | 1 - dev/oci_deps.bzl | 6 - .../background-information/sg/reference.md | 2 - docker-images/qdrant/BUILD.bazel | 42 - docker-images/qdrant/qdrant_image_test.yaml | 20 - go.mod | 1 - go.sum | 2 - internal/api/internalapi/v1/internalapi.pb.go | 47 +- internal/api/internalapi/v1/internalapi.proto | 4 +- internal/conf/computed.go | 54 -- internal/conf/computed_test.go | 20 - internal/conf/conftypes/conftypes.go | 4 - internal/conf/conftypes/consts.go | 26 - .../embeddings/background/repo/mocks_temp.go | 122 --- internal/embeddings/background/repo/store.go | 22 - .../embeddings/background/repo/store_test.go | 42 - internal/embeddings/db/BUILD.bazel | 46 - internal/embeddings/db/chunk_point.go | 151 ---- internal/embeddings/db/chunk_point_test.go | 26 - internal/embeddings/db/conf.go | 166 ---- internal/embeddings/db/conf_test.go | 52 -- internal/embeddings/db/db.go | 29 - internal/embeddings/db/migrate.go | 79 -- internal/embeddings/db/noop.go | 58 -- internal/embeddings/db/qdrant.go | 284 ------ internal/embeddings/embed/BUILD.bazel | 2 - internal/embeddings/embed/embed.go | 44 +- internal/embeddings/embed/embed_test.go | 41 +- schema/schema.go | 40 - schema/site.schema.json | 110 --- sg.config.yaml | 51 -- third-party-licenses/ThirdPartyLicenses.csv | 1 - wolfi-images/qdrant.lock.json | 828 ------------------ wolfi-images/qdrant.yaml | 20 - wolfi-packages/qdrant.yaml | 41 - 48 files changed, 53 insertions(+), 2580 deletions(-) delete mode 100644 docker-images/qdrant/BUILD.bazel delete mode 100644 docker-images/qdrant/qdrant_image_test.yaml delete mode 100644 internal/embeddings/db/BUILD.bazel delete mode 100644 internal/embeddings/db/chunk_point.go delete mode 100644 internal/embeddings/db/chunk_point_test.go delete mode 100644 internal/embeddings/db/conf.go delete mode 100644 internal/embeddings/db/conf_test.go delete mode 100644 internal/embeddings/db/db.go delete mode 100644 internal/embeddings/db/migrate.go delete mode 100644 internal/embeddings/db/noop.go delete mode 100644 internal/embeddings/db/qdrant.go delete mode 100755 wolfi-images/qdrant.lock.json delete mode 100644 wolfi-images/qdrant.yaml delete mode 100644 wolfi-packages/qdrant.yaml diff --git a/cmd/frontend/graphqlbackend/embeddings.go b/cmd/frontend/graphqlbackend/embeddings.go index 7da0fa2aaf9..2e21f12152a 100644 --- a/cmd/frontend/graphqlbackend/embeddings.go +++ b/cmd/frontend/graphqlbackend/embeddings.go @@ -14,7 +14,6 @@ type EmbeddingsResolver interface { EmbeddingsMultiSearch(ctx context.Context, args EmbeddingsMultiSearchInputArgs) (EmbeddingsSearchResultsResolver, error) IsContextRequiredForChatQuery(ctx context.Context, args IsContextRequiredForChatQueryInputArgs) (bool, error) RepoEmbeddingJobs(ctx context.Context, args ListRepoEmbeddingJobsArgs) (*graphqlutil.ConnectionResolver[RepoEmbeddingJobResolver], error) - MigrateToQdrant(ctx context.Context) (*EmptyResponse, error) ScheduleRepositoriesForEmbedding(ctx context.Context, args ScheduleRepositoriesForEmbeddingArgs) (*EmptyResponse, error) CancelRepoEmbeddingJob(ctx context.Context, args CancelRepoEmbeddingJobArgs) (*EmptyResponse, error) diff --git a/cmd/frontend/graphqlbackend/embeddings.graphql b/cmd/frontend/graphqlbackend/embeddings.graphql index 9dc4ed40050..5f1d553d5b7 100644 --- a/cmd/frontend/graphqlbackend/embeddings.graphql +++ b/cmd/frontend/graphqlbackend/embeddings.graphql @@ -99,11 +99,6 @@ extend type Mutation { Experimental: Cancels the embedding job with the given ID. The job must exist and be in either 'processing' or 'queued' state. """ cancelRepoEmbeddingJob(job: ID!): EmptyResponse! - """ - TEMPORARY: creates a new embedding job for all completed embeddings - jobs so that they will be moved from blobstore to qdrant - """ - migrateToQdrant: EmptyResponse! } """ diff --git a/cmd/frontend/internal/cli/config.go b/cmd/frontend/internal/cli/config.go index 431859c202c..f9b846bebf1 100644 --- a/cmd/frontend/internal/cli/config.go +++ b/cmd/frontend/internal/cli/config.go @@ -598,7 +598,6 @@ func serviceConnections(logger log.Logger) conftypes.ServiceConnections { Searchers: searcherAddrs, Symbols: symbolsAddrs, Embeddings: embeddingsAddrs, - Qdrant: qdrantAddr, Zoekts: zoektAddrs, ZoektListTTL: indexedListTTL, } @@ -617,8 +616,6 @@ var ( embeddingsURLsOnce sync.Once embeddingsURLs *endpoint.Map - qdrantAddr = os.Getenv("QDRANT_ENDPOINT") - indexedListTTL = func() time.Duration { ttl, _ := time.ParseDuration(env.Get("SRC_INDEXED_SEARCH_LIST_CACHE_TTL", "", "Indexed search list cache TTL")) if ttl == 0 { diff --git a/cmd/frontend/internal/codycontext/BUILD.bazel b/cmd/frontend/internal/codycontext/BUILD.bazel index 2f61c4fc172..3f0ff6dc75b 100644 --- a/cmd/frontend/internal/codycontext/BUILD.bazel +++ b/cmd/frontend/internal/codycontext/BUILD.bazel @@ -18,7 +18,6 @@ go_library( "//internal/database", "//internal/dotcom", "//internal/embeddings", - "//internal/embeddings/db", "//internal/embeddings/embed", "//internal/gitserver", "//internal/metrics", diff --git a/cmd/frontend/internal/codycontext/context.go b/cmd/frontend/internal/codycontext/context.go index 51680a76dd0..6526d962251 100644 --- a/cmd/frontend/internal/codycontext/context.go +++ b/cmd/frontend/internal/codycontext/context.go @@ -16,7 +16,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/embeddings" - vdb "github.com/sourcegraph/sourcegraph/internal/embeddings/db" "github.com/sourcegraph/sourcegraph/internal/embeddings/embed" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/metrics" @@ -40,7 +39,7 @@ type FileChunkContext struct { EndLine int } -func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embeddingsClient embeddings.Client, searchClient client.SearchClient, gitserverClient gitserver.Client, getQdrantSearcher func() (vdb.VectorSearcher, error)) *CodyContextClient { +func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embeddingsClient embeddings.Client, searchClient client.SearchClient, gitserverClient gitserver.Client) *CodyContextClient { redMetrics := metrics.NewREDMetrics( obsCtx.Registerer, "codycontext_client", @@ -59,11 +58,10 @@ func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embedding } return &CodyContextClient{ - db: db, - embeddingsClient: embeddingsClient, - searchClient: searchClient, - getQdrantSearcher: getQdrantSearcher, - contentFilter: newRepoContentFilter(obsCtx.Logger, gitserverClient), + db: db, + embeddingsClient: embeddingsClient, + searchClient: searchClient, + contentFilter: newRepoContentFilter(obsCtx.Logger, gitserverClient), obsCtx: obsCtx, getCodyContextOp: op("getCodyContext"), @@ -73,11 +71,10 @@ func NewCodyContextClient(obsCtx *observation.Context, db database.DB, embedding } type CodyContextClient struct { - db database.DB - embeddingsClient embeddings.Client - searchClient client.SearchClient - contentFilter RepoContentFilter - getQdrantSearcher func() (vdb.VectorSearcher, error) + db database.DB + embeddingsClient embeddings.Client + searchClient client.SearchClient + contentFilter RepoContentFilter obsCtx *observation.Context getCodyContextOp *observation.Operation diff --git a/cmd/frontend/internal/context/BUILD.bazel b/cmd/frontend/internal/context/BUILD.bazel index bf77cc33f89..90093d90562 100644 --- a/cmd/frontend/internal/context/BUILD.bazel +++ b/cmd/frontend/internal/context/BUILD.bazel @@ -13,7 +13,6 @@ go_library( "//internal/conf/conftypes", "//internal/database", "//internal/embeddings", - "//internal/embeddings/db", "//internal/gitserver", "//internal/observation", "//internal/search/client", diff --git a/cmd/frontend/internal/context/init.go b/cmd/frontend/internal/context/init.go index 9aa25b23491..ae7cd61786f 100644 --- a/cmd/frontend/internal/context/init.go +++ b/cmd/frontend/internal/context/init.go @@ -10,7 +10,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/embeddings" - vdb "github.com/sourcegraph/sourcegraph/internal/embeddings/db" "github.com/sourcegraph/sourcegraph/internal/gitserver" "github.com/sourcegraph/sourcegraph/internal/observation" "github.com/sourcegraph/sourcegraph/internal/search/client" @@ -29,8 +28,6 @@ func Init( embeddingsClient := embeddings.NewDefaultClient() searchClient := client.New(observationCtx.Logger, db, gitserver.NewClient("graphql.context.search")) - getQdrantDB := vdb.NewDBFromConfFunc(observationCtx.Logger, vdb.NewDisabledDB()) - getQdrantSearcher := func() (vdb.VectorSearcher, error) { return getQdrantDB() } contextClient := codycontext.NewCodyContextClient( observationCtx, @@ -38,7 +35,6 @@ func Init( embeddingsClient, searchClient, services.GitserverClient.Scoped("codycontext.client"), - getQdrantSearcher, ) enterpriseServices.CodyContextResolver = resolvers.NewResolver( db, diff --git a/cmd/frontend/internal/context/resolvers/context_test.go b/cmd/frontend/internal/context/resolvers/context_test.go index 45984bad313..045b209d24d 100644 --- a/cmd/frontend/internal/context/resolvers/context_test.go +++ b/cmd/frontend/internal/context/resolvers/context_test.go @@ -245,7 +245,6 @@ func TestContextResolver(t *testing.T) { mockEmbeddingsClient, mockSearchClient, mockGitserver, - nil, ) resolver := NewResolver( diff --git a/cmd/frontend/internal/embeddings/resolvers/resolvers.go b/cmd/frontend/internal/embeddings/resolvers/resolvers.go index d9693f74aeb..700fbe0e8e0 100644 --- a/cmd/frontend/internal/embeddings/resolvers/resolvers.go +++ b/cmd/frontend/internal/embeddings/resolvers/resolvers.go @@ -161,19 +161,6 @@ func (r *Resolver) ScheduleRepositoriesForEmbedding(ctx context.Context, args gr return &graphqlbackend.EmptyResponse{}, nil } -func (r *Resolver) MigrateToQdrant(ctx context.Context) (*graphqlbackend.EmptyResponse, error) { - if !conf.EmbeddingsEnabled() { - return nil, errors.New("embeddings are not configured or disabled") - } - - ec := conf.GetEmbeddingsConfig(conf.Get().SiteConfig()) - if ec == nil || !ec.Qdrant.Enabled { - return nil, errors.New("qdrant is not enabled") - } - - return &graphqlbackend.EmptyResponse{}, r.repoEmbeddingJobsStore.RescheduleAllRepos(ctx) -} - func (r *Resolver) CancelRepoEmbeddingJob(ctx context.Context, args graphqlbackend.CancelRepoEmbeddingJobArgs) (*graphqlbackend.EmptyResponse, error) { // 🚨 SECURITY: check whether user is site-admin if err := auth.CheckCurrentUserIsSiteAdmin(ctx, r.db); err != nil { diff --git a/cmd/worker/internal/embeddings/repo/BUILD.bazel b/cmd/worker/internal/embeddings/repo/BUILD.bazel index 56a20b09681..76bb36fdce3 100644 --- a/cmd/worker/internal/embeddings/repo/BUILD.bazel +++ b/cmd/worker/internal/embeddings/repo/BUILD.bazel @@ -25,7 +25,6 @@ go_library( "//internal/database", "//internal/embeddings", "//internal/embeddings/background/repo", - "//internal/embeddings/db", "//internal/embeddings/embed", "//internal/env", "//internal/featureflag", diff --git a/cmd/worker/internal/embeddings/repo/handler.go b/cmd/worker/internal/embeddings/repo/handler.go index ffc2c6189cf..e26b419dc7e 100644 --- a/cmd/worker/internal/embeddings/repo/handler.go +++ b/cmd/worker/internal/embeddings/repo/handler.go @@ -16,7 +16,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/embeddings" bgrepo "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo" - "github.com/sourcegraph/sourcegraph/internal/embeddings/db" "github.com/sourcegraph/sourcegraph/internal/embeddings/embed" "github.com/sourcegraph/sourcegraph/internal/env" "github.com/sourcegraph/sourcegraph/internal/featureflag" @@ -31,7 +30,6 @@ type handler struct { db database.DB uploadStore uploadstore.Store gitserverClient gitserver.Client - getQdrantInserter func() (db.VectorInserter, error) contextService embed.ContextService repoEmbeddingJobsStore bgrepo.RepoEmbeddingJobsStore rankingService *ranking.Service @@ -104,22 +102,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo. return err } - modelID := embeddingsClient.GetModelIdentifier() - modelDims, err := embeddingsClient.GetDimensions() - if err != nil { - return err - } - - qdrantInserter, err := h.getQdrantInserter() - if err != nil { - return err - } - - err = qdrantInserter.PrepareUpdate(ctx, modelID, uint64(modelDims)) - if err != nil { - return err - } - var previousIndex *embeddings.RepoEmbeddingIndex if embeddingsConfig.Incremental { previousIndex, err = embeddings.DownloadRepoEmbeddingIndex(ctx, h.uploadStore, repo.ID, repo.Name) @@ -151,18 +133,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo. if previousIndex != nil { logger.Info("found previous embeddings index. Attempting incremental update", log.String("old_revision", string(previousIndex.Revision))) opts.IndexedRevision = previousIndex.Revision - - hasPreviousIndex, err := qdrantInserter.HasIndex(ctx, modelID, repo.ID, previousIndex.Revision) - if err != nil { - return err - } - - if !hasPreviousIndex { - err = uploadPreviousIndex(ctx, modelID, qdrantInserter, repo.ID, previousIndex) - if err != nil { - return err - } - } } ranks, err := h.rankingService.GetDocumentRanks(ctx, repo.Name) @@ -179,7 +149,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo. repoEmbeddingIndex, toRemove, stats, err := embed.EmbedRepo( ctx, embeddingsClient, - qdrantInserter, h.contextService, fetcher, repo.IDName(), @@ -192,16 +161,6 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo. return err } - err = qdrantInserter.FinalizeUpdate(ctx, db.FinalizeUpdateParams{ - ModelID: modelID, - RepoID: repo.ID, - Revision: record.Revision, - FilesToRemove: toRemove, - }) - if err != nil { - return err - } - reportStats(stats) // final, complete report logger.Info( @@ -324,45 +283,3 @@ func (r *revisionFetcher) validateRevision(ctx context.Context) error { } return nil } - -func uploadPreviousIndex(ctx context.Context, modelID string, inserter db.VectorInserter, repoID api.RepoID, previousIndex *embeddings.RepoEmbeddingIndex) error { - const batchSize = 128 - batch := make([]db.ChunkPoint, batchSize) - - for indexNum, index := range []embeddings.EmbeddingIndex{previousIndex.CodeIndex, previousIndex.TextIndex} { - isCode := indexNum == 0 - - // returns the ith row in the index as a ChunkPoint - getChunkPoint := func(i int) db.ChunkPoint { - payload := db.ChunkPayload{ - RepoName: previousIndex.RepoName, - RepoID: repoID, - Revision: previousIndex.Revision, - FilePath: index.RowMetadata[i].FileName, - StartLine: uint32(index.RowMetadata[i].StartLine), - EndLine: uint32(index.RowMetadata[i].EndLine), - IsCode: isCode, - } - return db.NewChunkPoint(payload, embeddings.Dequantize(index.Row(i))) - } - - for batchStart := 0; batchStart < len(index.RowMetadata); batchStart += batchSize { - // Build a batch - batch = batch[:0] // reset batch - for i := batchStart; i < batchStart+batchSize && i < len(index.RowMetadata); i++ { - batch = append(batch, getChunkPoint(i)) - } - - // Insert the batch - err := inserter.InsertChunks(ctx, db.InsertParams{ - ModelID: modelID, - ChunkPoints: batch, - }) - if err != nil { - return err - } - } - } - - return nil -} diff --git a/cmd/worker/internal/embeddings/repo/worker.go b/cmd/worker/internal/embeddings/repo/worker.go index c7e8b492096..88663419fdd 100644 --- a/cmd/worker/internal/embeddings/repo/worker.go +++ b/cmd/worker/internal/embeddings/repo/worker.go @@ -12,7 +12,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/database" "github.com/sourcegraph/sourcegraph/internal/embeddings" repoembeddingsbg "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo" - vdb "github.com/sourcegraph/sourcegraph/internal/embeddings/db" "github.com/sourcegraph/sourcegraph/internal/embeddings/embed" "github.com/sourcegraph/sourcegraph/internal/env" "github.com/sourcegraph/sourcegraph/internal/gitserver" @@ -54,10 +53,6 @@ func (s *repoEmbeddingJob) Routines(_ context.Context, observationCtx *observati return nil, err } - // qdrant is going to be removed. For now we only ever set the noop db. - qdrantDB := vdb.NewNoopDB() - getQdrantInserter := func() (vdb.VectorInserter, error) { return qdrantDB, nil } - workCtx := actor.WithInternalActor(context.Background()) return []goroutine.BackgroundRoutine{ newRepoEmbeddingJobWorker( @@ -67,7 +62,6 @@ func (s *repoEmbeddingJob) Routines(_ context.Context, observationCtx *observati db, uploadStore, gitserver.NewClient("embeddings.worker"), - getQdrantInserter, services.ContextService, repoembeddingsbg.NewRepoEmbeddingJobsStore(db), services.RankingService, @@ -82,7 +76,6 @@ func newRepoEmbeddingJobWorker( db database.DB, uploadStore uploadstore.Store, gitserverClient gitserver.Client, - getQdrantInserter func() (vdb.VectorInserter, error), contextService embed.ContextService, repoEmbeddingJobsStore repoembeddingsbg.RepoEmbeddingJobsStore, rankingService *ranking.Service, @@ -91,7 +84,6 @@ func newRepoEmbeddingJobWorker( db: db, uploadStore: uploadStore, gitserverClient: gitserverClient, - getQdrantInserter: getQdrantInserter, contextService: contextService, repoEmbeddingJobsStore: repoEmbeddingJobsStore, rankingService: rankingService, diff --git a/deps.bzl b/deps.bzl index 14ffb29f349..49024155de9 100644 --- a/deps.bzl +++ b/deps.bzl @@ -4823,13 +4823,6 @@ def go_dependencies(): sum = "h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=", version = "v0.0.0-20170810143723-de5bf2ad4578", ) - go_repository( - name = "com_github_qdrant_go_client", - build_file_proto_mode = "disable_global", - importpath = "github.com/qdrant/go-client", - sum = "h1:kh5B4yWjrd5BcynJoA4014mZlI/j6++inCMMQoKUOtI=", - version = "v1.4.1", - ) go_repository( name = "com_github_qustavo_sqlhooks_v2", build_file_proto_mode = "disable_global", diff --git a/dev/ci/images/images.go b/dev/ci/images/images.go index daa3bb3d292..96e3686b00c 100644 --- a/dev/ci/images/images.go +++ b/dev/ci/images/images.go @@ -125,7 +125,6 @@ var DeploySourcegraphDockerImages = []string{ "postgres_exporter", "precise-code-intel-worker", "prometheus", - "qdrant", "redis-cache", "redis-store", "redis_exporter", diff --git a/dev/oci_deps.bzl b/dev/oci_deps.bzl index 8cc87d34725..987b830e201 100644 --- a/dev/oci_deps.bzl +++ b/dev/oci_deps.bzl @@ -196,12 +196,6 @@ def oci_deps(): image = "index.docker.io/sourcegraph/wolfi-blobstore-base", ) - oci_pull( - name = "wolfi_qdrant_base", - digest = "sha256:6b0210c48107719494a6b59c9542817bb4222c47038ae3bf1cc29fed563770b3", - image = "index.docker.io/sourcegraph/wolfi-qdrant-base", - ) - oci_pull( name = "scip-java", digest = "sha256:808b063b7376cfc0a4937d89ddc3d4dd9652d10609865fae3f3b34302132737a", diff --git a/doc/dev/background-information/sg/reference.md b/doc/dev/background-information/sg/reference.md index 8cf4bd44abc..4d1f80a3b97 100644 --- a/doc/dev/background-information/sg/reference.md +++ b/doc/dev/background-information/sg/reference.md @@ -52,7 +52,6 @@ Available commandsets in `sg.config.yaml`: * monitoring * monitoring-alerts * otel -* qdrant * single-program * web-standalone * web-standalone-prod @@ -129,7 +128,6 @@ Available commands in `sg.config.yaml`: * pings * postgres_exporter * prometheus -* qdrant * redis-postgres: Dockerized version of redis and postgres * repo-updater * searcher diff --git a/docker-images/qdrant/BUILD.bazel b/docker-images/qdrant/BUILD.bazel deleted file mode 100644 index 07bc7ac1f06..00000000000 --- a/docker-images/qdrant/BUILD.bazel +++ /dev/null @@ -1,42 +0,0 @@ -load("@container_structure_test//:defs.bzl", "container_structure_test") -load("//dev:oci_defs.bzl", "image_repository", "oci_image", "oci_push", "oci_tarball") -load("//wolfi-images:defs.bzl", "wolfi_base") - -oci_image( - name = "qdrant_image", - base = ":base_image", - entrypoint = [ - "/sbin/tini", - "--", - "/usr/local/bin/qdrant", - "--config-path=/etc/qdrant/config.yaml", - ], - env = {}, - user = "sourcegraph", -) - -container_structure_test( - name = "qdrant_image_test", - timeout = "short", - configs = ["qdrant_image_test.yaml"], - driver = "docker", - image = ":qdrant_image", - tags = [ - "exclusive", - "requires-network", - ], -) - -oci_tarball( - name = "qdrant_image_tarball", - image = ":qdrant_image", - repo_tags = ["qdrant:candidate"], -) - -oci_push( - name = "qdrant_candidate_push", - image = ":qdrant_image", - repository = image_repository("qdrant"), -) - -wolfi_base() diff --git a/docker-images/qdrant/qdrant_image_test.yaml b/docker-images/qdrant/qdrant_image_test.yaml deleted file mode 100644 index 5390ff47d90..00000000000 --- a/docker-images/qdrant/qdrant_image_test.yaml +++ /dev/null @@ -1,20 +0,0 @@ -schemaVersion: "2.0.0" - -commandTests: - - name: "not running as root" - command: "/usr/bin/id" - args: - - -u - excludedOutput: ["^0"] - exitCode: 0 - - name: "qdrant is runnable" - command: "/usr/local/bin/qdrant" - args: - - --version - -fileExistenceTests: -- name: 'Test for qdrant' - path: '/usr/local/bin/qdrant' - shouldExist: true - uid: 0 - gid: 0 diff --git a/go.mod b/go.mod index 9cd0411a6a5..6294d21184c 100644 --- a/go.mod +++ b/go.mod @@ -273,7 +273,6 @@ require ( github.com/oschwald/maxminddb-golang v1.12.0 github.com/pkoukk/tiktoken-go v0.1.6 github.com/prometheus/statsd_exporter v0.22.7 - github.com/qdrant/go-client v1.4.1 github.com/sourcegraph/cloud-api v0.0.0-20231205211631-907f2d5f11b7 github.com/sourcegraph/managed-services-platform-cdktf/gen/cloudflare v0.0.0-20230822024612-edb48c530722 github.com/sourcegraph/managed-services-platform-cdktf/gen/google v0.0.0-20240325114905-87053fe51a82 diff --git a/go.sum b/go.sum index ea996dca416..9518f481019 100644 --- a/go.sum +++ b/go.sum @@ -1553,8 +1553,6 @@ github.com/pseudomuto/protoc-gen-doc v1.5.1 h1:Ah259kcrio7Ix1Rhb6u8FCaOkzf9qRBqX github.com/pseudomuto/protoc-gen-doc v1.5.1/go.mod h1:XpMKYg6zkcpgfpCfQ8GcWBDRtRxOmMR5w7pz4Xo+dYM= github.com/pseudomuto/protokit v0.2.1 h1:kCYpE3thoR6Esm0CUvd5xbrDTOZPvQPTDeyXpZfrJdk= github.com/pseudomuto/protokit v0.2.1/go.mod h1:gt7N5Rz2flBzYafvaxyIxMZC0TTF5jDZfRnw25hAAyo= -github.com/qdrant/go-client v1.4.1 h1:kh5B4yWjrd5BcynJoA4014mZlI/j6++inCMMQoKUOtI= -github.com/qdrant/go-client v1.4.1/go.mod h1:680gkxNAsVtre0Z8hAQmtPzJtz1xFAyCu2TUxULtnoE= github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0= github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU= github.com/rafaeljusto/redigomock/v3 v3.1.2 h1:B4Y0XJQiPjpwYmkH55aratKX1VfR+JRqzmDKyZbC99o= diff --git a/internal/api/internalapi/v1/internalapi.pb.go b/internal/api/internalapi/v1/internalapi.pb.go index 7f979b9d3a0..0c4b28f7e8d 100644 --- a/internal/api/internalapi/v1/internalapi.pb.go +++ b/internal/api/internalapi/v1/internalapi.pb.go @@ -199,8 +199,6 @@ type ServiceConnections struct { // Embeddings is the addresses of embeddings instances that should be talked // to. Embeddings []string `protobuf:"bytes,7,rep,name=embeddings,proto3" json:"embeddings,omitempty"` - // Qdrant is the address of the Qdrant instance (or empty if disabled) - Qdrant string `protobuf:"bytes,8,opt,name=qdrant,proto3" json:"qdrant,omitempty"` // Zoekts is the addresses of Zoekt instances to talk to. Zoekts []string `protobuf:"bytes,9,rep,name=zoekts,proto3" json:"zoekts,omitempty"` // ZoektListTTL is the TTL of the internal cache that Zoekt clients use to @@ -290,13 +288,6 @@ func (x *ServiceConnections) GetEmbeddings() []string { return nil } -func (x *ServiceConnections) GetQdrant() string { - if x != nil { - return x.Qdrant - } - return "" -} - func (x *ServiceConnections) GetZoekts() []string { if x != nil { return x.Zoekts @@ -333,7 +324,7 @@ var file_internalapi_proto_rawDesc = []byte{ 0x0b, 0x32, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x12, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x84, 0x03, + 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xfa, 0x02, 0x0a, 0x12, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x67, 0x69, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x69, 0x74, 0x53, 0x65, @@ -351,25 +342,25 @@ var file_internalapi_proto_rawDesc = []byte{ 0x62, 0x6f, 0x6c, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x69, - 0x6e, 0x67, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x71, 0x64, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x71, 0x64, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x7a, - 0x6f, 0x65, 0x6b, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x7a, 0x6f, 0x65, - 0x6b, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0e, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x5f, 0x6c, 0x69, 0x73, - 0x74, 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x4c, 0x69, 0x73, - 0x74, 0x54, 0x74, 0x6c, 0x32, 0x6e, 0x0a, 0x0d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5d, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, + 0x6e, 0x67, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x73, 0x18, 0x09, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x06, 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0e, 0x7a, + 0x6f, 0x65, 0x6b, 0x74, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x74, 0x74, 0x6c, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, + 0x7a, 0x6f, 0x65, 0x6b, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x74, 0x6c, 0x4a, 0x04, 0x08, 0x08, + 0x10, 0x09, 0x52, 0x06, 0x71, 0x64, 0x72, 0x61, 0x6e, 0x74, 0x32, 0x6e, 0x0a, 0x0d, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5d, 0x0a, 0x09, 0x47, + 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x03, 0x90, 0x02, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, + 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x61, 0x70, 0x69, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x03, 0x90, 0x02, 0x01, 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, + 0x72, 0x61, 0x70, 0x68, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, + 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/internal/api/internalapi/v1/internalapi.proto b/internal/api/internalapi/v1/internalapi.proto index 750309a469e..65154843bff 100644 --- a/internal/api/internalapi/v1/internalapi.proto +++ b/internal/api/internalapi/v1/internalapi.proto @@ -50,8 +50,8 @@ message ServiceConnections { // Embeddings is the addresses of embeddings instances that should be talked // to. repeated string embeddings = 7; - // Qdrant is the address of the Qdrant instance (or empty if disabled) - string qdrant = 8; + reserved 8; + reserved "qdrant"; // Zoekts is the addresses of Zoekt instances to talk to. repeated string zoekts = 9; // ZoektListTTL is the TTL of the internal cache that Zoekt clients use to diff --git a/internal/conf/computed.go b/internal/conf/computed.go index 1e14000cc5c..e7485905f09 100644 --- a/internal/conf/computed.go +++ b/internal/conf/computed.go @@ -1060,59 +1060,6 @@ func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.Embeddi MaxFileSizeBytes: maxFileSizeLimit, } - // Default values should match the documented defaults in site.schema.json. - computedQdrantConfig := conftypes.QdrantConfig{ - Enabled: false, - QdrantHNSWConfig: conftypes.QdrantHNSWConfig{ - EfConstruct: nil, - FullScanThreshold: nil, - M: nil, - OnDisk: true, - PayloadM: nil, - }, - QdrantOptimizersConfig: conftypes.QdrantOptimizersConfig{ - IndexingThreshold: 0, - MemmapThreshold: 100, - }, - QdrantQuantizationConfig: conftypes.QdrantQuantizationConfig{ - Enabled: true, - Quantile: 0.98, - }, - } - if embeddingsConfig.Qdrant != nil { - qc := embeddingsConfig.Qdrant - computedQdrantConfig.Enabled = qc.Enabled - - if qc.Hnsw != nil { - computedQdrantConfig.QdrantHNSWConfig.EfConstruct = toUint64(qc.Hnsw.EfConstruct) - computedQdrantConfig.QdrantHNSWConfig.FullScanThreshold = toUint64(qc.Hnsw.FullScanThreshold) - computedQdrantConfig.QdrantHNSWConfig.M = toUint64(qc.Hnsw.M) - computedQdrantConfig.QdrantHNSWConfig.PayloadM = toUint64(qc.Hnsw.PayloadM) - if qc.Hnsw.OnDisk != nil { - computedQdrantConfig.QdrantHNSWConfig.OnDisk = *qc.Hnsw.OnDisk - } - } - - if qc.Optimizers != nil { - if qc.Optimizers.IndexingThreshold != nil { - computedQdrantConfig.QdrantOptimizersConfig.IndexingThreshold = uint64(*qc.Optimizers.IndexingThreshold) - } - if qc.Optimizers.MemmapThreshold != nil { - computedQdrantConfig.QdrantOptimizersConfig.MemmapThreshold = uint64(*qc.Optimizers.MemmapThreshold) - } - } - - if qc.Quantization != nil { - if qc.Quantization.Enabled != nil { - computedQdrantConfig.QdrantQuantizationConfig.Enabled = *qc.Quantization.Enabled - } - - if qc.Quantization.Quantile != nil { - computedQdrantConfig.QdrantQuantizationConfig.Quantile = float32(*qc.Quantization.Quantile) - } - } - } - computedConfig := &conftypes.EmbeddingsConfig{ Provider: conftypes.EmbeddingsProviderName(embeddingsConfig.Provider), AccessToken: embeddingsConfig.AccessToken, @@ -1126,7 +1073,6 @@ func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.Embeddi MaxTextEmbeddingsPerRepo: embeddingsConfig.MaxTextEmbeddingsPerRepo, PolicyRepositoryMatchLimit: embeddingsConfig.PolicyRepositoryMatchLimit, ExcludeChunkOnError: pointers.Deref(embeddingsConfig.ExcludeChunkOnError, true), - Qdrant: computedQdrantConfig, PerCommunityUserEmbeddingsMonthlyLimit: embeddingsConfig.PerCommunityUserEmbeddingsMonthlyLimit, PerProUserEmbeddingsMonthlyLimit: embeddingsConfig.PerProUserEmbeddingsMonthlyLimit, } diff --git a/internal/conf/computed_test.go b/internal/conf/computed_test.go index 0340dfa5c47..c14d0054e51 100644 --- a/internal/conf/computed_test.go +++ b/internal/conf/computed_test.go @@ -767,19 +767,6 @@ func TestGetFeaturesConfig(t *testing.T) { func TestGetEmbeddingsConfig(t *testing.T) { licenseKey := "theasdfkey" licenseAccessToken := license.GenerateLicenseKeyBasedAccessToken(licenseKey) - defaultQdrantConfig := conftypes.QdrantConfig{ - QdrantHNSWConfig: conftypes.QdrantHNSWConfig{ - OnDisk: true, - }, - QdrantOptimizersConfig: conftypes.QdrantOptimizersConfig{ - IndexingThreshold: 0, - MemmapThreshold: 100, - }, - QdrantQuantizationConfig: conftypes.QdrantQuantizationConfig{ - Enabled: true, - Quantile: 0.98, - }, - } zeroConfigDefaultWithLicense := &conftypes.EmbeddingsConfig{ Provider: "sourcegraph", AccessToken: licenseAccessToken, @@ -795,7 +782,6 @@ func TestGetEmbeddingsConfig(t *testing.T) { MaxFileSizeBytes: 1000000, }, ExcludeChunkOnError: true, - Qdrant: defaultQdrantConfig, } testCases := []struct { @@ -911,7 +897,6 @@ func TestGetEmbeddingsConfig(t *testing.T) { ExcludedFilePathPatterns: []string{"*.java"}, }, ExcludeChunkOnError: true, - Qdrant: defaultQdrantConfig, }, }, { @@ -945,7 +930,6 @@ func TestGetEmbeddingsConfig(t *testing.T) { ExcludedFilePathPatterns: []string{"*.java"}, }, ExcludeChunkOnError: true, - Qdrant: defaultQdrantConfig, }, }, { @@ -981,7 +965,6 @@ func TestGetEmbeddingsConfig(t *testing.T) { ExcludedFilePathPatterns: []string{"*.java"}, }, ExcludeChunkOnError: false, - Qdrant: defaultQdrantConfig, }, }, { @@ -1009,7 +992,6 @@ func TestGetEmbeddingsConfig(t *testing.T) { MaxFileSizeBytes: 1000000, }, ExcludeChunkOnError: true, - Qdrant: defaultQdrantConfig, }, }, { @@ -1050,7 +1032,6 @@ func TestGetEmbeddingsConfig(t *testing.T) { MaxFileSizeBytes: 1000000, }, ExcludeChunkOnError: true, - Qdrant: defaultQdrantConfig, }, }, { @@ -1094,7 +1075,6 @@ func TestGetEmbeddingsConfig(t *testing.T) { MaxFileSizeBytes: 1000000, }, ExcludeChunkOnError: true, - Qdrant: defaultQdrantConfig, }, }, { diff --git a/internal/conf/conftypes/conftypes.go b/internal/conf/conftypes/conftypes.go index d285cf0fe61..da52779bb2c 100644 --- a/internal/conf/conftypes/conftypes.go +++ b/internal/conf/conftypes/conftypes.go @@ -38,8 +38,6 @@ type ServiceConnections struct { Symbols []string `json:"symbols"` // Embeddings is the addresses of embeddings instances that should be talked to. Embeddings []string `json:"embeddings"` - // Qdrant is the address of the Qdrant instance (or empty if disabled) - Qdrant string `json:"qdrant"` // Zoekts is the addresses of Zoekt instances to talk to. Zoekts []string `json:"zoekts"` // ZoektListTTL is the TTL of the internal cache that Zoekt clients use to @@ -57,7 +55,6 @@ func (sc *ServiceConnections) ToProto() *proto.ServiceConnections { Searchers: sc.Searchers, Symbols: sc.Symbols, Embeddings: sc.Embeddings, - Qdrant: sc.Qdrant, Zoekts: sc.Zoekts, ZoektListTtl: durationpb.New(sc.ZoektListTTL), } @@ -72,7 +69,6 @@ func (sc *ServiceConnections) FromProto(in *proto.ServiceConnections) { Searchers: in.GetSearchers(), Symbols: in.GetSymbols(), Embeddings: in.GetEmbeddings(), - Qdrant: in.GetQdrant(), Zoekts: in.GetZoekts(), ZoektListTTL: in.GetZoektListTtl().AsDuration(), } diff --git a/internal/conf/conftypes/consts.go b/internal/conf/conftypes/consts.go index 68d0261e644..68807dfcb0c 100644 --- a/internal/conf/conftypes/consts.go +++ b/internal/conf/conftypes/consts.go @@ -57,36 +57,10 @@ type EmbeddingsConfig struct { MaxTextEmbeddingsPerRepo int PolicyRepositoryMatchLimit *int ExcludeChunkOnError bool - Qdrant QdrantConfig PerCommunityUserEmbeddingsMonthlyLimit int PerProUserEmbeddingsMonthlyLimit int } -type QdrantConfig struct { - Enabled bool - QdrantHNSWConfig QdrantHNSWConfig - QdrantOptimizersConfig QdrantOptimizersConfig - QdrantQuantizationConfig QdrantQuantizationConfig -} - -type QdrantHNSWConfig struct { - EfConstruct *uint64 - FullScanThreshold *uint64 - M *uint64 - OnDisk bool - PayloadM *uint64 -} - -type QdrantOptimizersConfig struct { - IndexingThreshold uint64 - MemmapThreshold uint64 -} - -type QdrantQuantizationConfig struct { - Enabled bool - Quantile float32 -} - type EmbeddingsProviderName string const ( diff --git a/internal/embeddings/background/repo/mocks_temp.go b/internal/embeddings/background/repo/mocks_temp.go index 403149bf20b..294396069da 100644 --- a/internal/embeddings/background/repo/mocks_temp.go +++ b/internal/embeddings/background/repo/mocks_temp.go @@ -58,9 +58,6 @@ type MockRepoEmbeddingJobsStore struct { // ListRepoEmbeddingJobsFunc is an instance of a mock function object // controlling the behavior of the method ListRepoEmbeddingJobs. ListRepoEmbeddingJobsFunc *RepoEmbeddingJobsStoreListRepoEmbeddingJobsFunc - // RescheduleAllReposFunc is an instance of a mock function object - // controlling the behavior of the method RescheduleAllRepos. - RescheduleAllReposFunc *RepoEmbeddingJobsStoreRescheduleAllReposFunc // TransactFunc is an instance of a mock function object controlling the // behavior of the method Transact. TransactFunc *RepoEmbeddingJobsStoreTransactFunc @@ -135,11 +132,6 @@ func NewMockRepoEmbeddingJobsStore() *MockRepoEmbeddingJobsStore { return }, }, - RescheduleAllReposFunc: &RepoEmbeddingJobsStoreRescheduleAllReposFunc{ - defaultHook: func(context.Context) (r0 error) { - return - }, - }, TransactFunc: &RepoEmbeddingJobsStoreTransactFunc{ defaultHook: func(context.Context) (r0 RepoEmbeddingJobsStore, r1 error) { return @@ -218,11 +210,6 @@ func NewStrictMockRepoEmbeddingJobsStore() *MockRepoEmbeddingJobsStore { panic("unexpected invocation of MockRepoEmbeddingJobsStore.ListRepoEmbeddingJobs") }, }, - RescheduleAllReposFunc: &RepoEmbeddingJobsStoreRescheduleAllReposFunc{ - defaultHook: func(context.Context) error { - panic("unexpected invocation of MockRepoEmbeddingJobsStore.RescheduleAllRepos") - }, - }, TransactFunc: &RepoEmbeddingJobsStoreTransactFunc{ defaultHook: func(context.Context) (RepoEmbeddingJobsStore, error) { panic("unexpected invocation of MockRepoEmbeddingJobsStore.Transact") @@ -277,9 +264,6 @@ func NewMockRepoEmbeddingJobsStoreFrom(i RepoEmbeddingJobsStore) *MockRepoEmbedd ListRepoEmbeddingJobsFunc: &RepoEmbeddingJobsStoreListRepoEmbeddingJobsFunc{ defaultHook: i.ListRepoEmbeddingJobs, }, - RescheduleAllReposFunc: &RepoEmbeddingJobsStoreRescheduleAllReposFunc{ - defaultHook: i.RescheduleAllRepos, - }, TransactFunc: &RepoEmbeddingJobsStoreTransactFunc{ defaultHook: i.Transact, }, @@ -1607,112 +1591,6 @@ func (c RepoEmbeddingJobsStoreListRepoEmbeddingJobsFuncCall) Results() []interfa return []interface{}{c.Result0, c.Result1} } -// RepoEmbeddingJobsStoreRescheduleAllReposFunc describes the behavior when -// the RescheduleAllRepos method of the parent MockRepoEmbeddingJobsStore -// instance is invoked. -type RepoEmbeddingJobsStoreRescheduleAllReposFunc struct { - defaultHook func(context.Context) error - hooks []func(context.Context) error - history []RepoEmbeddingJobsStoreRescheduleAllReposFuncCall - mutex sync.Mutex -} - -// RescheduleAllRepos delegates to the next hook function in the queue and -// stores the parameter and result values of this invocation. -func (m *MockRepoEmbeddingJobsStore) RescheduleAllRepos(v0 context.Context) error { - r0 := m.RescheduleAllReposFunc.nextHook()(v0) - m.RescheduleAllReposFunc.appendCall(RepoEmbeddingJobsStoreRescheduleAllReposFuncCall{v0, r0}) - return r0 -} - -// SetDefaultHook sets function that is called when the RescheduleAllRepos -// method of the parent MockRepoEmbeddingJobsStore instance is invoked and -// the hook queue is empty. -func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) SetDefaultHook(hook func(context.Context) error) { - f.defaultHook = hook -} - -// PushHook adds a function to the end of hook queue. Each invocation of the -// RescheduleAllRepos method of the parent MockRepoEmbeddingJobsStore -// instance invokes the hook at the front of the queue and discards it. -// After the queue is empty, the default hook function is invoked for any -// future action. -func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) PushHook(hook func(context.Context) error) { - f.mutex.Lock() - f.hooks = append(f.hooks, hook) - f.mutex.Unlock() -} - -// SetDefaultReturn calls SetDefaultHook with a function that returns the -// given values. -func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) SetDefaultReturn(r0 error) { - f.SetDefaultHook(func(context.Context) error { - return r0 - }) -} - -// PushReturn calls PushHook with a function that returns the given values. -func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) PushReturn(r0 error) { - f.PushHook(func(context.Context) error { - return r0 - }) -} - -func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) nextHook() func(context.Context) error { - f.mutex.Lock() - defer f.mutex.Unlock() - - if len(f.hooks) == 0 { - return f.defaultHook - } - - hook := f.hooks[0] - f.hooks = f.hooks[1:] - return hook -} - -func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) appendCall(r0 RepoEmbeddingJobsStoreRescheduleAllReposFuncCall) { - f.mutex.Lock() - f.history = append(f.history, r0) - f.mutex.Unlock() -} - -// History returns a sequence of -// RepoEmbeddingJobsStoreRescheduleAllReposFuncCall objects describing the -// invocations of this function. -func (f *RepoEmbeddingJobsStoreRescheduleAllReposFunc) History() []RepoEmbeddingJobsStoreRescheduleAllReposFuncCall { - f.mutex.Lock() - history := make([]RepoEmbeddingJobsStoreRescheduleAllReposFuncCall, len(f.history)) - copy(history, f.history) - f.mutex.Unlock() - - return history -} - -// RepoEmbeddingJobsStoreRescheduleAllReposFuncCall is an object that -// describes an invocation of method RescheduleAllRepos on an instance of -// MockRepoEmbeddingJobsStore. -type RepoEmbeddingJobsStoreRescheduleAllReposFuncCall struct { - // Arg0 is the value of the 1st argument passed to this method - // invocation. - Arg0 context.Context - // Result0 is the value of the 1st result returned from this method - // invocation. - Result0 error -} - -// Args returns an interface slice containing the arguments of this -// invocation. -func (c RepoEmbeddingJobsStoreRescheduleAllReposFuncCall) Args() []interface{} { - return []interface{}{c.Arg0} -} - -// Results returns an interface slice containing the results of this -// invocation. -func (c RepoEmbeddingJobsStoreRescheduleAllReposFuncCall) Results() []interface{} { - return []interface{}{c.Result0} -} - // RepoEmbeddingJobsStoreTransactFunc describes the behavior when the // Transact method of the parent MockRepoEmbeddingJobsStore instance is // invoked. diff --git a/internal/embeddings/background/repo/store.go b/internal/embeddings/background/repo/store.go index c2459035f8c..a26bdbca0ce 100644 --- a/internal/embeddings/background/repo/store.go +++ b/internal/embeddings/background/repo/store.go @@ -106,7 +106,6 @@ type RepoEmbeddingJobsStore interface { CountRepoEmbeddingJobs(ctx context.Context, args ListOpts) (int, error) GetEmbeddableRepos(ctx context.Context, opts EmbeddableRepoOpts) ([]EmbeddableRepo, error) CancelRepoEmbeddingJob(ctx context.Context, job int) error - RescheduleAllRepos(ctx context.Context) error UpdateRepoEmbeddingJobStats(ctx context.Context, jobID int, stats *EmbedRepoStats) error GetRepoEmbeddingJobStats(ctx context.Context, jobID int) (EmbedRepoStats, error) @@ -256,27 +255,6 @@ func (s *repoEmbeddingJobsStore) CreateRepoEmbeddingJob(ctx context.Context, rep return id, err } -func (s *repoEmbeddingJobsStore) RescheduleAllRepos(ctx context.Context) error { - const rescheduleAllQuery = ` - INSERT INTO repo_embedding_jobs (repo_id, revision) - SELECT * FROM ( - SELECT - DISTINCT repo_id, - ( - SELECT r2.revision - FROM repo_embedding_jobs r2 - WHERE r2.repo_id = r1.repo_id - AND state = 'completed' - ORDER BY finished_at DESC - LIMIT 1 - ) latest_successful_revision - FROM repo_embedding_jobs r1 - ) subquery - WHERE latest_successful_revision IS NOT NULL - ` - return s.Store.Exec(ctx, sqlf.Sprintf(rescheduleAllQuery)) -} - var repoEmbeddingJobStatsColumns = []*sqlf.Query{ sqlf.Sprintf("repo_embedding_job_stats.job_id"), sqlf.Sprintf("repo_embedding_job_stats.is_incremental"), diff --git a/internal/embeddings/background/repo/store_test.go b/internal/embeddings/background/repo/store_test.go index b19a146438a..ddc9e5f2382 100644 --- a/internal/embeddings/background/repo/store_test.go +++ b/internal/embeddings/background/repo/store_test.go @@ -142,48 +142,6 @@ func TestRepoEmbeddingJobsStore(t *testing.T) { }) } -func TestRescheduleAll(t *testing.T) { - logger := logtest.Scoped(t) - db := database.NewDB(logger, dbtest.NewDB(t)) - repoStore := db.Repos() - ctx := context.Background() - - repo1 := &types.Repo{Name: "github.com/sourcegraph/sourcegraph", URI: "github.com/sourcegraph/sourcegraph", ExternalRepo: api.ExternalRepoSpec{}} - err := repoStore.Create(ctx, repo1) - require.NoError(t, err) - - repo2 := &types.Repo{Name: "github.com/sourcegraph/sourcegraph2", URI: "github.com/sourcegraph/sourcegraph2", ExternalRepo: api.ExternalRepoSpec{}} - err = repoStore.Create(ctx, repo2) - require.NoError(t, err) - - repo3 := &types.Repo{Name: "github.com/sourcegraph/sourcegraph3", URI: "github.com/sourcegraph/sourcegraph3", ExternalRepo: api.ExternalRepoSpec{}} - err = repoStore.Create(ctx, repo3) - require.NoError(t, err) - - // Insert three completed jobs from two repos - _, err = db.Handle().ExecContext(ctx, fmt.Sprintf( - `insert into repo_embedding_jobs (repo_id, revision, state) values - (%d, 'rev1', 'completed'), - (%d, 'rev2', 'completed'), - (%d, 'rev3', 'completed'), - (%d, 'rev4', 'failed') - `, - repo1.ID, - repo1.ID, - repo2.ID, - repo3.ID, - )) - require.NoError(t, err) - - store := NewRepoEmbeddingJobsStore(db) - err = store.RescheduleAllRepos(ctx) - require.NoError(t, err) - - jobs, err := store.ListRepoEmbeddingJobs(ctx, ListOpts{PaginationArgs: &database.PaginationArgs{}}) - require.NoError(t, err) - require.Len(t, jobs, 6) // 4 jobs to start, added 2 -} - func TestCancelRepoEmbeddingJob(t *testing.T) { logger := logtest.Scoped(t) db := database.NewDB(logger, dbtest.NewDB(t)) diff --git a/internal/embeddings/db/BUILD.bazel b/internal/embeddings/db/BUILD.bazel deleted file mode 100644 index eb26c3ad937..00000000000 --- a/internal/embeddings/db/BUILD.bazel +++ /dev/null @@ -1,46 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") -load("//dev:go_defs.bzl", "go_test") - -go_library( - name = "db", - srcs = [ - "chunk_point.go", - "conf.go", - "db.go", - "migrate.go", - "noop.go", - "qdrant.go", - ], - importpath = "github.com/sourcegraph/sourcegraph/internal/embeddings/db", - visibility = ["//:__subpackages__"], - deps = [ - "//internal/api", - "//internal/conf", - "//internal/conf/conftypes", - "//internal/grpc/defaults", - "//lib/errors", - "//lib/pointers", - "@com_github_google_uuid//:uuid", - "@com_github_qdrant_go_client//qdrant", - "@com_github_sourcegraph_log//:log", - "@org_golang_google_grpc//:go_default_library", - ], -) - -go_test( - name = "db_test", - srcs = [ - "chunk_point_test.go", - "conf_test.go", - ], - embed = [":db"], - deps = [ - "//internal/conf", - "//internal/conf/conftypes", - "//internal/dotcom", - "//lib/pointers", - "//schema", - "@com_github_sourcegraph_log//logtest", - "@com_github_stretchr_testify//require", - ], -) diff --git a/internal/embeddings/db/chunk_point.go b/internal/embeddings/db/chunk_point.go deleted file mode 100644 index 57e9ef1e996..00000000000 --- a/internal/embeddings/db/chunk_point.go +++ /dev/null @@ -1,151 +0,0 @@ -package db - -import ( - "encoding/binary" - "hash/fnv" - - "github.com/google/uuid" - qdrant "github.com/qdrant/go-client/qdrant" - - "github.com/sourcegraph/sourcegraph/internal/api" -) - -// ChunkResult is a point along with its search score. -type ChunkResult struct { - Point ChunkPoint - Score float32 -} - -func (c *ChunkResult) FromQdrantResult(res *qdrant.ScoredPoint) error { - u, err := uuid.Parse(res.GetId().GetUuid()) - if err != nil { - return err - } - var payload ChunkPayload - payload.FromQdrantPayload(res.GetPayload()) - *c = ChunkResult{ - Point: ChunkPoint{ - ID: u, - Payload: payload, - Vector: res.GetVectors().GetVector().GetData(), - }, - Score: res.GetScore(), - } - return nil -} - -func NewChunkPoint(payload ChunkPayload, vector []float32) ChunkPoint { - return ChunkPoint{ - ID: chunkUUID( - payload.RepoID, - payload.Revision, - payload.FilePath, - payload.StartLine, - payload.EndLine, - ), - Payload: payload, - Vector: vector, - } -} - -type ChunkPoint struct { - ID uuid.UUID - Payload ChunkPayload - Vector []float32 -} - -func (c *ChunkPoint) ToQdrantPoint() *qdrant.PointStruct { - return &qdrant.PointStruct{ - Id: &qdrant.PointId{ - PointIdOptions: &qdrant.PointId_Uuid{ - Uuid: c.ID.String(), - }, - }, - Payload: c.Payload.ToQdrantPayload(), - Vectors: &qdrant.Vectors{ - VectorsOptions: &qdrant.Vectors_Vector{ - Vector: &qdrant.Vector{ - Data: c.Vector, - }, - }, - }, - } -} - -type ChunkPoints []ChunkPoint - -func (ps ChunkPoints) ToQdrantPoints() []*qdrant.PointStruct { - res := make([]*qdrant.PointStruct, len(ps)) - for i, p := range ps { - res[i] = p.ToQdrantPoint() - } - return res -} - -type PayloadField = string - -const ( - fieldRepoID PayloadField = "repoID" - fieldRepoName PayloadField = "repoName" - fieldRevision PayloadField = "revision" - fieldFilePath PayloadField = "filePath" - fieldStartLine PayloadField = "startLine" - fieldEndLine PayloadField = "endLine" - fieldIsCode PayloadField = "isCode" -) - -// ChunkPayload is a well-typed representation of the payload we store in the vector DB. -// Changes to the contents of this struct may require a migration of the data in the DB. -type ChunkPayload struct { - RepoName api.RepoName - RepoID api.RepoID - Revision api.CommitID - FilePath string - StartLine, EndLine uint32 - IsCode bool -} - -func (p *ChunkPayload) ToQdrantPayload() map[string]*qdrant.Value { - return map[string]*qdrant.Value{ - fieldRepoID: {Kind: &qdrant.Value_IntegerValue{IntegerValue: int64(p.RepoID)}}, - fieldRepoName: {Kind: &qdrant.Value_StringValue{StringValue: string(p.RepoName)}}, - fieldRevision: {Kind: &qdrant.Value_StringValue{StringValue: string(p.Revision)}}, - fieldFilePath: {Kind: &qdrant.Value_StringValue{StringValue: p.FilePath}}, - fieldStartLine: {Kind: &qdrant.Value_IntegerValue{IntegerValue: int64(p.StartLine)}}, - fieldEndLine: {Kind: &qdrant.Value_IntegerValue{IntegerValue: int64(p.EndLine)}}, - fieldIsCode: {Kind: &qdrant.Value_BoolValue{BoolValue: p.IsCode}}, - } -} - -func (p *ChunkPayload) FromQdrantPayload(payload map[string]*qdrant.Value) { - *p = ChunkPayload{ - RepoName: api.RepoName(payload[fieldRepoName].GetStringValue()), - RepoID: api.RepoID(payload[fieldRepoID].GetIntegerValue()), - Revision: api.CommitID(payload[fieldRevision].GetStringValue()), - FilePath: payload[fieldFilePath].GetStringValue(), - StartLine: uint32(payload[fieldStartLine].GetIntegerValue()), - EndLine: uint32(payload[fieldEndLine].GetIntegerValue()), - IsCode: payload[fieldIsCode].GetBoolValue(), - } -} - -// chunkUUID generates a stable UUID for a file chunk. It is not strictly necessary to have a stable ID, -// but it does make it easier to reason about idempotent updates. -func chunkUUID(repoID api.RepoID, revision api.CommitID, filePath string, startLine, endLine uint32) uuid.UUID { - hasher := fnv.New128() - - var buf [4]byte - - binary.LittleEndian.PutUint32(buf[:], uint32(repoID)) - hasher.Write(buf[:]) - hasher.Write([]byte(revision)) - hasher.Write([]byte(filePath)) - binary.LittleEndian.PutUint32(buf[:], startLine) - binary.LittleEndian.PutUint32(buf[:], endLine) - hasher.Write(buf[:]) - - var u uuid.UUID - sum := hasher.Sum(nil) - copy(u[:], sum) - return u -} diff --git a/internal/embeddings/db/chunk_point_test.go b/internal/embeddings/db/chunk_point_test.go deleted file mode 100644 index 62c444270f3..00000000000 --- a/internal/embeddings/db/chunk_point_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package db - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestPayload(t *testing.T) { - t.Run("roundtrip", func(t *testing.T) { - pp := ChunkPayload{ - RepoName: "a", - RepoID: 2, - Revision: "c", - FilePath: "d", - StartLine: 5, - EndLine: 6, - IsCode: false, - } - - qp := pp.ToQdrantPayload() - var newPP ChunkPayload - newPP.FromQdrantPayload(qp) - require.Equal(t, pp, newPP) - }) -} diff --git a/internal/embeddings/db/conf.go b/internal/embeddings/db/conf.go deleted file mode 100644 index 661ff43a26f..00000000000 --- a/internal/embeddings/db/conf.go +++ /dev/null @@ -1,166 +0,0 @@ -package db - -import ( - "sync/atomic" - - qdrant "github.com/qdrant/go-client/qdrant" - "github.com/sourcegraph/log" - - "github.com/sourcegraph/sourcegraph/internal/conf" - "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" - "github.com/sourcegraph/sourcegraph/internal/grpc/defaults" - "github.com/sourcegraph/sourcegraph/lib/pointers" - "google.golang.org/grpc" -) - -// NewDBFromConfFunc returns a function that can be called to get an -// VectorDB instance based on the connection info from the conf package. -// It will watch conf and update the connection if there are any changes. -// -// If Qdrant is disabled, it will instead return the provided default VectorDB. -func NewDBFromConfFunc(logger log.Logger, def VectorDB) func() (VectorDB, error) { - type connAndErr struct { - conn *grpc.ClientConn - err error - } - var ( - oldAddr string - ptr atomic.Pointer[connAndErr] - ) - - conf.Watch(func() { - c := conf.Get() - qc := conf.GetEmbeddingsConfig(c.SiteConfiguration) - if qc == nil || !qc.Qdrant.Enabled { - // Embeddings is disabled. Clear any errors and close any previous connection. - old := ptr.Swap(&connAndErr{nil, nil}) - if old != nil && old.conn != nil { - old.conn.Close() - } - return - } - if newAddr := c.ServiceConnections().Qdrant; newAddr != oldAddr { - // The address has changed or this is running for the first time. - // Attempt to open dial Qdrant. - newConn, newErr := defaults.Dial(newAddr, logger) - oldAddr = newAddr - old := ptr.Swap(&connAndErr{newConn, newErr}) - if old != nil && old.conn != nil { - old.conn.Close() - } - } - }) - - return func() (VectorDB, error) { - curr := ptr.Load() - if curr == nil { - return def, nil - } - if curr.err != nil { - return nil, curr.err - } - if curr.conn == nil { - return def, nil - } - - return NewQdrantDBFromConn(curr.conn), nil - } -} - -func createCollectionParams(name string, dims uint64, qc conftypes.QdrantConfig) *qdrant.CreateCollection { - return &qdrant.CreateCollection{ - CollectionName: name, - HnswConfig: getHnswConfigDiff(qc.QdrantHNSWConfig), - OptimizersConfig: getOptimizersConfigDiff(qc.QdrantOptimizersConfig), - QuantizationConfig: getQuantizationConfig(qc.QdrantQuantizationConfig), - OnDiskPayload: pointers.Ptr(true), - VectorsConfig: getVectorsConfig(dims), - ShardNumber: nil, // default - WalConfig: nil, // default - ReplicationFactor: nil, // default - WriteConsistencyFactor: nil, // default - InitFromCollection: nil, // default - } -} - -func updateCollectionParams(name string, qc conftypes.QdrantConfig) *qdrant.UpdateCollection { - return &qdrant.UpdateCollection{ - CollectionName: name, - HnswConfig: getHnswConfigDiff(qc.QdrantHNSWConfig), - OptimizersConfig: getOptimizersConfigDiff(qc.QdrantOptimizersConfig), - QuantizationConfig: getQuantizationConfigDiff(qc.QdrantQuantizationConfig), - Params: nil, // do not update collection params - // Do not update vectors config. - // TODO(camdencheek): consider making OnDisk configurable - VectorsConfig: nil, - } -} - -func getVectorsConfig(dims uint64) *qdrant.VectorsConfig { - return &qdrant.VectorsConfig{ - Config: &qdrant.VectorsConfig_Params{ - Params: &qdrant.VectorParams{ - Size: dims, - Distance: qdrant.Distance_Cosine, - OnDisk: pointers.Ptr(true), - HnswConfig: nil, // use collection default - QuantizationConfig: nil, // use collection default - }, - }, - } -} - -func getHnswConfigDiff(qhc conftypes.QdrantHNSWConfig) *qdrant.HnswConfigDiff { - return &qdrant.HnswConfigDiff{ - M: qhc.M, - PayloadM: qhc.PayloadM, - EfConstruct: qhc.EfConstruct, - FullScanThreshold: qhc.FullScanThreshold, - OnDisk: &qhc.OnDisk, - MaxIndexingThreads: nil, // default - } -} - -func getOptimizersConfigDiff(qoc conftypes.QdrantOptimizersConfig) *qdrant.OptimizersConfigDiff { - // Default values should match the documented defaults in site.schema.json. - return &qdrant.OptimizersConfigDiff{ - IndexingThreshold: &qoc.IndexingThreshold, - MemmapThreshold: &qoc.MemmapThreshold, - DefaultSegmentNumber: nil, - VacuumMinVectorNumber: nil, - MaxOptimizationThreads: nil, - } -} - -func getQuantizationConfigDiff(qqc conftypes.QdrantQuantizationConfig) *qdrant.QuantizationConfigDiff { - if !qqc.Enabled { - return &qdrant.QuantizationConfigDiff{ - Quantization: &qdrant.QuantizationConfigDiff_Disabled{}, - } - } - - return &qdrant.QuantizationConfigDiff{ - Quantization: &qdrant.QuantizationConfigDiff_Scalar{ - Scalar: &qdrant.ScalarQuantization{ - Type: qdrant.QuantizationType_Int8, - Quantile: &qqc.Quantile, - AlwaysRam: pointers.Ptr(false), - }, - }, - } -} - -func getQuantizationConfig(qqc conftypes.QdrantQuantizationConfig) *qdrant.QuantizationConfig { - if !qqc.Enabled { - return nil - } - return &qdrant.QuantizationConfig{ - Quantization: &qdrant.QuantizationConfig_Scalar{ - Scalar: &qdrant.ScalarQuantization{ - Type: qdrant.QuantizationType_Int8, - Quantile: &qqc.Quantile, - AlwaysRam: pointers.Ptr(false), - }, - }, - } -} diff --git a/internal/embeddings/db/conf_test.go b/internal/embeddings/db/conf_test.go deleted file mode 100644 index 36a16877e23..00000000000 --- a/internal/embeddings/db/conf_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package db - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/sourcegraph/log/logtest" - - "github.com/sourcegraph/sourcegraph/internal/conf" - "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" - "github.com/sourcegraph/sourcegraph/internal/dotcom" - "github.com/sourcegraph/sourcegraph/lib/pointers" - "github.com/sourcegraph/sourcegraph/schema" -) - -func TestNewDBFromConfFunc(t *testing.T) { - dotcom.MockSourcegraphDotComMode(t, true) - - t.Run("default nil", func(t *testing.T) { - conf.Mock(&conf.Unified{ - ServiceConnectionConfig: conftypes.ServiceConnections{ - Qdrant: "", - }, - }) - getDB := NewDBFromConfFunc(logtest.Scoped(t), nil) - got, err := getDB() - require.NoError(t, err) - require.Nil(t, got) - }) - - t.Run("fake addr", func(t *testing.T) { - conf.Mock(&conf.Unified{ - SiteConfiguration: schema.SiteConfiguration{ - Embeddings: &schema.Embeddings{ - Provider: "sourcegraph", - AccessToken: "fake", - Enabled: pointers.Ptr(true), - Qdrant: &schema.Qdrant{ - Enabled: true, - }, - }, - CodyEnabled: pointers.Ptr(true), - }, - ServiceConnectionConfig: conftypes.ServiceConnections{Qdrant: "fake_address_but_it_does_not_matter_because_grpc_dialing_is_lazy"}, - }) - getDB := NewDBFromConfFunc(logtest.Scoped(t), nil) - got, err := getDB() - require.NoError(t, err) - require.NotNil(t, got) - }) -} diff --git a/internal/embeddings/db/db.go b/internal/embeddings/db/db.go deleted file mode 100644 index c8afd3c1852..00000000000 --- a/internal/embeddings/db/db.go +++ /dev/null @@ -1,29 +0,0 @@ -package db - -import ( - "context" - "fmt" - "strings" - - "github.com/sourcegraph/sourcegraph/internal/api" -) - -type VectorDB interface { - VectorSearcher - VectorInserter -} - -type VectorSearcher interface { - Search(context.Context, SearchParams) ([]ChunkResult, error) -} - -type VectorInserter interface { - PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error - HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) - InsertChunks(context.Context, InsertParams) error - FinalizeUpdate(context.Context, FinalizeUpdateParams) error -} - -func CollectionName(modelID string) string { - return fmt.Sprintf("repos.%s", strings.ReplaceAll(modelID, "/", ".")) -} diff --git a/internal/embeddings/db/migrate.go b/internal/embeddings/db/migrate.go deleted file mode 100644 index 15c320f4453..00000000000 --- a/internal/embeddings/db/migrate.go +++ /dev/null @@ -1,79 +0,0 @@ -package db - -import ( - "context" - - qdrant "github.com/qdrant/go-client/qdrant" - "github.com/sourcegraph/sourcegraph/internal/conf" - "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" - "github.com/sourcegraph/sourcegraph/lib/errors" - "github.com/sourcegraph/sourcegraph/lib/pointers" -) - -func ensureModelCollection(ctx context.Context, db *qdrantDB, modelID string, modelDims uint64) error { - // Make the actual collection end with `.default` so we can switch between - // configurations with aliases. - name := CollectionName(modelID) - realName := name + ".default" - - err := ensureCollectionWithConfig(ctx, db.collectionsClient, realName, modelDims, conf.GetEmbeddingsConfig(conf.Get().SiteConfiguration).Qdrant) - if err != nil { - return err - } - - // Update the alias atomically to point to the new collection - _, err = db.collectionsClient.UpdateAliases(ctx, &qdrant.ChangeAliases{ - Actions: []*qdrant.AliasOperations{{ - Action: &qdrant.AliasOperations_CreateAlias{ - CreateAlias: &qdrant.CreateAlias{ - CollectionName: realName, - AliasName: name, - }, - }, - }}, - }) - if err != nil { - return errors.Wrap(err, "update aliases") - } - - err = ensureRepoIDIndex(ctx, db.pointsClient, realName) - if err != nil { - return errors.Wrap(err, "add repo index") - } - - return nil -} - -func ensureCollectionWithConfig(ctx context.Context, cc qdrant.CollectionsClient, name string, dims uint64, qc conftypes.QdrantConfig) error { - resp, err := cc.List(ctx, &qdrant.ListCollectionsRequest{}) - if err != nil { - return err - } - - for _, collection := range resp.GetCollections() { - if collection.GetName() == name { - _, err = cc.Update(ctx, updateCollectionParams(name, qc)) - return err - } - } - - _, err = cc.Create(ctx, createCollectionParams(name, dims, qc)) - return err -} - -func ensureRepoIDIndex(ctx context.Context, cc qdrant.PointsClient, name string) error { - // This is idempotent, so no need to check if it exists first - _, err := cc.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{ - CollectionName: name, - Wait: pointers.Ptr(true), - FieldName: fieldRepoID, - FieldType: pointers.Ptr(qdrant.FieldType_FieldTypeInteger), - FieldIndexParams: nil, - Ordering: nil, - }) - if err != nil { - return err - } - - return nil -} diff --git a/internal/embeddings/db/noop.go b/internal/embeddings/db/noop.go deleted file mode 100644 index 503acefd11e..00000000000 --- a/internal/embeddings/db/noop.go +++ /dev/null @@ -1,58 +0,0 @@ -package db - -import ( - "context" - - "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/lib/errors" -) - -func NewNoopDB() VectorDB { - return noopDB{} -} - -var _ VectorDB = noopDB{} - -type noopDB struct{} - -func (noopDB) Search(context.Context, SearchParams) ([]ChunkResult, error) { - return nil, nil -} -func (noopDB) PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error { - return nil -} -func (noopDB) HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) { - return false, nil -} -func (noopDB) InsertChunks(context.Context, InsertParams) error { - return nil -} -func (noopDB) FinalizeUpdate(context.Context, FinalizeUpdateParams) error { - return nil -} - -var ErrDisabled = errors.New("Qdrant is disabled. Enable by setting QDRANT_ENDPOINT") - -func NewDisabledDB() VectorDB { - return disabledDB{} -} - -var _ VectorDB = disabledDB{} - -type disabledDB struct{} - -func (disabledDB) Search(context.Context, SearchParams) ([]ChunkResult, error) { - return nil, ErrDisabled -} -func (disabledDB) PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error { - return ErrDisabled -} -func (disabledDB) HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) { - return false, ErrDisabled -} -func (disabledDB) InsertChunks(context.Context, InsertParams) error { - return ErrDisabled -} -func (disabledDB) FinalizeUpdate(context.Context, FinalizeUpdateParams) error { - return ErrDisabled -} diff --git a/internal/embeddings/db/qdrant.go b/internal/embeddings/db/qdrant.go deleted file mode 100644 index 0494743baa9..00000000000 --- a/internal/embeddings/db/qdrant.go +++ /dev/null @@ -1,284 +0,0 @@ -package db - -import ( - "context" - - qdrant "github.com/qdrant/go-client/qdrant" - "google.golang.org/grpc" - - "github.com/sourcegraph/sourcegraph/internal/api" - "github.com/sourcegraph/sourcegraph/lib/pointers" -) - -func NewQdrantDBFromConn(conn *grpc.ClientConn) VectorDB { - return NewQdrantDB(qdrant.NewPointsClient(conn), qdrant.NewCollectionsClient(conn)) -} - -func NewQdrantDB(pointsClient qdrant.PointsClient, collectionsClient qdrant.CollectionsClient) VectorDB { - return &qdrantDB{ - pointsClient: pointsClient, - collectionsClient: collectionsClient, - } -} - -type qdrantDB struct { - pointsClient qdrant.PointsClient - collectionsClient qdrant.CollectionsClient -} - -var _ VectorDB = (*qdrantDB)(nil) - -type SearchParams struct { - // RepoIDs is the set of repos to search. - // If empty, all repos are searched. - RepoIDs []api.RepoID - - // The ID of the model that the query was embedded with. - // Embeddings for other models will not be searched. - ModelID string - - // Query is the embedding for the search query. - // Its dimensions must match the model dimensions. - Query []float32 - - // The maximum number of code results to return - CodeLimit int - - // The maximum number of text results to return - TextLimit int -} - -func (db *qdrantDB) Search(ctx context.Context, params SearchParams) ([]ChunkResult, error) { - collectionName := CollectionName(params.ModelID) - - getSearchPoints := func(isCode bool) *qdrant.SearchPoints { - var limit uint64 - if isCode { - limit = uint64(params.CodeLimit) - } else { - limit = uint64(params.TextLimit) - } - return &qdrant.SearchPoints{ - CollectionName: collectionName, - Vector: params.Query, - WithPayload: fullPayloadSelector, - Filter: &qdrant.Filter{ - Should: repoIDsConditions(params.RepoIDs), - Must: []*qdrant.Condition{isCodeCondition(isCode)}, - }, - Limit: limit, - } - } - codeSearch := getSearchPoints(true) - textSearch := getSearchPoints(false) - - resp, err := db.pointsClient.SearchBatch(ctx, &qdrant.SearchBatchPoints{ - CollectionName: collectionName, - SearchPoints: []*qdrant.SearchPoints{codeSearch, textSearch}, - }) - if err != nil { - return nil, err - } - - results := make([]ChunkResult, 0, params.CodeLimit+params.TextLimit) - for _, group := range resp.GetResult() { - for _, res := range group.GetResult() { - var cr ChunkResult - if err := cr.FromQdrantResult(res); err != nil { - return nil, err - } - results = append(results, cr) - } - } - return results, nil -} - -func (db *qdrantDB) PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error { - return ensureModelCollection(ctx, db, modelID, modelDims) -} - -func (db *qdrantDB) HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) { - resp, err := db.pointsClient.Scroll(ctx, &qdrant.ScrollPoints{ - CollectionName: CollectionName(modelID), - Filter: &qdrant.Filter{ - Must: []*qdrant.Condition{ - repoIDCondition(repoID), - revisionCondition(revision), - }, - }, - Limit: pointers.Ptr(uint32(1)), - }) - if err != nil { - return false, err - } - - return len(resp.GetResult()) > 0, nil -} - -type InsertParams struct { - ModelID string - ChunkPoints ChunkPoints -} - -func (db *qdrantDB) InsertChunks(ctx context.Context, params InsertParams) error { - _, err := db.pointsClient.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: CollectionName(params.ModelID), - // Wait to avoid overloading the server - Wait: pointers.Ptr(true), - Points: params.ChunkPoints.ToQdrantPoints(), - Ordering: nil, - }) - return err -} - -type FinalizeUpdateParams struct { - ModelID string - RepoID api.RepoID - Revision api.CommitID - FilesToRemove []string -} - -// TODO: document that this is idempotent and why it's important -func (db *qdrantDB) FinalizeUpdate(ctx context.Context, params FinalizeUpdateParams) error { - // First, delete the old files - err := db.deleteFiles(ctx, params) - if err != nil { - return err - } - - // Then, update all the unchanged chunks to use the latest revision - err = db.updateRevisions(ctx, params) - if err != nil { - return err - } - - return nil -} - -func (db *qdrantDB) deleteFiles(ctx context.Context, params FinalizeUpdateParams) error { - // TODO: batch the deletes in case the file list is extremely large - filePathConditions := make([]*qdrant.Condition, len(params.FilesToRemove)) - for i, path := range params.FilesToRemove { - filePathConditions[i] = filePathCondition(path) - } - - _, err := db.pointsClient.Delete(ctx, &qdrant.DeletePoints{ - CollectionName: CollectionName(params.ModelID), - Wait: pointers.Ptr(true), // wait until deleted before sending update - Ordering: &qdrant.WriteOrdering{Type: qdrant.WriteOrderingType_Strong}, - Points: &qdrant.PointsSelector{ - PointsSelectorOneOf: &qdrant.PointsSelector_Filter{ - Filter: &qdrant.Filter{ - // Only chunks for this repo - Must: []*qdrant.Condition{repoIDCondition(params.RepoID)}, - // No chunks that are from the newest revision - MustNot: []*qdrant.Condition{revisionCondition(params.Revision)}, - // Chunks that match at least one of the "to remove" filenames - Should: filePathConditions, - }, - }, - }, - }) - return err -} - -func (db *qdrantDB) updateRevisions(ctx context.Context, params FinalizeUpdateParams) error { - _, err := db.pointsClient.SetPayload(ctx, &qdrant.SetPayloadPoints{ - CollectionName: CollectionName(params.ModelID), - Wait: pointers.Ptr(true), // wait until deleted before sending update - Ordering: &qdrant.WriteOrdering{Type: qdrant.WriteOrderingType_Strong}, - Payload: map[string]*qdrant.Value{ - fieldRevision: { - Kind: &qdrant.Value_StringValue{ - StringValue: string(params.Revision), - }, - }, - }, - PointsSelector: &qdrant.PointsSelector{ - PointsSelectorOneOf: &qdrant.PointsSelector_Filter{ - Filter: &qdrant.Filter{ - // Only chunks in this repo - Must: []*qdrant.Condition{repoIDCondition(params.RepoID)}, - // Only chunks that are not already marked as part of this revision - MustNot: []*qdrant.Condition{revisionCondition(params.Revision)}, - }, - }, - }, - }) - return err -} - -func filePathCondition(path string) *qdrant.Condition { - return &qdrant.Condition{ - ConditionOneOf: &qdrant.Condition_Field{ - Field: &qdrant.FieldCondition{ - Key: fieldFilePath, - Match: &qdrant.Match{ - MatchValue: &qdrant.Match_Keyword{ - Keyword: string(path), - }, - }, - }, - }, - } -} - -func revisionCondition(revision api.CommitID) *qdrant.Condition { - return &qdrant.Condition{ - ConditionOneOf: &qdrant.Condition_Field{ - Field: &qdrant.FieldCondition{ - Key: fieldRevision, - Match: &qdrant.Match{ - MatchValue: &qdrant.Match_Keyword{ - Keyword: string(revision), - }, - }, - }, - }, - } -} - -func isCodeCondition(isCode bool) *qdrant.Condition { - return &qdrant.Condition{ - ConditionOneOf: &qdrant.Condition_Field{ - Field: &qdrant.FieldCondition{ - Key: fieldIsCode, - Match: &qdrant.Match{ - MatchValue: &qdrant.Match_Boolean{ - Boolean: isCode, - }, - }, - }, - }, - } -} - -func repoIDsConditions(ids []api.RepoID) []*qdrant.Condition { - conds := make([]*qdrant.Condition, len(ids)) - for i, id := range ids { - conds[i] = repoIDCondition(id) - } - return conds -} - -func repoIDCondition(repoID api.RepoID) *qdrant.Condition { - return &qdrant.Condition{ - ConditionOneOf: &qdrant.Condition_Field{ - Field: &qdrant.FieldCondition{ - Key: fieldRepoID, - Match: &qdrant.Match{ - MatchValue: &qdrant.Match_Integer{ - Integer: int64(repoID), - }, - }, - }, - }, - } -} - -// Select the full payload -var fullPayloadSelector = &qdrant.WithPayloadSelector{ - SelectorOptions: &qdrant.WithPayloadSelector_Enable{ - Enable: true, - }, -} diff --git a/internal/embeddings/embed/BUILD.bazel b/internal/embeddings/embed/BUILD.bazel index 68e3c5c5c0f..679944f5139 100644 --- a/internal/embeddings/embed/BUILD.bazel +++ b/internal/embeddings/embed/BUILD.bazel @@ -19,7 +19,6 @@ go_library( "//internal/conf/conftypes", "//internal/embeddings", "//internal/embeddings/background/repo", - "//internal/embeddings/db", "//internal/embeddings/embed/client", "//internal/embeddings/embed/client/azureopenai", "//internal/embeddings/embed/client/openai", @@ -47,7 +46,6 @@ go_test( "//internal/codeintel/types", "//internal/embeddings", "//internal/embeddings/background/repo", - "//internal/embeddings/db", "//internal/embeddings/embed/client", "//internal/paths", "//internal/types", diff --git a/internal/embeddings/embed/embed.go b/internal/embeddings/embed/embed.go index 9a4aa0cbe09..d3117623ce7 100644 --- a/internal/embeddings/embed/embed.go +++ b/internal/embeddings/embed/embed.go @@ -11,7 +11,6 @@ import ( "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/embeddings" bgrepo "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo" - "github.com/sourcegraph/sourcegraph/internal/embeddings/db" "github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client" "github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client/azureopenai" "github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client/openai" @@ -41,7 +40,6 @@ func NewEmbeddingsClient(config *conftypes.EmbeddingsConfig) (client.EmbeddingsC func EmbedRepo( ctx context.Context, client client.EmbeddingsClient, - inserter db.VectorInserter, contextService ContextService, readLister FileReadLister, repo types.RepoIDName, @@ -106,13 +104,6 @@ func EmbedRepo( IsIncremental: isIncremental, } - insertDB := func(batch []embeddings.RepoEmbeddingRowMetadata, embeddings []float32, isCode bool) error { - return inserter.InsertChunks(ctx, db.InsertParams{ - ModelID: client.GetModelIdentifier(), - ChunkPoints: batchToChunkPoints(repo, opts.Revision, batch, embeddings, isCode), - }) - } - insertIndex := func(index *embeddings.EmbeddingIndex, metadata []embeddings.RepoEmbeddingRowMetadata, vectors []float32) { index.RowMetadata = append(index.RowMetadata, metadata...) index.Embeddings = append(index.Embeddings, embeddings.Quantize(vectors, nil)...) @@ -125,9 +116,8 @@ func EmbedRepo( } codeIndex := newIndex(len(codeFileNames)) - insertCode := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) error { + insertCode := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) { insertIndex(&codeIndex, md, embeddings) - return insertDB(md, embeddings, true) } reportCodeProgress := func(codeIndexStats bgrepo.EmbedFilesStats) { @@ -150,9 +140,8 @@ func EmbedRepo( stats.CodeIndexStats = codeIndexStats textIndex := newIndex(len(textFileNames)) - insertText := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) error { + insertText := func(md []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) { insertIndex(&textIndex, md, embeddings) - return insertDB(md, embeddings, false) } reportTextProgress := func(textIndexStats bgrepo.EmbedFilesStats) { @@ -214,7 +203,7 @@ type FileFilters struct { MaxFileSizeBytes int } -type batchInserter func(metadata []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) error +type batchInserter func(metadata []embeddings.RepoEmbeddingRowMetadata, embeddings []float32) type FlushResults struct { size int @@ -335,9 +324,7 @@ func embedFiles( cursor++ } - if err := insert(metadata, batchEmbeddings.Embeddings[:cursor*dimensions]); err != nil { - return nil, err - } + insert(metadata, batchEmbeddings.Embeddings[:cursor*dimensions]) batch = batch[:0] // reset batch reportProgress(stats) @@ -418,29 +405,6 @@ func embedFiles( return stats, nil } -func batchToChunkPoints(repo types.RepoIDName, revision api.CommitID, batch []embeddings.RepoEmbeddingRowMetadata, embeddings []float32, isCode bool) []db.ChunkPoint { - if len(batch) == 0 { - return nil - } - - dimensions := len(embeddings) / len(batch) - points := make([]db.ChunkPoint, 0, len(batch)) - for i, chunk := range batch { - payload := db.ChunkPayload{ - RepoName: repo.Name, - RepoID: repo.ID, - Revision: revision, - FilePath: chunk.FileName, - StartLine: uint32(chunk.StartLine), - EndLine: uint32(chunk.EndLine), - IsCode: isCode, - } - point := db.NewChunkPoint(payload, embeddings[i*dimensions:(i+1)*dimensions]) - points = append(points, point) - } - return points -} - type FileReadLister interface { FileReader FileLister diff --git a/internal/embeddings/embed/embed_test.go b/internal/embeddings/embed/embed_test.go index fe328c56c0a..a8a0e9ac199 100644 --- a/internal/embeddings/embed/embed_test.go +++ b/internal/embeddings/embed/embed_test.go @@ -17,7 +17,6 @@ import ( citypes "github.com/sourcegraph/sourcegraph/internal/codeintel/types" "github.com/sourcegraph/sourcegraph/internal/embeddings" bgrepo "github.com/sourcegraph/sourcegraph/internal/embeddings/background/repo" - "github.com/sourcegraph/sourcegraph/internal/embeddings/db" "github.com/sourcegraph/sourcegraph/internal/embeddings/embed/client" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -39,7 +38,6 @@ func TestEmbedRepo(t *testing.T) { } revision := api.CommitID("deadbeef") embeddingsClient := NewMockEmbeddingsClient() - inserter := db.NewNoopDB() contextService := NewMockContextService() contextService.SplitIntoEmbeddableChunksFunc.SetDefaultHook(defaultSplitter) splitOptions := codeintelContext.SplitOptions{ChunkTokensThreshold: 8} @@ -150,7 +148,7 @@ func TestEmbedRepo(t *testing.T) { noopReport := func(*bgrepo.EmbedRepoStats) {} t.Run("no files", func(t *testing.T) { - index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, newReadLister(), repoIDName, mockRepoPathRanks, opts, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, newReadLister(), repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) require.Len(t, index.CodeIndex.Embeddings, 0) require.Len(t, index.TextIndex.Embeddings, 0) @@ -167,7 +165,7 @@ func TestEmbedRepo(t *testing.T) { }) t.Run("code files only", func(t *testing.T) { - index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, newReadLister("a.go"), repoIDName, mockRepoPathRanks, opts, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, newReadLister("a.go"), repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) require.Len(t, index.TextIndex.Embeddings, 0) require.Len(t, index.CodeIndex.Embeddings, 6) @@ -191,7 +189,7 @@ func TestEmbedRepo(t *testing.T) { }) t.Run("text files only", func(t *testing.T) { - index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, newReadLister("b.md"), repoIDName, mockRepoPathRanks, opts, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, newReadLister("b.md"), repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) require.Len(t, index.CodeIndex.Embeddings, 0) require.Len(t, index.TextIndex.Embeddings, 6) @@ -216,7 +214,7 @@ func TestEmbedRepo(t *testing.T) { t.Run("mixed code and text files", func(t *testing.T) { rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin") - index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) require.Len(t, index.CodeIndex.Embeddings, 15) require.Len(t, index.CodeIndex.RowMetadata, 5) @@ -252,7 +250,7 @@ func TestEmbedRepo(t *testing.T) { t.Run("not included files", func(t *testing.T) { rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin", "not_included.jl") - index, _, stats, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) require.Len(t, index.CodeIndex.Embeddings, 15) require.Len(t, index.CodeIndex.RowMetadata, 5) @@ -297,7 +295,7 @@ func TestEmbedRepo(t *testing.T) { countingReporter := func(*bgrepo.EmbedRepoStats) { statReports++ } - _, _, _, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, countingReporter) + _, _, _, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, countingReporter) require.NoError(t, err) require.Equal(t, 2, statReports, ` Expected one update for flush. This is subject to change if the @@ -312,7 +310,7 @@ func TestEmbedRepo(t *testing.T) { optsCopy.MaxTextEmbeddings = 1 rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin") - index, _, _, err := EmbedRepo(ctx, embeddingsClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + index, _, _, err := EmbedRepo(ctx, embeddingsClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.NoError(t, err) // a.md has 2 chunks, c.java has 3 chunks @@ -332,19 +330,19 @@ func TestEmbedRepo(t *testing.T) { rl := newReadLister("a.go", "b.md", "c.java", "autogen.py", "empty.rb", "lines_too_long.c", "binary.bin") misbehavingClient := &misbehavingEmbeddingsClient{embeddingsClient, 32} // too many dimensions - _, _, _, err := EmbedRepo(ctx, misbehavingClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + _, _, _, err := EmbedRepo(ctx, misbehavingClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.ErrorContains(t, err, "expected embeddings for batch to have length") misbehavingClient = &misbehavingEmbeddingsClient{embeddingsClient, 32} // too few dimensions - _, _, _, err = EmbedRepo(ctx, misbehavingClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + _, _, _, err = EmbedRepo(ctx, misbehavingClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.ErrorContains(t, err, "expected embeddings for batch to have length") misbehavingClient = &misbehavingEmbeddingsClient{embeddingsClient, 0} // empty return - _, _, _, err = EmbedRepo(ctx, misbehavingClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + _, _, _, err = EmbedRepo(ctx, misbehavingClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.ErrorContains(t, err, "expected embeddings for batch to have length") erroringClient := &erroringEmbeddingsClient{embeddingsClient, errors.New("whoops")} // normal error - _, _, _, err = EmbedRepo(ctx, erroringClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + _, _, _, err = EmbedRepo(ctx, erroringClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.ErrorContains(t, err, "whoops") }) @@ -358,7 +356,7 @@ func TestEmbedRepo(t *testing.T) { failed[1] = struct{}{} partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed} - _, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + _, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.ErrorContains(t, err, "batch failed on file") require.ErrorContains(t, err, "a.go", "for a chunk error, the error message should contain the file name") @@ -374,7 +372,7 @@ func TestEmbedRepo(t *testing.T) { failed[3] = struct{}{} partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed} - _, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + _, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.ErrorContains(t, err, "batch failed on file") require.ErrorContains(t, err, "b.md", "for a chunk error, the error message should contain the file name") @@ -388,7 +386,6 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) { repoIDName := types.RepoIDName{Name: repoName} embeddingsClient := NewMockEmbeddingsClient() contextService := NewMockContextService() - inserter := db.NewNoopDB() contextService.SplitIntoEmbeddableChunksFunc.SetDefaultHook(defaultSplitter) splitOptions := codeintelContext.SplitOptions{ChunkTokensThreshold: 8} mockFiles := map[string][]byte{ @@ -478,7 +475,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) { remainingFailures: 1, err: errors.New("FAIL"), } - _, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) + _, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) require.True(t, stats.CodeIndexStats.ChunksEmbedded > 0) }) @@ -493,7 +490,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) { remainingFailures: 100, err: errors.New("FAIL"), } - _, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) + _, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.Error(t, err) }) @@ -507,7 +504,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) { remainingFailures: 1, err: client.NewRateLimitExceededError(time.Now().Add(time.Minute)), } - _, _, _, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) + _, _, _, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.Error(t, err) }) @@ -522,7 +519,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) { failed[7] = struct{}{} partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed} - index, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) @@ -572,7 +569,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) { failed[8] = struct{}{} partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed} - index, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, opts, logger, noopReport) require.NoError(t, err) @@ -624,7 +621,7 @@ func TestEmbedRepo_ExcludeChunkOnError(t *testing.T) { failed[8] = struct{}{} partialFailureClient := &partialFailureEmbeddingsClient{embeddingsClient, 0, failed} - index, _, stats, err := EmbedRepo(ctx, partialFailureClient, inserter, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) + index, _, stats, err := EmbedRepo(ctx, partialFailureClient, contextService, rl, repoIDName, mockRepoPathRanks, optsCopy, logger, noopReport) require.NoError(t, err) diff --git a/schema/schema.go b/schema/schema.go index 90ea170803a..9c3bbc4b408 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -795,8 +795,6 @@ type Embeddings struct { PolicyRepositoryMatchLimit *int `json:"policyRepositoryMatchLimit,omitempty"` // Provider description: The provider to use for generating embeddings. Defaults to sourcegraph. Provider string `json:"provider,omitempty"` - // Qdrant description: Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team. - Qdrant *Qdrant `json:"qdrant,omitempty"` // Url description: The url to the external embedding API service. Deprecated, use endpoint instead. Url string `json:"url,omitempty"` } @@ -1534,20 +1532,6 @@ type Header struct { Value string `json:"value"` } -// Hnsw description: Overrides for the HNSW index config. -type Hnsw struct { - // EfConstruct description: Number of neighbours to consider during the index building. Larger the value, more accurate the search, more time required to build the index. - EfConstruct *int `json:"efConstruct,omitempty"` - // FullScanThreshold description: Minimal size (in KiloBytes) of vectors for additional payload-based indexing. - FullScanThreshold *int `json:"fullScanThreshold,omitempty"` - // M description: Number of edges per node in the index graph. Larger the value - more accurate the search, more space required. - M *int `json:"m,omitempty"` - // OnDisk description: Store HNSW index on disk. - OnDisk *bool `json:"onDisk,omitempty"` - // PayloadM description: Number of edges per node in the index graph. Larger the value, more accurate the search, more space required. - PayloadM *int `json:"payloadM,omitempty"` -} - // IdentityProvider description: The source of identity to use when computing permissions. This defines how to compute the GitLab identity to use for a given Sourcegraph user. type IdentityProvider struct { Oauth *OAuthIdentity @@ -1947,12 +1931,6 @@ type OpenTelemetry struct { // Endpoint description: OpenTelemetry tracing collector endpoint. By default, Sourcegraph's "/-/debug/otlp" endpoint forwards data to the configured collector backend. Endpoint string `json:"endpoint,omitempty"` } -type Optimizers struct { - // IndexingThreshold description: Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing. Set to 0 to disable indexing - IndexingThreshold *int `json:"indexingThreshold,omitempty"` - // MemmapThreshold description: Maximum size (in kilobytes) of vectors to store in-memory per segment. - MemmapThreshold *int `json:"memmapThreshold,omitempty"` -} // OrganizationInvitations description: Configuration for organization invitations. type OrganizationInvitations struct { @@ -2113,24 +2091,6 @@ type PythonRateLimit struct { // RequestsPerHour description: Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1. RequestsPerHour float64 `json:"requestsPerHour"` } - -// Qdrant description: Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team. -type Qdrant struct { - Enabled bool `json:"enabled,omitempty"` - // Hnsw description: Overrides for the HNSW index config. - Hnsw *Hnsw `json:"hnsw,omitempty"` - Optimizers *Optimizers `json:"optimizers,omitempty"` - // Quantization description: Overrides for quantization config. - Quantization *Quantization `json:"quantization,omitempty"` -} - -// Quantization description: Overrides for quantization config. -type Quantization struct { - // Enabled description: Whether to enable int8 scalar quantization - Enabled *bool `json:"enabled,omitempty"` - // Quantile description: Any values that lie outside the quantile range will be truncated - Quantile *float64 `json:"quantile,omitempty"` -} type QuickLink struct { // Description description: A description for this quick link Description string `json:"description,omitempty"` diff --git a/schema/site.schema.json b/schema/site.schema.json index b13b5ce3561..ba7a9843e09 100644 --- a/schema/site.schema.json +++ b/schema/site.schema.json @@ -2762,116 +2762,6 @@ "pointer": true }, "default": true - }, - "qdrant": { - "description": "Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team.", - "type": "object", - "$comment": "Skipped fields from HnswConfigDiff: MaxIndexingThreads", - "properties": { - "enabled": { - "type": "boolean", - "default": false - }, - "hnsw": { - "description": "Overrides for the HNSW index config.", - "type": "object", - "properties": { - "m": { - "description": "Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.", - "type": "integer", - "minimum": 0, - "!go": { - "pointer": true - }, - "default": 8 - }, - "efConstruct": { - "description": "Number of neighbours to consider during the index building. Larger the value, more accurate the search, more time required to build the index.", - "type": "integer", - "minimum": 0, - "!go": { - "pointer": true - }, - "default": 100 - }, - "fullScanThreshold": { - "description": "Minimal size (in KiloBytes) of vectors for additional payload-based indexing.", - "type": "integer", - "minimum": 0, - "!go": { - "pointer": true - }, - "default": 1000 - }, - "onDisk": { - "description": "Store HNSW index on disk.", - "type": "boolean", - "!go": { - "pointer": true - }, - "default": true - }, - "payloadM": { - "description": "Number of edges per node in the index graph. Larger the value, more accurate the search, more space required.", - "type": "integer", - "minimum": 0, - "!go": { - "pointer": true - }, - "default": 8 - } - } - }, - "quantization": { - "description": "Overrides for quantization config.", - "type": "object", - "$comment": "Skipped fields from ScalarQuantization: AlwaysRam", - "properties": { - "enabled": { - "description": "Whether to enable int8 scalar quantization", - "type": "boolean", - "!go": { - "pointer": true - }, - "default": true - }, - "quantile": { - "description": "Any values that lie outside the quantile range will be truncated", - "type": "number", - "minimum": 0, - "maximum": 1, - "!go": { - "pointer": true - }, - "default": 0.98 - } - } - }, - "optimizers": { - "type": "object", - "$comment": "Skipped fields from OptimizersConfigDiff: DeletedThreshold, VacuumMinVectorNumber, DefaultSegmentNumber, MaxSegmentSize, FlushIntervalSec, MaxOptimizationThreads", - "properties": { - "indexingThreshold": { - "description": "Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing. Set to 0 to disable indexing", - "type": "integer", - "minimum": 0, - "!go": { - "pointer": true - }, - "default": 0 - }, - "memmapThreshold": { - "description": "Maximum size (in kilobytes) of vectors to store in-memory per segment.", - "type": "integer", - "minimum": 0, - "!go": { - "pointer": true - }, - "default": 100 - } - } - } - } } }, "examples": [ diff --git a/sg.config.yaml b/sg.config.yaml index f54f9b52139..5463590f7bd 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -273,18 +273,6 @@ commands: - cmd/embeddings - internal/embeddings - qdrant: - cmd: | - docker run -p 6333:6333 -p 6334:6334 \ - -v $HOME/.sourcegraph-dev/data/qdrant_data:/data \ - -e QDRANT__SERVICE__GRPC_PORT="6334" \ - -e QDRANT__LOG_LEVEL=INFO \ - -e QDRANT__STORAGE__STORAGE_PATH=/data \ - -e QDRANT__STORAGE__SNAPSHOTS_PATH=/data \ - -e QDRANT_INIT_FILE_PATH=/data/.qdrant-initialized \ - --entrypoint /usr/local/bin/qdrant \ - sourcegraph/qdrant:insiders - worker: cmd: | export SOURCEGRAPH_LICENSE_GENERATION_KEY=$(cat ../dev-private/enterprise/dev/test-license-generation-key.pem) @@ -1250,22 +1238,6 @@ dockerCommands: WORKERS: 1 ROCKET_ADDRESS: 0.0.0.0 - qdrant: - docker: - image: sourcegraph/qdrant:insiders - ports: - - 6333 - - 6334 - volumes: - - from: $HOME/.sourcegraph-dev/data/qdrant_data - to: /data - env: - QDRANT__SERVICE__GRPC_PORT: 6334 - QDRANT__LOG_LEVEL: INFO - QDRANT__STORAGE__STORAGE_PATH: /data - QDRANT__STORAGE__SNAPSHOTS_PATH: /data - QDRANT_INIT_FILE_PATH: /data/.qdrant-initialized - # # CommandSets ################################################################ # @@ -1748,29 +1720,6 @@ commandsets: bazelCommands: - cody-gateway - qdrant: - commands: - - qdrant - - frontend - - worker - - repo-updater - - web - - gitserver-0 - - gitserver-1 - - searcher - - caddy - - symbols - - docsite - - syntax-highlighter - - zoekt-index-0 - - zoekt-index-1 - - zoekt-web-0 - - zoekt-web-1 - - blobstore - - embeddings - env: - QDRANT_ENDPOINT: 'localhost:6334' - enterprise-bazel-sveltekit: <<: *enterprise_bazel_set env: diff --git a/third-party-licenses/ThirdPartyLicenses.csv b/third-party-licenses/ThirdPartyLicenses.csv index d5031a03a17..9d055c9092e 100644 --- a/third-party-licenses/ThirdPartyLicenses.csv +++ b/third-party-licenses/ThirdPartyLicenses.csv @@ -1514,7 +1514,6 @@ Go,github.com/prometheus/common,v0.32.1,"Apache 2.0,New BSD",github.com/promethe Go,github.com/prometheus/common/sigv4,v0.1.0,Apache 2.0,github.com/prometheus/common,Approved Go,github.com/prometheus/procfs,v0.11.1,Apache 2.0,github.com/prometheus/procfs,Approved Go,github.com/prometheus/prometheus,v0.40.5,Apache 2.0,github.com/prometheus/prometheus,Approved -Go,github.com/qdrant/go-client,v1.4.1,Apache 2.0,github.com/qdrant/go-client,Approved Go,github.com/qustavo/sqlhooks/v2,v2.1.0,MIT,github.com/qustavo/sqlhooks,Approved Go,github.com/rickb777/date,v1.14.3,New BSD,github.com/rickb777/date,Approved Go,github.com/rickb777/plural,v1.2.2,New BSD,github.com/rickb777/plural,Approved diff --git a/wolfi-images/qdrant.lock.json b/wolfi-images/qdrant.lock.json deleted file mode 100755 index 4612eb56599..00000000000 --- a/wolfi-images/qdrant.lock.json +++ /dev/null @@ -1,828 +0,0 @@ -{ - "configHash": "f5dfaea80b9aa5f34f3aa22bfa12474efd99bdde5587dbff22187b5731e4b5ae", - "contents": { - "keyring": [ - { - "name": "packages.wolfi.dev/os/wolfi-signing.rsa.pub", - "url": "https://packages.wolfi.dev/os/wolfi-signing.rsa.pub" - }, - { - "name": "packages.sgdev.org/sourcegraph-melange-prod.rsa.pub", - "url": "https://packages.sgdev.org/sourcegraph-melange-prod.rsa.pub" - } - ], - "packages": [ - { - "architecture": "x86_64", - "checksum": "Q1YQmPfQ1Ym4tfjrCMChbESrrRg/o=", - "control": { - "checksum": "sha1-YQmPfQ1Ym4tfjrCMChbESrrRg/o=", - "range": "bytes=696-1032" - }, - "data": { - "checksum": "sha256-5hhCQURRrKVfPk8TOZVxfjceIUkVE0fh3/vEJBk88Ps=", - "range": "bytes=1033-256258" - }, - "name": "ca-certificates-bundle", - "signature": { - "checksum": "sha1-E1NIpx8yCH6x5GcSqB4MzKQxuq4=", - "range": "bytes=0-695" - }, - "url": "https://packages.wolfi.dev/os/x86_64/ca-certificates-bundle-20240315-r0.apk", - "version": "20240315-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1OHhyuiUviNHTg939DA0lyeRee18=", - "control": { - "checksum": "sha1-OHhyuiUviNHTg939DA0lyeRee18=", - "range": "bytes=702-1052" - }, - "data": { - "checksum": "sha256-om3EZzEM+3dD9a77sOB2uOuAKBlf7XoUW/ORnDHQvZY=", - "range": "bytes=1053-125427" - }, - "name": "wolfi-baselayout", - "signature": { - "checksum": "sha1-1CcRiULOFhX8ldA/Ae2qCMUGNmQ=", - "range": "bytes=0-701" - }, - "url": "https://packages.wolfi.dev/os/x86_64/wolfi-baselayout-20230201-r7.apk", - "version": "20230201-r7" - }, - { - "architecture": "x86_64", - "checksum": "Q1uuAznXPkDNBMP/eILVO7UDGj1pU=", - "control": { - "checksum": "sha1-uuAznXPkDNBMP/eILVO7UDGj1pU=", - "range": "bytes=702-1112" - }, - "data": { - "checksum": "sha256-Ath31YTTsiakoEktGl5txvNQpnr/grvFQHlmXa5E7fI=", - "range": "bytes=1113-267815" - }, - "name": "ld-linux", - "signature": { - "checksum": "sha1-oh4vr00LXL4ArbAOR9LS1VzzfYc=", - "range": "bytes=0-701" - }, - "url": "https://packages.wolfi.dev/os/x86_64/ld-linux-2.39-r2.apk", - "version": "2.39-r2" - }, - { - "architecture": "x86_64", - "checksum": "Q1TXuyiDJBsTNDwvPq7HFxaeBQ33w=", - "control": { - "checksum": "sha1-TXuyiDJBsTNDwvPq7HFxaeBQ33w=", - "range": "bytes=703-1059" - }, - "data": { - "checksum": "sha256-3s2Fygt+ZuHj/StxP7YffswmhHE7EJ4TxZ7EWlRQ4NA=", - "range": "bytes=1060-408275" - }, - "name": "glibc-locale-posix", - "signature": { - "checksum": "sha1-FbeSNtuEgFmzNamDx5Dj1LGVgsQ=", - "range": "bytes=0-702" - }, - "url": "https://packages.wolfi.dev/os/x86_64/glibc-locale-posix-2.39-r2.apk", - "version": "2.39-r2" - }, - { - "architecture": "x86_64", - "checksum": "Q1SD8az8RMNr5tnb8/mPZdR+A6V5k=", - "control": { - "checksum": "sha1-SD8az8RMNr5tnb8/mPZdR+A6V5k=", - "range": "bytes=701-1330" - }, - "data": { - "checksum": "sha256-QvhWs/fGBaX5calu0uES9fcTMx6+R1xKZHdxkxxSxsg=", - "range": "bytes=1331-5861481" - }, - "name": "glibc", - "signature": { - "checksum": "sha1-v4LRB2eP5VLe623v8sJ3f4wDNFM=", - "range": "bytes=0-700" - }, - "url": "https://packages.wolfi.dev/os/x86_64/glibc-2.39-r2.apk", - "version": "2.39-r2" - }, - { - "architecture": "x86_64", - "checksum": "Q1mVgCtcYDHEkUa+8x41i7w9cQ4Qg=", - "control": { - "checksum": "sha1-mVgCtcYDHEkUa+8x41i7w9cQ4Qg=", - "range": "bytes=704-1079" - }, - "data": { - "checksum": "sha256-xE4spLqr7qIgImPfMC1kMY1a7Xu7xu1/eLkBMgOzSSc=", - "range": "bytes=1080-77936" - }, - "name": "protobuf-c", - "signature": { - "checksum": "sha1-NGL0ELlBK8mhhAzuOkm17d2LbRo=", - "range": "bytes=0-703" - }, - "url": "https://packages.wolfi.dev/os/x86_64/protobuf-c-1.5.0-r3.apk", - "version": "1.5.0-r3" - }, - { - "architecture": "x86_64", - "checksum": "Q135v8eEv8ZI/s/HXKkE96INoEoJk=", - "control": { - "checksum": "sha1-35v8eEv8ZI/s/HXKkE96INoEoJk=", - "range": "bytes=704-1040" - }, - "data": { - "checksum": "sha256-3y4Tb3jy8M/ZXhnY6jxwj2YQFhdjLr/kjXhqJX7I+is=", - "range": "bytes=1041-27155" - }, - "name": "krb5-conf", - "signature": { - "checksum": "sha1-WWjewHF5gekYrUPqdUHQEPIc97M=", - "range": "bytes=0-703" - }, - "url": "https://packages.wolfi.dev/os/x86_64/krb5-conf-1.0-r1.apk", - "version": "1.0-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q1TAnNLCVTaCylmbN84TlliyM47qM=", - "control": { - "checksum": "sha1-TAnNLCVTaCylmbN84TlliyM47qM=", - "range": "bytes=704-1069" - }, - "data": { - "checksum": "sha256-+6lzaltti6IVf7RNynwcL9LmP9cJKMOjONPFUhHtnsA=", - "range": "bytes=1070-57492" - }, - "name": "keyutils-libs", - "signature": { - "checksum": "sha1-MY+r6+HKyYcrgJtVqyG50KdP+QY=", - "range": "bytes=0-703" - }, - "url": "https://packages.wolfi.dev/os/x86_64/keyutils-libs-1.6.3-r1.apk", - "version": "1.6.3-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q1baVeVcWh84uv5G9/ZuxJCTg06uQ=", - "control": { - "checksum": "sha1-baVeVcWh84uv5G9/ZuxJCTg06uQ=", - "range": "bytes=700-1058" - }, - "data": { - "checksum": "sha256-9BtoieN8gK8o4nzCDUyWkp6RmEsXrOwdu/XeTdZtDSE=", - "range": "bytes=1059-61938" - }, - "name": "libverto", - "signature": { - "checksum": "sha1-TVkKg7JApxa7nvaj9DNvOsTv3Io=", - "range": "bytes=0-699" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libverto-0.3.2-r1.apk", - "version": "0.3.2-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q1M+CfES+G7micWuVpGjyxcTOj9zQ=", - "control": { - "checksum": "sha1-M+CfES+G7micWuVpGjyxcTOj9zQ=", - "range": "bytes=702-1120" - }, - "data": { - "checksum": "sha256-UqwHAn95sL7AsIRMN0DM1TtSJ2Q5KD1WN4+uU8+Knqg=", - "range": "bytes=1121-52564" - }, - "name": "libcom_err", - "signature": { - "checksum": "sha1-qciIi1MZRLclhn8K+GnrOJlNNfE=", - "range": "bytes=0-701" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libcom_err-1.47.0-r1.apk", - "version": "1.47.0-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q1+rBEmKEIywe0o/a65ZPjWoUECRQ=", - "control": { - "checksum": "sha1-+rBEmKEIywe0o/a65ZPjWoUECRQ=", - "range": "bytes=703-1054" - }, - "data": { - "checksum": "sha256-fcShKjtj6wR6kgjwToU1J8hU+AGRf1OGZt9jeXJdM0Q=", - "range": "bytes=1055-82406" - }, - "name": "openssl-config", - "signature": { - "checksum": "sha1-RT/o4gi2oC/l+6IhikP9/PH5awg=", - "range": "bytes=0-702" - }, - "url": "https://packages.wolfi.dev/os/x86_64/openssl-config-3.3.0-r5.apk", - "version": "3.3.0-r5" - }, - { - "architecture": "x86_64", - "checksum": "Q11DkfNRzraQMj7ue8LFH6VN6d7Bo=", - "control": { - "checksum": "sha1-1DkfNRzraQMj7ue8LFH6VN6d7Bo=", - "range": "bytes=707-1092" - }, - "data": { - "checksum": "sha256-c95GJNThldUhfxVSGbchUI0pS4j6h45Yx6JDPXHaAI0=", - "range": "bytes=1093-5915048" - }, - "name": "libcrypto3", - "signature": { - "checksum": "sha1-EmjggcTaE4QTKI1FzSom0rXam1c=", - "range": "bytes=0-706" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libcrypto3-3.3.0-r5.apk", - "version": "3.3.0-r5" - }, - { - "architecture": "x86_64", - "checksum": "Q15sT5bGF6kgE7DepzZIHCpPpuP70=", - "control": { - "checksum": "sha1-5sT5bGF6kgE7DepzZIHCpPpuP70=", - "range": "bytes=702-1085" - }, - "data": { - "checksum": "sha256-fWOAfZYgXO6+yM4Dpo/y7dvV3uxSEhhfO4SUX79TQDw=", - "range": "bytes=1086-1196497" - }, - "name": "libssl3", - "signature": { - "checksum": "sha1-4+7eITv9weTmoYBAAyd92WsXS24=", - "range": "bytes=0-701" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libssl3-3.3.0-r5.apk", - "version": "3.3.0-r5" - }, - { - "architecture": "x86_64", - "checksum": "Q1myu18Yt+0lLtpxOyrQ3LKnV6SoI=", - "control": { - "checksum": "sha1-myu18Yt+0lLtpxOyrQ3LKnV6SoI=", - "range": "bytes=698-1216" - }, - "data": { - "checksum": "sha256-elcsQDiEgNifO11g4MoJLGo5XJu8wQZaytjIKYxzVjw=", - "range": "bytes=1217-2564165" - }, - "name": "krb5-libs", - "signature": { - "checksum": "sha1-NdbRDYVgwMQYQQrqt9GSGuBkjuA=", - "range": "bytes=0-697" - }, - "url": "https://packages.wolfi.dev/os/x86_64/krb5-libs-1.21.2-r1.apk", - "version": "1.21.2-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q1BMcnbxIFejKxlfYLJaX4uo/+X0M=", - "control": { - "checksum": "sha1-BMcnbxIFejKxlfYLJaX4uo/+X0M=", - "range": "bytes=699-1059" - }, - "data": { - "checksum": "sha256-Rp339LV5T4+zr98USK12eoyKGF84NviKlq3jh1AvbQQ=", - "range": "bytes=1060-96440" - }, - "name": "fstrm", - "signature": { - "checksum": "sha1-tOY0x5olyX6V9uRtlgxtf4WEhR0=", - "range": "bytes=0-698" - }, - "url": "https://packages.wolfi.dev/os/x86_64/fstrm-0.6.1-r1.apk", - "version": "0.6.1-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q126C/3voUst+sFakQOGVOKucs6v8=", - "control": { - "checksum": "sha1-26C/3voUst+sFakQOGVOKucs6v8=", - "range": "bytes=698-1076" - }, - "data": { - "checksum": "sha256-S2VfC729Glys7iRmR7sX4RoS4LLyNi9KmSzDc8wkKrQ=", - "range": "bytes=1077-277291" - }, - "name": "libuv", - "signature": { - "checksum": "sha1-23pSGkhyhlBtbVxNtDXYlWnt+vk=", - "range": "bytes=0-697" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libuv-1.48.0-r0.apk", - "version": "1.48.0-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1ybud27/W+hJ0asiUj3hfrXVjcMs=", - "control": { - "checksum": "sha1-ybud27/W+hJ0asiUj3hfrXVjcMs=", - "range": "bytes=698-1081" - }, - "data": { - "checksum": "sha256-3uFtBCAT2Gu/AplcYbKYMCuMMC94JLJDNyoWnSMLqjc=", - "range": "bytes=1082-156680" - }, - "name": "zlib", - "signature": { - "checksum": "sha1-JYfhgb71ZjFG0VIAKLwOQSlHmlg=", - "range": "bytes=0-697" - }, - "url": "https://packages.wolfi.dev/os/x86_64/zlib-1.3.1-r0.apk", - "version": "1.3.1-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1sGD1MEAazAXpfNixX74EpwafGo0=", - "control": { - "checksum": "sha1-sGD1MEAazAXpfNixX74EpwafGo0=", - "range": "bytes=698-1059" - }, - "data": { - "checksum": "sha256-s+9m0l/R7hVKQ5YLbU90D2E6DPC2QVTxdjhToMi5iYU=", - "range": "bytes=1060-251831" - }, - "name": "libnghttp2-14", - "signature": { - "checksum": "sha1-wOIYtvU72n76UGvbargK2ccZx0E=", - "range": "bytes=0-697" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libnghttp2-14-1.61.0-r0.apk", - "version": "1.61.0-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1cCNlvL7pVXPEgMJQz7uLXVW0w5g=", - "control": { - "checksum": "sha1-cCNlvL7pVXPEgMJQz7uLXVW0w5g=", - "range": "bytes=697-1073" - }, - "data": { - "checksum": "sha256-7m+RnrAOS8ZCnFW/j2P2NcQJxOzfwSjOLYhEZXIwBG8=", - "range": "bytes=1074-114096" - }, - "name": "libev", - "signature": { - "checksum": "sha1-PUsxiEtEuEoDpgKP2L36G/X55s0=", - "range": "bytes=0-696" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libev-4.33-r4.apk", - "version": "4.33-r4" - }, - { - "architecture": "x86_64", - "checksum": "Q1jyz//Wx+59L1JtSRpS5LPxe5iaM=", - "control": { - "checksum": "sha1-jyz//Wx+59L1JtSRpS5LPxe5iaM=", - "range": "bytes=708-1084" - }, - "data": { - "checksum": "sha256-v6jAIoRu7n3hQJ8nwWCLtvRsszuMGCAEyZm4zUF1cVI=", - "range": "bytes=1085-185893" - }, - "name": "libgcc", - "signature": { - "checksum": "sha1-3fVn7jRkfOtxSgpmdey4iJJqQPM=", - "range": "bytes=0-707" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libgcc-13.2.0-r5.apk", - "version": "13.2.0-r5" - }, - { - "architecture": "x86_64", - "checksum": "Q1NsUsznaiP7XyU6U/5pssXQgGJgU=", - "control": { - "checksum": "sha1-NsUsznaiP7XyU6U/5pssXQgGJgU=", - "range": "bytes=701-1093" - }, - "data": { - "checksum": "sha256-0XrQ++geRWYRUazF23zdsuGVLFkiIDM1FKmAbbz9ojQ=", - "range": "bytes=1094-3156830" - }, - "name": "libstdc++", - "signature": { - "checksum": "sha1-Zu2LUNkKQt3BnFU9+4PX0ud8D6Q=", - "range": "bytes=0-700" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libstdc++-13.2.0-r5.apk", - "version": "13.2.0-r5" - }, - { - "architecture": "x86_64", - "checksum": "Q1EHyaNLx1UadaqJXEC86gtIjZGMM=", - "control": { - "checksum": "sha1-EHyaNLx1UadaqJXEC86gtIjZGMM=", - "range": "bytes=696-1076" - }, - "data": { - "checksum": "sha256-+ValBcpIsLeYUFo73SZrUM2vpLGefts7Qx95EPMATaY=", - "range": "bytes=1077-232308" - }, - "name": "c-ares", - "signature": { - "checksum": "sha1-OtwgYMiySwl+l6divnnSgQu+QUg=", - "range": "bytes=0-695" - }, - "url": "https://packages.wolfi.dev/os/x86_64/c-ares-1.28.1-r0.apk", - "version": "1.28.1-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1NZCvjmGBZ7soi6T4endRPtDztk4=", - "control": { - "checksum": "sha1-NZCvjmGBZ7soi6T4endRPtDztk4=", - "range": "bytes=698-1162" - }, - "data": { - "checksum": "sha256-q79pSkGvnDzz1dHsQceLPqyVC+cFCyWWJB5L2EfkZRc=", - "range": "bytes=1163-2556930" - }, - "name": "nghttp2", - "signature": { - "checksum": "sha1-UuOeekThV8UC737772jFTRj0Y50=", - "range": "bytes=0-697" - }, - "url": "https://packages.wolfi.dev/os/x86_64/nghttp2-1.61.0-r0.apk", - "version": "1.61.0-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1O/sL8Kjkxzh39Ffh9gJl+0olkF4=", - "control": { - "checksum": "sha1-O/sL8Kjkxzh39Ffh9gJl+0olkF4=", - "range": "bytes=700-1061" - }, - "data": { - "checksum": "sha256-6qS8u1yYrzO+4Ypa+g4zfz3m16/7O/U4wjiRrKl8/vw=", - "range": "bytes=1062-631650" - }, - "name": "nghttp2-dev", - "signature": { - "checksum": "sha1-avAz/6snsG86C9jh8Gxa59gs29U=", - "range": "bytes=0-699" - }, - "url": "https://packages.wolfi.dev/os/x86_64/nghttp2-dev-1.61.0-r0.apk", - "version": "1.61.0-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1sJOz3InYXKj1qNN3oPm3pb30VO0=", - "control": { - "checksum": "sha1-sJOz3InYXKj1qNN3oPm3pb30VO0=", - "range": "bytes=697-1149" - }, - "data": { - "checksum": "sha256-fahwYvY6Lv9AhtEHBsSaoFAoKFWlvbbVC8jYGuRmTcg=", - "range": "bytes=1150-2380016" - }, - "name": "xz", - "signature": { - "checksum": "sha1-aQxFIS7BG8u/jlY2JU8oQIEloYw=", - "range": "bytes=0-696" - }, - "url": "https://packages.wolfi.dev/os/x86_64/xz-5.4.6-r0.apk", - "version": "5.4.6-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1aNAiDiOAPLkPMJFYq6mpzKq4V18=", - "control": { - "checksum": "sha1-aNAiDiOAPLkPMJFYq6mpzKq4V18=", - "range": "bytes=699-1084" - }, - "data": { - "checksum": "sha256-Wc0u/JyHwxC7ubVFz4UlXEc5URwWiBZm4jNKqVv9LF0=", - "range": "bytes=1085-4698210" - }, - "name": "libxml2", - "signature": { - "checksum": "sha1-v7pbNfh/TdC3LzRewdC3GeA9rec=", - "range": "bytes=0-698" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libxml2-2.12.6-r0.apk", - "version": "2.12.6-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1wJPtj5XZtnPiVWGM5dsT+nyfWXc=", - "control": { - "checksum": "sha1-wJPtj5XZtnPiVWGM5dsT+nyfWXc=", - "range": "bytes=705-1234" - }, - "data": { - "checksum": "sha256-sw+CjlpsjO01q6lqzXgHnniiJKn7bqOSNO6vNqdGc7I=", - "range": "bytes=1235-3873775" - }, - "name": "bind-libs", - "signature": { - "checksum": "sha1-9zDzmHAeu7mWpidXAatqrLZVzUs=", - "range": "bytes=0-704" - }, - "url": "https://packages.wolfi.dev/os/x86_64/bind-libs-9.18.26-r0.apk", - "version": "9.18.26-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1AxEQdi8fVcYtGc3zHTCW3OZdY2Q=", - "control": { - "checksum": "sha1-AxEQdi8fVcYtGc3zHTCW3OZdY2Q=", - "range": "bytes=694-1208" - }, - "data": { - "checksum": "sha256-/dt3qJw5D9iSJ/mKkRQkhWD8R52selhpcwXyw7KLMhg=", - "range": "bytes=1209-879444" - }, - "name": "bind-tools", - "signature": { - "checksum": "sha1-VTb0YdUzPMk8Po0p4nj0y5Jey6k=", - "range": "bytes=0-693" - }, - "url": "https://packages.wolfi.dev/os/x86_64/bind-tools-9.18.26-r0.apk", - "version": "9.18.26-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q11cs1/Vkyp8KEwqtqZvPrB+Mfb8A=", - "control": { - "checksum": "sha1-1cs1/Vkyp8KEwqtqZvPrB+Mfb8A=", - "range": "bytes=698-1093" - }, - "data": { - "checksum": "sha256-t284K9/cZQaQMy4y4nYXMIjKUTbyaBa/QnUj0cYmTNk=", - "range": "bytes=1094-234977" - }, - "name": "libxcrypt", - "signature": { - "checksum": "sha1-hhR4Puw7nMj2H9OzUMTKhK1/7N0=", - "range": "bytes=0-697" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libxcrypt-4.4.36-r4.apk", - "version": "4.4.36-r4" - }, - { - "architecture": "x86_64", - "checksum": "Q1drCieAMJpJBP1KWvJk6Vhq7Zj20=", - "control": { - "checksum": "sha1-drCieAMJpJBP1KWvJk6Vhq7Zj20=", - "range": "bytes=701-1108" - }, - "data": { - "checksum": "sha256-LcGaT+nLHN7YtUab5sY0EoXErbOj/S58FXtf1JVxWbM=", - "range": "bytes=1109-21605" - }, - "name": "libcrypt1", - "signature": { - "checksum": "sha1-wwLYQc1joetP6IOipSVSCQsZHsM=", - "range": "bytes=0-700" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libcrypt1-2.39-r2.apk", - "version": "2.39-r2" - }, - { - "architecture": "x86_64", - "checksum": "Q17FDk2/BvxV3n5UBi4rz7m8aR1Wc=", - "control": { - "checksum": "sha1-7FDk2/BvxV3n5UBi4rz7m8aR1Wc=", - "range": "bytes=701-1208" - }, - "data": { - "checksum": "sha256-67DYE+o9zQIS2KUyXkBN8SAEkWVh2+isnGTn78VdLMg=", - "range": "bytes=1209-636015" - }, - "name": "busybox", - "signature": { - "checksum": "sha1-70uMRez2BMN2clrT3wFBWsR5Gew=", - "range": "bytes=0-700" - }, - "url": "https://packages.wolfi.dev/os/x86_64/busybox-1.36.1-r7.apk", - "version": "1.36.1-r7" - }, - { - "architecture": "x86_64", - "checksum": "Q1f9ldLw5Jdbm9CkZfsDWXP2YaQWE=", - "control": { - "checksum": "sha1-f9ldLw5Jdbm9CkZfsDWXP2YaQWE=", - "range": "bytes=706-1103" - }, - "data": { - "checksum": "sha256-ttIYvsu5Vp2YYKv/C7vK1hFUI8kNsOJxD65/SsOtWvQ=", - "range": "bytes=1104-2862070" - }, - "name": "libunistring", - "signature": { - "checksum": "sha1-IwR2lnVv+Ixi8qtznw+ruuV9OVw=", - "range": "bytes=0-705" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libunistring-1.2-r0.apk", - "version": "1.2-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1tBy70+JqCVQbecHMDxN0U9PKn8k=", - "control": { - "checksum": "sha1-tBy70+JqCVQbecHMDxN0U9PKn8k=", - "range": "bytes=697-1102" - }, - "data": { - "checksum": "sha256-xFjkADRrilqBaMgedKfRkaUGOqAlLDunZnmM/nggwDo=", - "range": "bytes=1103-411419" - }, - "name": "libidn2", - "signature": { - "checksum": "sha1-AkpnAB73nCuJM4BJIXJsWn2/urk=", - "range": "bytes=0-696" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libidn2-2.3.7-r0.apk", - "version": "2.3.7-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1QobDOHNHcnrYAlkCuVI1Te8I5V0=", - "control": { - "checksum": "sha1-QobDOHNHcnrYAlkCuVI1Te8I5V0=", - "range": "bytes=702-1082" - }, - "data": { - "checksum": "sha256-cEuUD1tNXYeUHPC7dK9BSHOx8vt7IIRBGd181Sd0RXA=", - "range": "bytes=1083-114314" - }, - "name": "libpsl", - "signature": { - "checksum": "sha1-H2Bp5J4UMCXPQlfvEiFxLXG5rEY=", - "range": "bytes=0-701" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libpsl-0.21.5-r0.apk", - "version": "0.21.5-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1LcDEPPjJrWwhueUvesRr4kLyGPI=", - "control": { - "checksum": "sha1-LcDEPPjJrWwhueUvesRr4kLyGPI=", - "range": "bytes=707-1052" - }, - "data": { - "checksum": "sha256-xtIiQnSjJGdZnAzaBoJD6q3TbCpnAwTfD9SvbafkaP0=", - "range": "bytes=1053-174068" - }, - "name": "libbrotlicommon1", - "signature": { - "checksum": "sha1-kcyKhs8jgdpGEEUhbyXHPJQpvyM=", - "range": "bytes=0-706" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libbrotlicommon1-1.1.0-r1.apk", - "version": "1.1.0-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q10jZlLouHAzMUvZYphGGNjOoZ1ug=", - "control": { - "checksum": "sha1-0jZlLouHAzMUvZYphGGNjOoZ1ug=", - "range": "bytes=701-1051" - }, - "data": { - "checksum": "sha256-nx7lapEgf5fd4ZrtBxqXtAWE0tEVndBBEwXOUbhY2OA=", - "range": "bytes=1052-81979" - }, - "name": "libbrotlidec1", - "signature": { - "checksum": "sha1-iUdAt3okR6P8rJ4sNv4yjC0I8Ys=", - "range": "bytes=0-700" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libbrotlidec1-1.1.0-r1.apk", - "version": "1.1.0-r1" - }, - { - "architecture": "x86_64", - "checksum": "Q1hgEBBo1tLreSINI2qiCUpUrIE9k=", - "control": { - "checksum": "sha1-hgEBBo1tLreSINI2qiCUpUrIE9k=", - "range": "bytes=700-1139" - }, - "data": { - "checksum": "sha256-OFDWkK8l015fs0YvTCXVKLQmOxq+7W5RgJHIqKYZsTE=", - "range": "bytes=1140-808582" - }, - "name": "libcurl-openssl4", - "signature": { - "checksum": "sha1-sygIdNZXB/W8fyOrdelY5RXh3aA=", - "range": "bytes=0-699" - }, - "url": "https://packages.wolfi.dev/os/x86_64/libcurl-openssl4-8.6.0-r0.apk", - "version": "8.6.0-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1eQzLI9XN4FM69fm9B4LWd9frcwM=", - "control": { - "checksum": "sha1-eQzLI9XN4FM69fm9B4LWd9frcwM=", - "range": "bytes=703-1098" - }, - "data": { - "checksum": "sha256-zkew7+IbyyyENezhlStwO062zaWd+yP/lpbI+CCJVZQ=", - "range": "bytes=1099-281166" - }, - "name": "curl", - "signature": { - "checksum": "sha1-igXspQSxo3Eu3o8DeNrYt3YxSbY=", - "range": "bytes=0-702" - }, - "url": "https://packages.wolfi.dev/os/x86_64/curl-8.6.0-r0.apk", - "version": "8.6.0-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q19yTFZBeYlCCMWWWtfHBXivqZKfc=", - "control": { - "checksum": "sha1-9yTFZBeYlCCMWWWtfHBXivqZKfc=", - "range": "bytes=657-991" - }, - "data": { - "checksum": "sha256-2FUyZ+/OJ9o/PTteTNS/jIlOBJuw7eJrhAbxiFIbU9Q=", - "range": "bytes=992-42513766" - }, - "name": "qdrant", - "signature": { - "checksum": "sha1-ZfLA+OrzXH+JAysAXgT9IoFbiRA=", - "range": "bytes=0-656" - }, - "url": "https://packages.sgdev.org/main/x86_64/qdrant-1.5.1-r4.apk", - "version": "1.5.1-r4" - }, - { - "architecture": "x86_64", - "checksum": "Q1m7SsrH+XnrPIapzhnK0vkoxMen4=", - "control": { - "checksum": "sha1-m7SsrH+XnrPIapzhnK0vkoxMen4=", - "range": "bytes=698-1077" - }, - "data": { - "checksum": "sha256-0WaULqZyE0zNNC6D7YeaXVN4shAVEouyqAEiRlB09C0=", - "range": "bytes=1078-54479" - }, - "name": "tini", - "signature": { - "checksum": "sha1-eomfhWCyCB1OA/2DHboLUzXunvA=", - "range": "bytes=0-697" - }, - "url": "https://packages.wolfi.dev/os/x86_64/tini-0.19.0-r3.apk", - "version": "0.19.0-r3" - }, - { - "architecture": "x86_64", - "checksum": "Q1lvoD8CoCRqulbibjC3gSGuEe8K0=", - "control": { - "checksum": "sha1-lvoD8CoCRqulbibjC3gSGuEe8K0=", - "range": "bytes=702-1035" - }, - "data": { - "checksum": "sha256-YB3jK9thvtrl7FCbwBPhBgHnz6ncykVxex/dHC4wYc8=", - "range": "bytes=1036-3022935" - }, - "name": "tzdata", - "signature": { - "checksum": "sha1-AJ7WqmxNNMiSUQ8WuwjXg5Y23Gw=", - "range": "bytes=0-701" - }, - "url": "https://packages.wolfi.dev/os/x86_64/tzdata-2024a-r0.apk", - "version": "2024a-r0" - }, - { - "architecture": "x86_64", - "checksum": "Q1JVK/s+yGDiDotAujuHcQwzrchvI=", - "control": { - "checksum": "sha1-JVK/s+yGDiDotAujuHcQwzrchvI=", - "range": "bytes=699-1099" - }, - "data": { - "checksum": "sha256-6MWjAN767fVREf1xZ18eV4QLzKBUdZusnhj7ljPZuhU=", - "range": "bytes=1100-786256" - }, - "name": "wget", - "signature": { - "checksum": "sha1-cJ/swsh0XVO9ISogqXDo8oBBG7M=", - "range": "bytes=0-698" - }, - "url": "https://packages.wolfi.dev/os/x86_64/wget-1.24.5-r0.apk", - "version": "1.24.5-r0" - } - ], - "repositories": [ - { - "architecture": "x86_64", - "name": "packages.wolfi.dev/os/x86_64", - "url": "https://packages.wolfi.dev/os/x86_64/APKINDEX.tar.gz" - }, - { - "architecture": "x86_64", - "name": "@sourcegraph https://packages.sgdev.org/main/x86_64", - "url": "@sourcegraph https://packages.sgdev.org/main/x86_64/APKINDEX.tar.gz" - } - ] - }, - "version": "v1" -} \ No newline at end of file diff --git a/wolfi-images/qdrant.yaml b/wolfi-images/qdrant.yaml deleted file mode 100644 index c8ef65c2acf..00000000000 --- a/wolfi-images/qdrant.yaml +++ /dev/null @@ -1,20 +0,0 @@ -include: ./sourcegraph-template.yaml - -contents: - packages: - # Included by existing SG base image - - tini - - - qdrant@sourcegraph - -paths: - - path: /data - type: directory - uid: 100 - gid: 101 - permissions: 0o755 - -entrypoint: - command: /sbin/tini -- /usr/local/bin/qdrant --config-path /etc/qdrant/config.yaml - -# MANUAL REBUILD: diff --git a/wolfi-packages/qdrant.yaml b/wolfi-packages/qdrant.yaml deleted file mode 100644 index 3deeb74a4a3..00000000000 --- a/wolfi-packages/qdrant.yaml +++ /dev/null @@ -1,41 +0,0 @@ -package: - name: qdrant - version: 1.5.1 - epoch: 4 - description: "Qdrant vector database" - target-architecture: - - x86_64 - copyright: - - paths: - - "*" - license: 'Apache 2.0' - dependencies: - runtime: - -environment: - contents: - repositories: - - https://packages.wolfi.dev/os - keyring: - - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub - packages: - - wolfi-base - - busybox - - ca-certificates-bundle - -pipeline: - - uses: fetch - with: - uri: https://github.com/qdrant/qdrant/releases/download/v${{package.version}}/qdrant-x86_64-unknown-linux-gnu.tar.gz - expected-sha256: 0d8b385590c1f1145f6114cde39eb8105305c6123b9a0505606cd489f322dbaa - strip-components: 0 - - runs: | - chmod +x qdrant - mkdir -p ${{targets.destdir}}/usr/local/bin - mv qdrant ${{targets.destdir}}/usr/local/bin/qdrant - -update: - enabled: true - github: - identifier: qdrant/qdrant - strip-prefix: "v"