artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 19:51:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Qdrant: expose configuration options (#56260)

Browse Source 
This exposes configuration options for Qdrant so we can experiment with different settings for large installations.
This commit is contained in:
Camden Cheek 2023-09-27 14:05:52 -05:00 committed by GitHub
parent df3e78f9a8
commit c4092733e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 422 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
deps.bzl
View File

@ -5817,8 +5817,8 @@ def go_dependencies():
name = "com_github_qdrant_go_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/qdrant/go-client",
sum = "h1:LckV8C0TtMPDqWPd5g5sIa9zELQMelRlcZk9ANSZ2H8=",
version = "v1.3.0",
sum = "h1:kh5B4yWjrd5BcynJoA4014mZlI/j6++inCMMQoKUOtI=",
version = "v1.4.1",
)
go_repository(

2
go.mod
View File

@ -274,7 +274,7 @@ require (
github.com/mroth/weightedrand/v2 v2.0.1
github.com/pkoukk/tiktoken-go v0.1.5
github.com/prometheus/statsd_exporter v0.22.7
github.com/qdrant/go-client v1.3.0
github.com/qdrant/go-client v1.4.1
github.com/sourcegraph/managed-services-platform-cdktf/gen/cloudflare v0.0.0-20230822024612-edb48c530722
github.com/sourcegraph/managed-services-platform-cdktf/gen/google v0.0.0-20230822024612-edb48c530722
github.com/sourcegraph/managed-services-platform-cdktf/gen/random v0.0.0-20230822024612-edb48c530722

4
go.sum
View File

@ -1897,8 +1897,8 @@ github.com/pseudomuto/protoc-gen-doc v1.5.1 h1:Ah259kcrio7Ix1Rhb6u8FCaOkzf9qRBqX
github.com/pseudomuto/protoc-gen-doc v1.5.1/go.mod h1:XpMKYg6zkcpgfpCfQ8GcWBDRtRxOmMR5w7pz4Xo+dYM=
github.com/pseudomuto/protokit v0.2.0 h1:hlnBDcy3YEDXH7kc9gV+NLaN0cDzhDvD1s7Y6FZ8RpM=
github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q=
github.com/qdrant/go-client v1.3.0 h1:LckV8C0TtMPDqWPd5g5sIa9zELQMelRlcZk9ANSZ2H8=
github.com/qdrant/go-client v1.3.0/go.mod h1:680gkxNAsVtre0Z8hAQmtPzJtz1xFAyCu2TUxULtnoE=
github.com/qdrant/go-client v1.4.1 h1:kh5B4yWjrd5BcynJoA4014mZlI/j6++inCMMQoKUOtI=
github.com/qdrant/go-client v1.4.1/go.mod h1:680gkxNAsVtre0Z8hAQmtPzJtz1xFAyCu2TUxULtnoE=
github.com/qustavo/sqlhooks/v2 v2.1.0 h1:54yBemHnGHp/7xgT+pxwmIlMSDNYKx5JW5dfRAiCZi0=
github.com/qustavo/sqlhooks/v2 v2.1.0/go.mod h1:aMREyKo7fOKTwiLuWPsaHRXEmtqG4yREztO0idF83AU=
github.com/rafaeljusto/redigomock/v3 v3.1.2 h1:B4Y0XJQiPjpwYmkH55aratKX1VfR+JRqzmDKyZbC99o=

62
internal/conf/computed.go
View File

@ -970,6 +970,59 @@ func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.Embeddi
MaxFileSizeBytes: maxFileSizeLimit,
}
// Default values should match the documented defaults in site.schema.json.
computedQdrantConfig := conftypes.QdrantConfig{
Enabled: false,
QdrantHNSWConfig: conftypes.QdrantHNSWConfig{
EfConstruct: nil,
FullScanThreshold: nil,
M: nil,
OnDisk: true,
PayloadM: nil,
},
QdrantOptimizersConfig: conftypes.QdrantOptimizersConfig{
IndexingThreshold: 0,
MemmapThreshold: 100,
},
QdrantQuantizationConfig: conftypes.QdrantQuantizationConfig{
Enabled: true,
Quantile: 0.98,
},
}
if embeddingsConfig.Qdrant != nil {
qc := embeddingsConfig.Qdrant
computedQdrantConfig.Enabled = qc.Enabled
if qc.Hnsw != nil {
computedQdrantConfig.QdrantHNSWConfig.EfConstruct = toUint64(qc.Hnsw.EfConstruct)
computedQdrantConfig.QdrantHNSWConfig.FullScanThreshold = toUint64(qc.Hnsw.FullScanThreshold)
computedQdrantConfig.QdrantHNSWConfig.M = toUint64(qc.Hnsw.M)
computedQdrantConfig.QdrantHNSWConfig.PayloadM = toUint64(qc.Hnsw.PayloadM)
if qc.Hnsw.OnDisk != nil {
computedQdrantConfig.QdrantHNSWConfig.OnDisk = *qc.Hnsw.OnDisk
}
}
if qc.Optimizers != nil {
if qc.Optimizers.IndexingThreshold != nil {
computedQdrantConfig.QdrantOptimizersConfig.IndexingThreshold = uint64(*qc.Optimizers.IndexingThreshold)
}
if qc.Optimizers.MemmapThreshold != nil {
computedQdrantConfig.QdrantOptimizersConfig.MemmapThreshold = uint64(*qc.Optimizers.MemmapThreshold)
}
}
if qc.Quantization != nil {
if qc.Quantization.Enabled != nil {
computedQdrantConfig.QdrantQuantizationConfig.Enabled = *qc.Quantization.Enabled
}
if qc.Quantization.Quantile != nil {
computedQdrantConfig.QdrantQuantizationConfig.Quantile = float32(*qc.Quantization.Quantile)
}
}
}
computedConfig := &conftypes.EmbeddingsConfig{
Provider: conftypes.EmbeddingsProviderName(embeddingsConfig.Provider),
AccessToken: embeddingsConfig.AccessToken,
@ -983,6 +1036,7 @@ func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.Embeddi
MaxTextEmbeddingsPerRepo: embeddingsConfig.MaxTextEmbeddingsPerRepo,
PolicyRepositoryMatchLimit: embeddingsConfig.PolicyRepositoryMatchLimit,
ExcludeChunkOnError: pointers.Deref(embeddingsConfig.ExcludeChunkOnError, true),
Qdrant: computedQdrantConfig,
}
d, err := time.ParseDuration(embeddingsConfig.MinimumInterval)
if err != nil {
@ -994,6 +1048,14 @@ func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.Embeddi
return computedConfig
}
func toUint64(input *int) *uint64 {
if input == nil {
return nil
}
u := uint64(*input)
return &u
}
func getSourcegraphProviderAccessToken(accessToken string, config schema.SiteConfiguration) string {
// If an access token is configured, use it.
if accessToken != "" {

22
internal/conf/computed_test.go
View File

@ -696,6 +696,19 @@ func TestGetCompletionsConfig(t *testing.T) {
func TestGetEmbeddingsConfig(t *testing.T) {
licenseKey := "theasdfkey"
licenseAccessToken := license.GenerateLicenseKeyBasedAccessToken(licenseKey)
defaultQdrantConfig := conftypes.QdrantConfig{
QdrantHNSWConfig: conftypes.QdrantHNSWConfig{
OnDisk: true,
},
QdrantOptimizersConfig: conftypes.QdrantOptimizersConfig{
IndexingThreshold: 0,
MemmapThreshold: 100,
},
QdrantQuantizationConfig: conftypes.QdrantQuantizationConfig{
Enabled: true,
Quantile: 0.98,
},
}
zeroConfigDefaultWithLicense := &conftypes.EmbeddingsConfig{
Provider: "sourcegraph",
AccessToken: licenseAccessToken,
@ -711,6 +724,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
}
testCases := []struct {
@ -817,6 +831,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
ExcludedFilePathPatterns: []string{"*.java"},
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -851,6 +866,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
ExcludedFilePathPatterns: []string{"*.java"},
},
ExcludeChunkOnError: false,
Qdrant: defaultQdrantConfig,
},
},
{
@ -877,6 +893,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -915,6 +932,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -956,6 +974,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -982,6 +1001,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -1021,6 +1041,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{
@ -1048,6 +1069,7 @@ func TestGetEmbeddingsConfig(t *testing.T) {
MaxFileSizeBytes: 1000000,
},
ExcludeChunkOnError: true,
Qdrant: defaultQdrantConfig,
},
},
{

26
internal/conf/conftypes/consts.go
View File

@ -43,6 +43,32 @@ type EmbeddingsConfig struct {
MaxTextEmbeddingsPerRepo int
PolicyRepositoryMatchLimit *int
ExcludeChunkOnError bool
Qdrant QdrantConfig
}
type QdrantConfig struct {
Enabled bool
QdrantHNSWConfig QdrantHNSWConfig
QdrantOptimizersConfig QdrantOptimizersConfig
QdrantQuantizationConfig QdrantQuantizationConfig
}
type QdrantHNSWConfig struct {
EfConstruct *uint64
FullScanThreshold *uint64
M *uint64
OnDisk bool
PayloadM *uint64
}
type QdrantOptimizersConfig struct {
IndexingThreshold uint64
MemmapThreshold uint64
}
type QdrantQuantizationConfig struct {
Enabled bool
Quantile float32
}
type EmbeddingsProviderName string

3
internal/embeddings/db/BUILD.bazel
View File

@ -16,6 +16,7 @@ go_library(
deps = [
"//internal/api",
"//internal/conf",
"//internal/conf/conftypes",
"//internal/grpc/defaults",
"//lib/errors",
"//lib/pointers",
@ -36,6 +37,8 @@ go_test(
deps = [
"//internal/conf",
"//internal/conf/conftypes",
"//lib/pointers",
"//schema",
"@com_github_sourcegraph_log//logtest",
"@com_github_stretchr_testify//require",
],

142
internal/embeddings/db/conf.go
View File

@ -3,10 +3,13 @@ package db
import (
"sync/atomic"
qdrant "github.com/qdrant/go-client/qdrant"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/grpc/defaults"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"google.golang.org/grpc"
)
@ -16,34 +19,145 @@ import (
//
// If Qdrant is disabled, it will instead return the provided default VectorDB.
func NewDBFromConfFunc(logger log.Logger, def VectorDB) func() (VectorDB, error) {
type connAndErr struct {
conn *grpc.ClientConn
err error
}
var (
oldAddr string
err error
ptr atomic.Pointer[grpc.ClientConn]
ptr atomic.Pointer[connAndErr]
)
conf.Watch(func() {
if newAddr := conf.Get().ServiceConnections().Qdrant; newAddr != oldAddr {
newConn, dialErr := defaults.Dial(newAddr, logger)
c := conf.Get()
qc := conf.GetEmbeddingsConfig(c.SiteConfiguration)
if qc == nil || !qc.Qdrant.Enabled {
// Embeddings is disabled. Clear any errors and close any previous connection.
old := ptr.Swap(&connAndErr{nil, nil})
if old != nil && old.conn != nil {
old.conn.Close()
}
return
}
if newAddr := c.ServiceConnections().Qdrant; newAddr != oldAddr {
// The address has changed or this is running for the first time.
// Attempt to open dial Qdrant.
newConn, newErr := defaults.Dial(newAddr, logger)
oldAddr = newAddr
err = dialErr
oldConn := ptr.Swap(newConn)
if oldConn != nil {
oldConn.Close()
old := ptr.Swap(&connAndErr{newConn, newErr})
if old != nil && old.conn != nil {
old.conn.Close()
}
}
})
return func() (VectorDB, error) {
if err != nil {
return nil, err
curr := ptr.Load()
if curr.err != nil {
return nil, curr.err
}
conn := ptr.Load()
if conn == nil {
if curr.conn == nil {
return def, nil
}
return NewQdrantDBFromConn(conn), nil
return NewQdrantDBFromConn(curr.conn), nil
}
}
func createCollectionParams(name string, dims uint64, qc conftypes.QdrantConfig) *qdrant.CreateCollection {
return &qdrant.CreateCollection{
CollectionName: name,
HnswConfig: getHnswConfigDiff(qc.QdrantHNSWConfig),
OptimizersConfig: getOptimizersConfigDiff(qc.QdrantOptimizersConfig),
QuantizationConfig: getQuantizationConfig(qc.QdrantQuantizationConfig),
OnDiskPayload: pointers.Ptr(true),
VectorsConfig: getVectorsConfig(dims),
ShardNumber: nil, // default
WalConfig: nil, // default
ReplicationFactor: nil, // default
WriteConsistencyFactor: nil, // default
InitFromCollection: nil, // default
}
}
func updateCollectionParams(name string, qc conftypes.QdrantConfig) *qdrant.UpdateCollection {
return &qdrant.UpdateCollection{
CollectionName: name,
HnswConfig: getHnswConfigDiff(qc.QdrantHNSWConfig),
OptimizersConfig: getOptimizersConfigDiff(qc.QdrantOptimizersConfig),
QuantizationConfig: getQuantizationConfigDiff(qc.QdrantQuantizationConfig),
Params: nil, // do not update collection params
// Do not update vectors config.
// TODO(camdencheek): consider making OnDisk configurable
VectorsConfig: nil,
}
}
func getVectorsConfig(dims uint64) *qdrant.VectorsConfig {
return &qdrant.VectorsConfig{
Config: &qdrant.VectorsConfig_Params{
Params: &qdrant.VectorParams{
Size: dims,
Distance: qdrant.Distance_Cosine,
OnDisk: pointers.Ptr(true),
HnswConfig: nil, // use collection default
QuantizationConfig: nil, // use collection default
},
},
}
}
func getHnswConfigDiff(qhc conftypes.QdrantHNSWConfig) *qdrant.HnswConfigDiff {
return &qdrant.HnswConfigDiff{
M: qhc.M,
PayloadM: qhc.PayloadM,
EfConstruct: qhc.EfConstruct,
FullScanThreshold: qhc.FullScanThreshold,
OnDisk: &qhc.OnDisk,
MaxIndexingThreads: nil, // default
}
}
func getOptimizersConfigDiff(qoc conftypes.QdrantOptimizersConfig) *qdrant.OptimizersConfigDiff {
// Default values should match the documented defaults in site.schema.json.
return &qdrant.OptimizersConfigDiff{
IndexingThreshold: &qoc.IndexingThreshold,
MemmapThreshold: &qoc.MemmapThreshold,
DefaultSegmentNumber: nil,
VacuumMinVectorNumber: nil,
MaxOptimizationThreads: nil,
}
}
func getQuantizationConfigDiff(qqc conftypes.QdrantQuantizationConfig) *qdrant.QuantizationConfigDiff {
if !qqc.Enabled {
return &qdrant.QuantizationConfigDiff{
Quantization: &qdrant.QuantizationConfigDiff_Disabled{},
}
}
return &qdrant.QuantizationConfigDiff{
Quantization: &qdrant.QuantizationConfigDiff_Scalar{
Scalar: &qdrant.ScalarQuantization{
Type: qdrant.QuantizationType_Int8,
Quantile: &qqc.Quantile,
AlwaysRam: pointers.Ptr(false),
},
},
}
}
func getQuantizationConfig(qqc conftypes.QdrantQuantizationConfig) *qdrant.QuantizationConfig {
if !qqc.Enabled {
return nil
}
return &qdrant.QuantizationConfig{
Quantization: &qdrant.QuantizationConfig_Scalar{
Scalar: &qdrant.ScalarQuantization{
Type: qdrant.QuantizationType_Int8,
Quantile: &qqc.Quantile,
AlwaysRam: pointers.Ptr(false),
},
},
}
}

19
internal/embeddings/db/conf_test.go
View File

@ -3,10 +3,14 @@ package db
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/log/logtest"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"github.com/sourcegraph/sourcegraph/schema"
)
func TestNewDBFromConfFunc(t *testing.T) {
@ -24,9 +28,18 @@ func TestNewDBFromConfFunc(t *testing.T) {
t.Run("fake addr", func(t *testing.T) {
conf.Mock(&conf.Unified{
ServiceConnectionConfig: conftypes.ServiceConnections{
Qdrant: "fake_address_but_it_does_not_matter_because_grpc_dialing_is_lazy",
SiteConfiguration: schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Provider: "sourcegraph",
AccessToken: "fake",
Enabled: pointers.Ptr(true),
Qdrant: &schema.Qdrant{
Enabled: true,
},
},
CodyEnabled: pointers.Ptr(true),
},
ServiceConnectionConfig: conftypes.ServiceConnections{Qdrant: "fake_address_but_it_does_not_matter_because_grpc_dialing_is_lazy"},
})
getDB := NewDBFromConfFunc(logtest.Scoped(t), nil)
got, err := getDB()

65
internal/embeddings/db/migrate.go
View File

@ -4,17 +4,19 @@ import (
"context"
qdrant "github.com/qdrant/go-client/qdrant"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/pointers"
)
func ensureModelCollectionWithDefaultConfig(ctx context.Context, db *qdrantDB, modelID string, modelDims uint64) error {
func ensureModelCollection(ctx context.Context, db *qdrantDB, modelID string, modelDims uint64) error {
// Make the actual collection end with `.default` so we can switch between
// configurations with aliases.
name := CollectionName(modelID)
realName := name + ".default"
err := ensureCollection(ctx, db.collectionsClient, realName, defaultConfig(modelDims))
err := ensureCollectionWithConfig(ctx, db.collectionsClient, realName, modelDims, conf.GetEmbeddingsConfig(conf.Get().SiteConfiguration).Qdrant)
if err != nil {
return err
}
@ -42,7 +44,7 @@ func ensureModelCollectionWithDefaultConfig(ctx context.Context, db *qdrantDB, m
return nil
}
func ensureCollection(ctx context.Context, cc qdrant.CollectionsClient, name string, config *qdrant.CollectionConfig) error {
func ensureCollectionWithConfig(ctx context.Context, cc qdrant.CollectionsClient, name string, dims uint64, qc conftypes.QdrantConfig) error {
resp, err := cc.List(ctx, &qdrant.ListCollectionsRequest{})
if err != nil {
return err
@ -50,26 +52,12 @@ func ensureCollection(ctx context.Context, cc qdrant.CollectionsClient, name str
for _, collection := range resp.GetCollections() {
if collection.GetName() == name {
// Collection already exists
return nil
_, err = cc.Update(ctx, updateCollectionParams(name, qc))
return err
}
}
// Create a new collection with the new config using the data of the old collection
_, err = cc.Create(ctx, &qdrant.CreateCollection{
CollectionName: name,
HnswConfig: config.HnswConfig,
WalConfig: config.WalConfig,
OptimizersConfig: config.OptimizerConfig,
ShardNumber: &config.Params.ShardNumber,
OnDiskPayload: &config.Params.OnDiskPayload,
VectorsConfig: config.Params.VectorsConfig,
ReplicationFactor: config.Params.ReplicationFactor,
WriteConsistencyFactor: config.Params.WriteConsistencyFactor,
InitFromCollection: nil,
QuantizationConfig: config.QuantizationConfig,
})
_, err = cc.Create(ctx, createCollectionParams(name, dims, qc))
return err
}
@ -89,40 +77,3 @@ func ensureRepoIDIndex(ctx context.Context, cc qdrant.PointsClient, name string)
return nil
}
func defaultConfig(dims uint64) *qdrant.CollectionConfig {
return &qdrant.CollectionConfig{
Params: &qdrant.CollectionParams{
ShardNumber: 1,
OnDiskPayload: true,
VectorsConfig: &qdrant.VectorsConfig{
Config: &qdrant.VectorsConfig_Params{
Params: &qdrant.VectorParams{
Size: dims,
Distance: qdrant.Distance_Cosine,
HnswConfig: nil, // use collection default
QuantizationConfig: nil, // use collection default
OnDisk: pointers.Ptr(true), // use collection default
},
},
},
ReplicationFactor: nil, // default
WriteConsistencyFactor: nil, // default
},
OptimizerConfig: &qdrant.OptimizersConfigDiff{
IndexingThreshold: pointers.Ptr(uint64(0)), // disable indexing
},
WalConfig: nil, // default
QuantizationConfig: &qdrant.QuantizationConfig{
// scalar is faster than product, but doesn't compress as well
Quantization: &qdrant.QuantizationConfig_Scalar{
Scalar: &qdrant.ScalarQuantization{
Type: qdrant.QuantizationType_Int8,
// Truncate outliers for better compression
Quantile: pointers.Ptr(float32(0.98)),
AlwaysRam: nil, // default false
},
},
},
}
}

2
internal/embeddings/db/qdrant.go
View File

@ -94,7 +94,7 @@ func (db *qdrantDB) Search(ctx context.Context, params SearchParams) ([]ChunkRes
}
func (db *qdrantDB) PrepareUpdate(ctx context.Context, modelID string, modelDims uint64) error {
return ensureModelCollectionWithDefaultConfig(ctx, db, modelID, modelDims)
return ensureModelCollection(ctx, db, modelID, modelDims)
}
func (db *qdrantDB) HasIndex(ctx context.Context, modelID string, repoID api.RepoID, revision api.CommitID) (bool, error) {

40
schema/schema.go
View File

@ -682,6 +682,8 @@ type Embeddings struct {
PolicyRepositoryMatchLimit *int `json:"policyRepositoryMatchLimit,omitempty"`
// Provider description: The provider to use for generating embeddings. Defaults to sourcegraph.
Provider string `json:"provider,omitempty"`
// Qdrant description: Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team.
Qdrant *Qdrant `json:"qdrant,omitempty"`
// Url description: The url to the external embedding API service. Deprecated, use endpoint instead.
Url string `json:"url,omitempty"`
}
@ -1388,6 +1390,20 @@ type Header struct {
Value string `json:"value"`
}
// Hnsw description: Overrides for the HNSW index config.
type Hnsw struct {
// EfConstruct description: Number of neighbours to consider during the index building. Larger the value, more accurate the search, more time required to build the index.
EfConstruct *int `json:"efConstruct,omitempty"`
// FullScanThreshold description: Minimal size (in KiloBytes) of vectors for additional payload-based indexing.
FullScanThreshold *int `json:"fullScanThreshold,omitempty"`
// M description: Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.
M *int `json:"m,omitempty"`
// OnDisk description: Store HNSW index on disk.
OnDisk *bool `json:"onDisk,omitempty"`
// PayloadM description: Number of edges per node in the index graph. Larger the value, more accurate the search, more space required.
PayloadM *int `json:"payloadM,omitempty"`
}
// IdentityProvider description: The source of identity to use when computing permissions. This defines how to compute the GitLab identity to use for a given Sourcegraph user.
type IdentityProvider struct {
Oauth *OAuthIdentity
@ -1745,6 +1761,12 @@ type OpenTelemetry struct {
// Endpoint description: OpenTelemetry tracing collector endpoint. By default, Sourcegraph's "/-/debug/otlp" endpoint forwards data to the configured collector backend.
Endpoint string `json:"endpoint,omitempty"`
}
type Optimizers struct {
// IndexingThreshold description: Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing. Set to 0 to disable indexing
IndexingThreshold *int `json:"indexingThreshold,omitempty"`
// MemmapThreshold description: Maximum size (in kilobytes) of vectors to store in-memory per segment.
MemmapThreshold *int `json:"memmapThreshold,omitempty"`
}
// OrganizationInvitations description: Configuration for organization invitations.
type OrganizationInvitations struct {
@ -1917,6 +1939,24 @@ type PythonRateLimit struct {
// RequestsPerHour description: Requests per hour permitted. This is an average, calculated per second. Internally, the burst limit is set to 100, which implies that for a requests per hour limit as low as 1, users will continue to be able to send a maximum of 100 requests immediately, provided that the complexity cost of each request is 1.
RequestsPerHour float64 `json:"requestsPerHour"`
}
// Qdrant description: Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team.
type Qdrant struct {
Enabled bool `json:"enabled,omitempty"`
// Hnsw description: Overrides for the HNSW index config.
Hnsw *Hnsw `json:"hnsw,omitempty"`
Optimizers *Optimizers `json:"optimizers,omitempty"`
// Quantization description: Overrides for quantization config.
Quantization *Quantization `json:"quantization,omitempty"`
}
// Quantization description: Overrides for quantization config.
type Quantization struct {
// Enabled description: Whether to enable int8 scalar quantization
Enabled *bool `json:"enabled,omitempty"`
// Quantile description: Any values that lie outside the quantile range will be truncated
Quantile *float64 `json:"quantile,omitempty"`
}
type QuickLink struct {
// Description description: A description for this quick link
Description string `json:"description,omitempty"`

110
schema/site.schema.json
View File

@ -2577,6 +2577,116 @@
"pointer": true
},
"default": true
},
"qdrant": {
"description": "Overrides for the default qdrant config. These should generally not be modified without direction from the Sourcegraph support team.",
"type": "object",
"$comment": "Skipped fields from HnswConfigDiff: MaxIndexingThreads",
"properties": {
"enabled": {
"type": "boolean",
"default": false
},
"hnsw": {
"description": "Overrides for the HNSW index config.",
"type": "object",
"properties": {
"m": {
"description": "Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 8
},
"efConstruct": {
"description": "Number of neighbours to consider during the index building. Larger the value, more accurate the search, more time required to build the index.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 100
},
"fullScanThreshold": {
"description": "Minimal size (in KiloBytes) of vectors for additional payload-based indexing.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 1000
},
"onDisk": {
"description": "Store HNSW index on disk.",
"type": "boolean",
"!go": {
"pointer": true
},
"default": true
},
"payloadM": {
"description": "Number of edges per node in the index graph. Larger the value, more accurate the search, more space required.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 8
}
}
},
"quantization": {
"description": "Overrides for quantization config.",
"type": "object",
"$comment": "Skipped fields from ScalarQuantization: AlwaysRam",
"properties": {
"enabled": {
"description": "Whether to enable int8 scalar quantization",
"type": "boolean",
"!go": {
"pointer": true
},
"default": true
},
"quantile": {
"description": "Any values that lie outside the quantile range will be truncated",
"type": "number",
"minimum": 0,
"maximum": 1,
"!go": {
"pointer": true
},
"default": 0.98
}
}
},
"optimizers": {
"type": "object",
"$comment": "Skipped fields from OptimizersConfigDiff: DeletedThreshold, VacuumMinVectorNumber, DefaultSegmentNumber, MaxSegmentSize, FlushIntervalSec, MaxOptimizationThreads",
"properties": {
"indexingThreshold": {
"description": "Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing. Set to 0 to disable indexing",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 0
},
"memmapThreshold": {
"description": "Maximum size (in kilobytes) of vectors to store in-memory per segment.",
"type": "integer",
"minimum": 0,
"!go": {
"pointer": true
},
"default": 100
}
}
}
}
}
},
"examples": [

1
sg.config.yaml
View File

@ -272,6 +272,7 @@ commands:
-e QDRANT__LOG_LEVEL=INFO \
-e QDRANT__STORAGE__STORAGE_PATH=/data \
-e QDRANT__STORAGE__SNAPSHOTS_PATH=/data \
-e QDRANT_INIT_FILE_PATH=/data/.qdrant-initialized \
--entrypoint /usr/local/bin/qdrant \
sourcegraph/qdrant:insiders
worker: