artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 19:21:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Unify cody configuration into single source of truth (#53471)

Browse Source 
This unified all cody configuration generation into a single source of
truth and tests for the defaults settings for embeddings and completions
we want for 5.1.

Ticks off a bunch of items in
https://github.com/sourcegraph/sourcegraph/issues/52980.

## Test plan

Added a bunch of tests.
This commit is contained in:
Erik Seliger 2023-06-15 03:07:16 +02:00 committed by GitHub
parent 909b063702
commit bef22627e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
108 changed files with 1256 additions and 779 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/frontend/graphqlbackend/site.go
View File

@ -24,6 +24,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/auth"
"github.com/sourcegraph/sourcegraph/internal/cody"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/migration"
@ -527,31 +528,31 @@ func (r *schemaResolver) SetAutoUpgrade(ctx context.Context, args *struct {
}
func (r *siteResolver) PerUserCompletionsQuota() *int32 {
c := conf.Get()
if c.Completions != nil && c.Completions.PerUserDailyLimit > 0 {
i := int32(c.Completions.PerUserDailyLimit)
c := conf.GetCompletionsConfig(conf.Get().SiteConfig())
if c != nil && c.PerUserDailyLimit > 0 {
i := int32(c.PerUserDailyLimit)
return &i
}
return nil
}
func (r *siteResolver) PerUserCodeCompletionsQuota() *int32 {
c := conf.Get()
if c.Completions != nil && c.Completions.PerUserCodeCompletionsDailyLimit > 0 {
i := int32(c.Completions.PerUserCodeCompletionsDailyLimit)
c := conf.GetCompletionsConfig(conf.Get().SiteConfig())
if c != nil && c.PerUserCodeCompletionsDailyLimit > 0 {
i := int32(c.PerUserCodeCompletionsDailyLimit)
return &i
}
return nil
}
func (r *siteResolver) RequiresVerifiedEmailForCody(ctx context.Context) bool {
// App is typically forwarding Cody requests to dotcom.
// This section can be removed if dotcom stops requiring verified emails
if deploy.IsApp() {
// App users can specify their own keys
// if they use their own keys requests are not forwarded to dotcom
// which means a verified email is not needed
return conf.Get().Completions == nil
c := conf.GetCompletionsConfig(conf.Get().SiteConfig())
// App users can specify their own keys using one of the regular providers.
// If they use their own keys requests are not going through Cody Gateway
// which means a verified email is not needed.
return c == nil || c.Provider == conftypes.CompletionsProviderNameSourcegraph
}
// We only require this on dotcom

4
enterprise/cmd/cody-gateway/internal/actor/BUILD.bazel generated
View File

@ -13,7 +13,7 @@ go_library(
deps = [
"//enterprise/cmd/cody-gateway/internal/limiter",
"//enterprise/cmd/cody-gateway/internal/notify",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/goroutine",
"//internal/observation",
"//internal/trace",
@ -42,7 +42,7 @@ go_test(
],
deps = [
"//enterprise/cmd/cody-gateway/internal/limiter",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/observation",
"//internal/redispool",
"//lib/errors",

2
enterprise/cmd/cody-gateway/internal/actor/actor.go
View File

@ -11,7 +11,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/notify"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
)
type Actor struct {

2
enterprise/cmd/cody-gateway/internal/actor/actor_test.go
View File

@ -8,7 +8,7 @@ import (
"github.com/hexops/autogold/v2"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
)
func TestActor_TraceAttributes(t *testing.T) {

2
enterprise/cmd/cody-gateway/internal/actor/anonymous/BUILD.bazel generated
View File

@ -8,6 +8,6 @@ go_library(
deps = [
"//enterprise/cmd/cody-gateway/internal/actor",
"//enterprise/cmd/cody-gateway/internal/httpapi/embeddings",
"//enterprise/internal/codygateway",
"//internal/codygateway",
],
)

2
enterprise/cmd/cody-gateway/internal/actor/anonymous/anonymous.go
View File

@ -6,7 +6,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/actor"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/httpapi/embeddings"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
)
type Source struct {

4
enterprise/cmd/cody-gateway/internal/actor/dotcomuser/BUILD.bazel generated
View File

@ -9,7 +9,7 @@ go_library(
deps = [
"//enterprise/cmd/cody-gateway/internal/actor",
"//enterprise/cmd/cody-gateway/internal/dotcom",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/trace",
"//lib/errors",
"@com_github_gregjones_httpcache//:httpcache",
@ -25,7 +25,7 @@ go_test(
embed = [":dotcomuser"],
deps = [
"//enterprise/cmd/cody-gateway/internal/dotcom",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"@com_github_stretchr_testify//assert",
],
)

2
enterprise/cmd/cody-gateway/internal/actor/dotcomuser/dotcomuser.go
View File

@ -13,7 +13,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/actor"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/dotcom"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/cmd/cody-gateway/internal/actor/dotcomuser/dotcomuser_test.go
View File

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/dotcom"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
)
func TestNewActor(t *testing.T) {

2
enterprise/cmd/cody-gateway/internal/actor/limits.go
View File

@ -12,7 +12,7 @@ import (
oteltrace "go.opentelemetry.io/otel/trace"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/cmd/cody-gateway/internal/actor/limits_test.go
View File

@ -10,7 +10,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

5
enterprise/cmd/cody-gateway/internal/actor/productsubscription/BUILD.bazel generated
View File

@ -9,9 +9,10 @@ go_library(
deps = [
"//enterprise/cmd/cody-gateway/internal/actor",
"//enterprise/cmd/cody-gateway/internal/dotcom",
"//enterprise/internal/codygateway",
"//enterprise/internal/licensing",
"//enterprise/internal/productsubscription",
"//internal/codygateway",
"//internal/licensing",
"//internal/trace",
"//lib/errors",
"@com_github_gregjones_httpcache//:httpcache",
@ -30,7 +31,7 @@ go_test(
embed = [":productsubscription"],
deps = [
"//enterprise/cmd/cody-gateway/internal/dotcom",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"@com_github_stretchr_testify//assert",
],
)

7
enterprise/cmd/cody-gateway/internal/actor/productsubscription/productsubscription.go
View File

@ -16,9 +16,10 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/actor"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/dotcom"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
elicensing "github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/enterprise/internal/productsubscription"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/licensing"
sgtrace "github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -217,7 +218,7 @@ func newActor(source *Source, token string, s dotcom.ProductSubscriptionState, i
// In internal mode, only allow dev and internal licenses.
disallowedLicense := internalMode &&
(s.ActiveLicense == nil || s.ActiveLicense.Info == nil ||
!containsOneOf(s.ActiveLicense.Info.Tags, licensing.DevTag, licensing.InternalTag))
!containsOneOf(s.ActiveLicense.Info.Tags, elicensing.DevTag, elicensing.InternalTag))
now := time.Now()
a := &actor.Actor{

2
enterprise/cmd/cody-gateway/internal/actor/productsubscription/productsubscription_test.go
View File

@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/dotcom"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
)
func TestNewActor(t *testing.T) {

4
enterprise/cmd/cody-gateway/internal/auth/BUILD.bazel generated
View File

@ -13,7 +13,7 @@ go_library(
"//enterprise/cmd/cody-gateway/internal/actor",
"//enterprise/cmd/cody-gateway/internal/events",
"//enterprise/cmd/cody-gateway/internal/response",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/trace",
"//lib/errors",
"@com_github_sourcegraph_log//:log",
@ -33,8 +33,8 @@ go_test(
"//enterprise/cmd/cody-gateway/internal/actor/productsubscription",
"//enterprise/cmd/cody-gateway/internal/dotcom",
"//enterprise/cmd/cody-gateway/internal/events",
"//enterprise/internal/codygateway",
"//enterprise/internal/licensing",
"//internal/codygateway",
"//lib/errors",
"@com_github_derision_test_go_mockgen//testutil/require",
"@com_github_gregjones_httpcache//:httpcache",

2
enterprise/cmd/cody-gateway/internal/auth/auth.go
View File

@ -8,7 +8,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/actor"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/events"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/response"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/cmd/cody-gateway/internal/auth/auth_test.go
View File

@ -20,8 +20,8 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/actor/productsubscription"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/dotcom"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/events"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/cmd/cody-gateway/internal/events/BUILD.bazel generated
View File

@ -12,7 +12,7 @@ go_library(
importpath = "github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/events",
visibility = ["//enterprise/cmd/cody-gateway:__subpackages__"],
deps = [
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/goroutine",
"//internal/trace",
"//lib/errors",

2
enterprise/cmd/cody-gateway/internal/events/events.go
View File

@ -9,7 +9,7 @@ import (
"github.com/sourcegraph/log"
oteltrace "go.opentelemetry.io/otel/trace"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

3
enterprise/cmd/cody-gateway/internal/httpapi/completions/BUILD.bazel generated
View File

@ -16,9 +16,10 @@ go_library(
"//enterprise/cmd/cody-gateway/internal/limiter",
"//enterprise/cmd/cody-gateway/internal/notify",
"//enterprise/cmd/cody-gateway/internal/response",
"//enterprise/internal/codygateway",
"//enterprise/internal/completions/client/anthropic",
"//enterprise/internal/completions/client/openai",
"//internal/codygateway",
"//internal/conf/conftypes",
"//internal/httpcli",
"//internal/trace",
"//lib/errors",

3
enterprise/cmd/cody-gateway/internal/httpapi/completions/anthropic.go
View File

@ -12,6 +12,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/notify"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/anthropic"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
)
const anthropicAPIURL = "https://api.anthropic.com/v1/complete"
@ -29,7 +30,7 @@ func NewAnthropicHandler(
eventLogger,
rs,
rateLimitNotifier,
anthropic.ProviderName,
string(conftypes.CompletionsProviderNameAnthropic),
anthropicAPIURL,
allowedModels,
upstreamHandlerMethods[anthropicRequest]{

3
enterprise/cmd/cody-gateway/internal/httpapi/completions/openai.go
View File

@ -12,6 +12,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/notify"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/openai"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
)
const openAIURL = "https://api.openai.com/v1/chat/completions"
@ -30,7 +31,7 @@ func NewOpenAIHandler(
eventLogger,
rs,
rateLimitNotifier,
openai.ProviderName,
string(conftypes.CompletionsProviderNameOpenAI),
openAIURL,
allowedModels,
upstreamHandlerMethods[openaiRequest]{

2
enterprise/cmd/cody-gateway/internal/httpapi/completions/upstream.go
View File

@ -20,7 +20,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/notify"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/response"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
sgtrace "github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"

2
enterprise/cmd/cody-gateway/internal/httpapi/embeddings/BUILD.bazel generated
View File

@ -16,7 +16,7 @@ go_library(
"//enterprise/cmd/cody-gateway/internal/limiter",
"//enterprise/cmd/cody-gateway/internal/notify",
"//enterprise/cmd/cody-gateway/internal/response",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/httpcli",
"//internal/trace",
"//lib/errors",

2
enterprise/cmd/cody-gateway/internal/httpapi/embeddings/handler.go
View File

@ -17,7 +17,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/notify"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/response"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
sgtrace "github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/cmd/cody-gateway/internal/httpapi/embeddings/models.go
View File

@ -8,7 +8,7 @@ import (
"golang.org/x/exp/slices"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/actor"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
)
type ModelName string

2
enterprise/cmd/cody-gateway/internal/httpapi/embeddings/openai.go
View File

@ -9,7 +9,7 @@ import (
"sort"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/response"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

4
enterprise/cmd/cody-gateway/internal/httpapi/featurelimiter/BUILD.bazel generated
View File

@ -11,8 +11,8 @@ go_library(
"//enterprise/cmd/cody-gateway/internal/limiter",
"//enterprise/cmd/cody-gateway/internal/notify",
"//enterprise/cmd/cody-gateway/internal/response",
"//enterprise/internal/codygateway",
"//enterprise/internal/completions/types",
"//internal/codygateway",
"//internal/completions/types",
"//internal/trace",
"//lib/errors",
"@com_github_sourcegraph_log//:log",

4
enterprise/cmd/cody-gateway/internal/httpapi/featurelimiter/featurelimiter.go
View File

@ -12,8 +12,8 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/limiter"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/notify"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/response"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
sgtrace "github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

4
enterprise/cmd/cody-gateway/internal/notify/BUILD.bazel generated
View File

@ -7,7 +7,7 @@ go_library(
importpath = "github.com/sourcegraph/sourcegraph/enterprise/cmd/cody-gateway/internal/notify",
visibility = ["//enterprise/cmd/cody-gateway:__subpackages__"],
deps = [
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/redislock",
"//internal/redispool",
"@com_github_gomodule_redigo//redis",
@ -21,7 +21,7 @@ go_test(
srcs = ["rate_limit_test.go"],
embed = [":notify"],
deps = [
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/redispool",
"@com_github_slack_go_slack//:slack",
"@com_github_sourcegraph_log//logtest",

2
enterprise/cmd/cody-gateway/internal/notify/rate_limit.go
View File

@ -10,7 +10,7 @@ import (
"github.com/slack-go/slack"
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/redislock"
"github.com/sourcegraph/sourcegraph/internal/redispool"
)

2
enterprise/cmd/cody-gateway/internal/notify/rate_limit_test.go
View File

@ -9,7 +9,7 @@ import (
"github.com/sourcegraph/log/logtest"
"github.com/stretchr/testify/assert"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/redispool"
)

2
enterprise/cmd/cody-gateway/shared/BUILD.bazel generated
View File

@ -22,7 +22,7 @@ go_library(
"//enterprise/cmd/cody-gateway/internal/httpapi",
"//enterprise/cmd/cody-gateway/internal/limiter",
"//enterprise/cmd/cody-gateway/internal/notify",
"//enterprise/internal/codygateway",
"//internal/codygateway",
"//internal/debugserver",
"//internal/env",
"//internal/goroutine",

2
enterprise/cmd/cody-gateway/shared/config.go
View File

@ -7,7 +7,7 @@ import (
"strings"
"time"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/trace/policy"
"github.com/sourcegraph/sourcegraph/lib/errors"

1
enterprise/cmd/embeddings/qa/embed_queries/BUILD.bazel generated
View File

@ -7,6 +7,7 @@ go_library(
visibility = ["//visibility:private"],
deps = [
"//enterprise/internal/embeddings/embed",
"//internal/conf",
"//internal/jsonc",
"//lib/errors",
"//schema",

3
enterprise/cmd/embeddings/qa/embed_queries/main.go
View File

@ -25,6 +25,7 @@ import (
"strings"
"github.com/sourcegraph/sourcegraph/enterprise/internal/embeddings/embed"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/jsonc"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
@ -64,7 +65,7 @@ func embedQueries(queries []string, siteConfigPath string) error {
for _, query := range queries {
fmt.Printf("Embedding query %s\n", query)
c, err := embed.NewEmbeddingsClient(siteConfig)
c, err := embed.NewEmbeddingsClient(conf.GetEmbeddingsConfig(*siteConfig))
if err != nil {
return err
}

6
enterprise/cmd/embeddings/shared/main.go
View File

@ -167,7 +167,11 @@ func NewHandler(
}
func getQueryEmbedding(ctx context.Context, query string) ([]float32, string, error) {
client, err := embed.NewEmbeddingsClient(&conf.Get().SiteConfiguration)
c := conf.GetEmbeddingsConfig(conf.Get().SiteConfig())
if c == nil {
return nil, "", errors.New("embeddings not configured or disabled")
}
client, err := embed.NewEmbeddingsClient(c)
if err != nil {
return nil, "", errors.Wrap(err, "getting embeddings client")
}

2
enterprise/cmd/frontend/internal/completions/resolvers/BUILD.bazel generated
View File

@ -10,7 +10,7 @@ go_library(
"//enterprise/internal/cody",
"//enterprise/internal/completions/client",
"//enterprise/internal/completions/httpapi",
"//enterprise/internal/completions/types",
"//internal/completions/types",
"//internal/conf",
"//internal/database",
"//internal/redispool",

4
enterprise/cmd/frontend/internal/completions/resolvers/resolver.go
View File

@ -10,7 +10,7 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/internal/cody"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/httpapi"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/redispool"
@ -40,7 +40,7 @@ func (c *completionsResolver) Completions(ctx context.Context, args graphqlbacke
return "", err
}
completionsConfig := client.GetCompletionsConfig(conf.Get().SiteConfig())
completionsConfig := conf.GetCompletionsConfig(conf.Get().SiteConfig())
if completionsConfig == nil {
return "", errors.New("completions are not configured")
}

2
enterprise/cmd/frontend/internal/context/resolvers/context_test.go
View File

@ -143,8 +143,6 @@ func TestContextResolver(t *testing.T) {
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Embeddings: &schema.Embeddings{Enabled: true},
Completions: &schema.Completions{},
},
})

6
enterprise/cmd/frontend/internal/dotcom/productsubscription/BUILD.bazel generated
View File

@ -26,13 +26,13 @@ go_library(
deps = [
"//cmd/frontend/graphqlbackend",
"//cmd/frontend/graphqlbackend/graphqlutil",
"//enterprise/internal/codygateway",
"//enterprise/internal/completions/types",
"//enterprise/internal/license",
"//enterprise/internal/licensing",
"//enterprise/internal/productsubscription",
"//internal/auth",
"//internal/authz",
"//internal/codygateway",
"//internal/completions/types",
"//internal/conf",
"//internal/database",
"//internal/database/basestore",
@ -42,6 +42,7 @@ go_library(
"//internal/featureflag",
"//internal/gqlutil",
"//internal/hashutil",
"//internal/licensing",
"//internal/redispool",
"//internal/slack",
"//internal/types",
@ -96,6 +97,7 @@ go_test(
"//internal/errcode",
"//internal/featureflag",
"//internal/hashutil",
"//internal/licensing",
"//internal/redispool",
"//internal/slack",
"//internal/timeutil",

14
enterprise/cmd/frontend/internal/dotcom/productsubscription/codygateway_dotcom_user.go
View File

@ -6,9 +6,9 @@ import (
"math"
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/errcode"
@ -176,15 +176,15 @@ func getCompletionsRateLimit(ctx context.Context, db database.DB, userID int32,
if limit == nil {
source = graphqlbackend.CodyGatewayRateLimitSourcePlan
// Otherwise, fall back to the global limit.
cfg := conf.Get()
cfg := conf.GetCompletionsConfig(conf.Get().SiteConfig())
switch scope {
case types.CompletionsFeatureChat:
if cfg.Completions != nil && cfg.Completions.PerUserDailyLimit > 0 {
limit = pointers.Ptr(cfg.Completions.PerUserDailyLimit)
if cfg != nil && cfg.PerUserDailyLimit > 0 {
limit = pointers.Ptr(cfg.PerUserDailyLimit)
}
case types.CompletionsFeatureCode:
if cfg.Completions != nil && cfg.Completions.PerUserCodeCompletionsDailyLimit > 0 {
limit = pointers.Ptr(cfg.Completions.PerUserCodeCompletionsDailyLimit)
if cfg != nil && cfg.PerUserCodeCompletionsDailyLimit > 0 {
limit = pointers.Ptr(cfg.PerUserCodeCompletionsDailyLimit)
}
default:
return licensing.CodyGatewayRateLimit{}, graphqlbackend.CodyGatewayRateLimitSourcePlan, errors.Newf("unknown scope: %s", scope)

4
enterprise/cmd/frontend/internal/dotcom/productsubscription/codygateway_dotcom_user_test.go
View File

@ -29,9 +29,13 @@ func TestCodyGatewayDotcomUserResolver(t *testing.T) {
var chatOverrideLimit int = 200
var codeOverrideLimit int = 400
tru := true
cfg := &conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &tru,
LicenseKey: "asdf",
Completions: &schema.Completions{
Provider: "sourcegraph",
PerUserCodeCompletionsDailyLimit: 20,
PerUserDailyLimit: 10,
},

4
enterprise/cmd/frontend/internal/dotcom/productsubscription/codygateway_graphql.go
View File

@ -5,9 +5,9 @@ import (
"time"
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/gqlutil"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

4
enterprise/cmd/frontend/internal/dotcom/productsubscription/codygateway_service.go
View File

@ -9,8 +9,8 @@ import (
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/lib/errors"

25
enterprise/cmd/frontend/internal/dotcom/productsubscription/license_check_handler_test.go
View File

@ -13,8 +13,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
elicensing "github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/pointers"
)
@ -84,14 +85,14 @@ func TestNewLicenseCheckHandler(t *testing.T) {
name string
body string
headers http.Header
want licensing.LicenseCheckResponse
want elicensing.LicenseCheckResponse
wantStatus int
}{
{
name: "no access token",
body: getBody(""),
headers: nil,
want: licensing.LicenseCheckResponse{Error: ErrInvalidAccessTokenMsg},
want: elicensing.LicenseCheckResponse{Error: ErrInvalidAccessTokenMsg},
wantStatus: http.StatusUnauthorized,
},
{
@ -100,7 +101,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
headers: http.Header{
"Authorization": {"Bearer invalid-token"},
},
want: licensing.LicenseCheckResponse{Error: ErrInvalidAccessTokenMsg},
want: elicensing.LicenseCheckResponse{Error: ErrInvalidAccessTokenMsg},
wantStatus: http.StatusUnauthorized,
},
{
@ -109,7 +110,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
headers: http.Header{
"Authorization": {"Bearer " + hex.EncodeToString(expiredLicense.LicenseCheckToken)},
},
want: licensing.LicenseCheckResponse{Error: ErrExpiredLicenseMsg},
want: elicensing.LicenseCheckResponse{Error: ErrExpiredLicenseMsg},
wantStatus: http.StatusForbidden,
},
{
@ -118,7 +119,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
headers: http.Header{
"Authorization": {"Bearer " + hex.EncodeToString(revokedLicense.LicenseCheckToken)},
},
want: licensing.LicenseCheckResponse{Data: &licensing.LicenseCheckResponseData{IsValid: false, Reason: "license revoked"}},
want: elicensing.LicenseCheckResponse{Data: &elicensing.LicenseCheckResponseData{IsValid: false, Reason: "license revoked"}},
wantStatus: http.StatusForbidden,
},
{
@ -127,7 +128,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
headers: http.Header{
"Authorization": {"Bearer " + hex.EncodeToString(validLicense.LicenseCheckToken)},
},
want: licensing.LicenseCheckResponse{Error: ErrInvalidRequestBodyMsg},
want: elicensing.LicenseCheckResponse{Error: ErrInvalidRequestBodyMsg},
wantStatus: http.StatusBadRequest,
},
{
@ -136,7 +137,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
"Authorization": {"Bearer " + hex.EncodeToString(assignedLicense.LicenseCheckToken)},
},
body: getBody(""),
want: licensing.LicenseCheckResponse{Data: &licensing.LicenseCheckResponseData{IsValid: false, Reason: "license is already in use"}},
want: elicensing.LicenseCheckResponse{Data: &elicensing.LicenseCheckResponseData{IsValid: false, Reason: "license is already in use"}},
wantStatus: http.StatusOK,
},
{
@ -145,7 +146,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
headers: http.Header{
"Authorization": {"Bearer " + hex.EncodeToString(assignedLicense.LicenseCheckToken)},
},
want: licensing.LicenseCheckResponse{Data: &licensing.LicenseCheckResponseData{IsValid: true}},
want: elicensing.LicenseCheckResponse{Data: &elicensing.LicenseCheckResponseData{IsValid: true}},
wantStatus: http.StatusOK,
},
{
@ -154,7 +155,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
headers: http.Header{
"Authorization": {"Bearer " + hex.EncodeToString(validLicense.LicenseCheckToken)},
},
want: licensing.LicenseCheckResponse{Error: ErrInvalidSiteIDMsg},
want: elicensing.LicenseCheckResponse{Error: ErrInvalidSiteIDMsg},
wantStatus: http.StatusBadRequest,
},
{
@ -163,7 +164,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
headers: http.Header{
"Authorization": {"Bearer " + hex.EncodeToString(validLicense.LicenseCheckToken)},
},
want: licensing.LicenseCheckResponse{Data: &licensing.LicenseCheckResponseData{IsValid: true}},
want: elicensing.LicenseCheckResponse{Data: &elicensing.LicenseCheckResponseData{IsValid: true}},
wantStatus: http.StatusOK,
},
}
@ -183,7 +184,7 @@ func TestNewLicenseCheckHandler(t *testing.T) {
require.Equal(t, test.wantStatus, res.Code)
require.Equal(t, "application/json", res.Header().Get("Content-Type"))
var got licensing.LicenseCheckResponse
var got elicensing.LicenseCheckResponse
json.Unmarshal([]byte(res.Body.String()), &got)
require.Equal(t, test.want, got)
})

2
enterprise/cmd/frontend/internal/dotcom/productsubscription/licenses_db.go
View File

@ -10,10 +10,10 @@ import (
"github.com/lib/pq"
"github.com/sourcegraph/sourcegraph/enterprise/internal/license"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/dbutil"
"github.com/sourcegraph/sourcegraph/internal/hashutil"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/cmd/frontend/internal/dotcom/productsubscription/licenses_db_test.go
View File

@ -13,9 +13,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/internal/license"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/dbtest"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/timeutil"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/pointers"

2
enterprise/cmd/frontend/internal/dotcom/productsubscription/subscriptions_graphql.go
View File

@ -13,11 +13,11 @@ import (
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend"
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/graphqlutil"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/auth"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/errcode"
"github.com/sourcegraph/sourcegraph/internal/gqlutil"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/cmd/frontend/internal/dotcom/productsubscription/tokens_db.go
View File

@ -7,10 +7,10 @@ import (
"github.com/keegancsmith/sqlf"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/enterprise/internal/productsubscription"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/basestore"
"github.com/sourcegraph/sourcegraph/internal/licensing"
)
type dbTokens struct {

2
enterprise/cmd/frontend/internal/dotcom/productsubscription/tokens_db_test.go
View File

@ -10,10 +10,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/internal/license"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/enterprise/internal/productsubscription"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/dbtest"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/timeutil"
)

2
enterprise/cmd/frontend/internal/dotcom/productsubscription/tokens_graphql.go
View File

@ -5,7 +5,7 @@ import (
"fmt"
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/errcode"
)

1
enterprise/cmd/frontend/internal/embeddings/resolvers/BUILD.bazel generated
View File

@ -49,6 +49,7 @@ go_test(
"//internal/featureflag",
"//internal/gitserver",
"//internal/types",
"//lib/pointers",
"//schema",
"@com_github_graph_gophers_graphql_go//:graphql-go",
"@com_github_sourcegraph_log//logtest",

7
enterprise/cmd/frontend/internal/embeddings/resolvers/resolvers_test.go
View File

@ -21,6 +21,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/featureflag"
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"github.com/sourcegraph/sourcegraph/schema"
)
@ -71,12 +72,10 @@ func TestEmbeddingSearchResolver(t *testing.T) {
repoEmbeddingJobsStore,
)
truePtr := true
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Embeddings: &schema.Embeddings{Enabled: true},
Completions: &schema.Completions{},
CodyEnabled: pointers.Ptr(true),
LicenseKey: "asdf",
},
})

24
enterprise/cmd/worker/internal/embeddings/repo/handler.go
View File

@ -38,9 +38,6 @@ const (
embedEntireFileTokensThreshold = 384
embeddingChunkTokensThreshold = 256
embeddingChunkEarlySplitTokensThreshold = embeddingChunkTokensThreshold - 32
defaultMaxCodeEmbeddingsPerRepo = 3_072_000
defaultMaxTextEmbeddingsPerRepo = 512_000
)
var splitOptions = codeintelContext.SplitOptions{
@ -50,7 +47,8 @@ var splitOptions = codeintelContext.SplitOptions{
}
func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.RepoEmbeddingJob) error {
if !conf.EmbeddingsEnabled() {
embeddingsConfig := conf.GetEmbeddingsConfig(conf.Get().SiteConfig())
if embeddingsConfig == nil {
return errors.New("embeddings are not configured or disabled")
}
@ -61,7 +59,7 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.
return err
}
embeddingsClient, err := embed.NewEmbeddingsClient(&conf.Get().SiteConfiguration)
embeddingsClient, err := embed.NewEmbeddingsClient(embeddingsConfig)
if err != nil {
return err
}
@ -71,7 +69,7 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.
// otherwise we fall back to a full index.
var lastSuccessfulJobRevision api.CommitID
var previousIndex *embeddings.RepoEmbeddingIndex
if conf.Get().Embeddings.Incremental == nil || *conf.Get().Embeddings.Incremental {
if embeddingsConfig.Incremental {
lastSuccessfulJobRevision, previousIndex = h.getPreviousEmbeddingIndex(ctx, logger, repo)
if previousIndex != nil && !previousIndex.IsModelCompatible(embeddingsClient.GetModelIdentifier()) {
@ -86,17 +84,16 @@ func (h *handler) Handle(ctx context.Context, logger log.Logger, record *bgrepo.
gitserver: h.gitserverClient,
}
config := conf.Get().Embeddings
excludedGlobPatterns := embed.GetDefaultExcludedFilePathPatterns()
excludedGlobPatterns = append(excludedGlobPatterns, embed.CompileGlobPatterns(config.ExcludedFilePathPatterns)...)
excludedGlobPatterns = append(excludedGlobPatterns, embed.CompileGlobPatterns(embeddingsConfig.ExcludedFilePathPatterns)...)
opts := embed.EmbedRepoOpts{
RepoName: repo.Name,
Revision: record.Revision,
ExcludePatterns: excludedGlobPatterns,
SplitOptions: splitOptions,
MaxCodeEmbeddings: defaultTo(config.MaxCodeEmbeddingsPerRepo, defaultMaxCodeEmbeddingsPerRepo),
MaxTextEmbeddings: defaultTo(config.MaxTextEmbeddingsPerRepo, defaultMaxTextEmbeddingsPerRepo),
MaxCodeEmbeddings: embeddingsConfig.MaxCodeEmbeddingsPerRepo,
MaxTextEmbeddings: embeddingsConfig.MaxTextEmbeddingsPerRepo,
IndexedRevision: lastSuccessfulJobRevision,
}
@ -166,13 +163,6 @@ func (h *handler) getPreviousEmbeddingIndex(ctx context.Context, logger log.Logg
return lastSuccessfulJob.Revision, index
}
func defaultTo(input, def int) int {
if input == 0 {
return def
}
return input
}
type revisionFetcher struct {
repo api.RepoName
revision api.CommitID

5
enterprise/internal/cody/feature_flag_test.go
View File

@ -49,7 +49,6 @@ func TestIsCodyEnabled(t *testing.T) {
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Completions: &schema.Completions{},
},
})
t.Cleanup(func() {
@ -82,7 +81,6 @@ func TestIsCodyEnabled(t *testing.T) {
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &falsePtr,
Completions: &schema.Completions{},
},
})
t.Cleanup(func() {
@ -114,7 +112,6 @@ func TestIsCodyEnabled(t *testing.T) {
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Completions: &schema.Completions{},
CodyRestrictUsersFeatureFlag: &truePtr,
},
})
@ -137,7 +134,6 @@ func TestIsCodyEnabled(t *testing.T) {
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Completions: &schema.Completions{},
CodyRestrictUsersFeatureFlag: &truePtr,
},
})
@ -164,7 +160,6 @@ func TestIsCodyEnabled(t *testing.T) {
conf.Mock(&conf.Unified{
SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Completions: &schema.Completions{},
},
})
t.Cleanup(func() {

12
enterprise/internal/codygateway/BUILD.bazel generated
View File

@ -1,12 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "codygateway",
srcs = [
"consts.go",
"types.go",
],
importpath = "github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway",
visibility = ["//enterprise:__subpackages__"],
deps = ["//enterprise/internal/completions/types"],
)

18
enterprise/internal/completions/client/BUILD.bazel generated
View File

@ -10,23 +10,9 @@ go_library(
"//enterprise/internal/completions/client/anthropic",
"//enterprise/internal/completions/client/codygateway",
"//enterprise/internal/completions/client/openai",
"//enterprise/internal/completions/types",
"//enterprise/internal/dotcomuser",
"//enterprise/internal/licensing",
"//internal/conf/deploy",
"//internal/completions/types",
"//internal/conf/conftypes",
"//internal/httpcli",
"//lib/errors",
"//schema",
],
)
go_test(
name = "client_test",
srcs = ["client_test.go"],
embed = [":client"],
deps = [
"//internal/conf/deploy",
"//schema",
"@com_github_hexops_autogold_v2//:autogold",
],
)

4
enterprise/internal/completions/client/anthropic/BUILD.bazel generated
View File

@ -11,7 +11,7 @@ go_library(
importpath = "github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/anthropic",
visibility = ["//enterprise:__subpackages__"],
deps = [
"//enterprise/internal/completions/types",
"//internal/completions/types",
"//internal/httpcli",
"//lib/errors",
],
@ -28,7 +28,7 @@ go_test(
data = glob(["testdata/**"]),
embed = [":anthropic"],
deps = [
"//enterprise/internal/completions/types",
"//internal/completions/types",
"@com_github_hexops_autogold_v2//:autogold",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",

10
enterprise/internal/completions/client/anthropic/anthropic.go
View File

@ -6,17 +6,12 @@ import (
"encoding/json"
"net/http"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
const ProviderName = "anthropic"
func NewClient(cli httpcli.Doer, apiURL, accessToken string) types.CompletionsClient {
if apiURL == "" {
apiURL = defaultAPIURL
}
return &anthropicClient{
cli: cli,
accessToken: accessToken,
@ -25,8 +20,7 @@ func NewClient(cli httpcli.Doer, apiURL, accessToken string) types.CompletionsCl
}
const (
defaultAPIURL = "https://api.anthropic.com/v1/complete"
clientID = "sourcegraph/1.0"
clientID = "sourcegraph/1.0"
)
type anthropicClient struct {

2
enterprise/internal/completions/client/anthropic/anthropic_test.go
View File

@ -12,7 +12,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
)
type mockDoer struct {

2
enterprise/internal/completions/client/anthropic/prompt.go
View File

@ -3,7 +3,7 @@ package anthropic
import (
"strings"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

2
enterprise/internal/completions/client/anthropic/prompt_test.go
View File

@ -3,7 +3,7 @@ package anthropic
import (
"testing"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
)
func TestGetPrompt(t *testing.T) {

99
enterprise/internal/completions/client/client.go
View File

@ -4,108 +4,21 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/anthropic"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/openai"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/enterprise/internal/dotcomuser"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
)
func Get(endpoint, provider, accessToken string) (types.CompletionsClient, error) {
func Get(endpoint string, provider conftypes.CompletionsProviderName, accessToken string) (types.CompletionsClient, error) {
switch provider {
case anthropic.ProviderName:
case conftypes.CompletionsProviderNameAnthropic:
return anthropic.NewClient(httpcli.ExternalDoer, endpoint, accessToken), nil
case openai.ProviderName:
case conftypes.CompletionsProviderNameOpenAI:
return openai.NewClient(httpcli.ExternalDoer, endpoint, accessToken), nil
case codygateway.ProviderName:
case conftypes.CompletionsProviderNameSourcegraph:
return codygateway.NewClient(httpcli.ExternalDoer, endpoint, accessToken)
default:
return nil, errors.Newf("unknown completion stream provider: %s", provider)
}
}
// GetCompletionsConfig evaluates a complete completions configuration based on
// site configuration. The configuration may be nil if completions is disabled.
func GetCompletionsConfig(siteConfig schema.SiteConfiguration) *schema.Completions {
codyEnabled := siteConfig.CodyEnabled
completionsConfig := siteConfig.Completions
// If `cody.enabled` is used but no completions config, we assume defaults
if codyEnabled != nil && *codyEnabled {
if completionsConfig == nil {
completionsConfig = &schema.Completions{}
}
// Since `cody.enabled` is true, we override the `completions.enabled` value
completionsConfig.Enabled = true
}
// Otherwise, if we don't have a config, or it's disabled, we return nil
if completionsConfig == nil || (completionsConfig != nil && !completionsConfig.Enabled) {
return nil
}
// If a provider is not set, or if the provider is Cody Gateway, set up
// magic defaults. Note that we do NOT enable completions for the user -
// that still needs to be explicitly configured.
if completionsConfig.Provider == "" || completionsConfig.Provider == codygateway.ProviderName {
// Set provider to Cody Gateway in case it's empty.
completionsConfig.Provider = codygateway.ProviderName
// Configure accessToken. We don't validate the license here because
// Cody Gateway will check and reject the request.
if completionsConfig.AccessToken == "" {
switch deploy.Type() {
case deploy.App:
if siteConfig.App != nil && siteConfig.App.DotcomAuthToken != "" {
completionsConfig.AccessToken = dotcomuser.GenerateDotcomUserGatewayAccessToken(siteConfig.App.DotcomAuthToken)
}
default:
if siteConfig.LicenseKey != "" {
completionsConfig.AccessToken = licensing.GenerateLicenseKeyBasedAccessToken(siteConfig.LicenseKey)
}
}
}
// Configure endpoint
if completionsConfig.Endpoint == "" {
completionsConfig.Endpoint = codygateway.DefaultEndpoint
}
// Configure chatModel
if completionsConfig.ChatModel == "" {
completionsConfig.ChatModel = "anthropic/claude-v1"
}
// Configure completionModel
if completionsConfig.CompletionModel == "" {
completionsConfig.CompletionModel = "anthropic/claude-instant-v1"
}
// NOTE: We explicitly aren't adding back-compat for completions.model
// because Cody Gateway disallows the use of most chat models for
// code completions, so in most cases the back-compat wouldn't work
// anyway.
return completionsConfig
}
if completionsConfig.ChatModel == "" {
// If no model for chat is configured, nothing we can do.
if completionsConfig.Model == "" {
return nil
}
completionsConfig.ChatModel = completionsConfig.Model
}
// TODO: Temporary workaround to fix instances where no completion model is set.
if completionsConfig.CompletionModel == "" {
completionsConfig.CompletionModel = "claude-instant-v1"
}
if completionsConfig.FastChatModel == "" {
completionsConfig.FastChatModel = completionsConfig.CompletionModel
}
return completionsConfig
}

179
enterprise/internal/completions/client/client_test.go
View File

@ -1,179 +0,0 @@
package client
import (
"testing"
"github.com/hexops/autogold/v2"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/schema"
)
func TestGetCompletionsConfig(t *testing.T) {
truePtr := true
for _, tc := range []struct {
name string
deployType string
config schema.SiteConfiguration
want autogold.Value
}{
{
name: "cody not enabled",
config: schema.SiteConfiguration{
CodyEnabled: nil,
},
want: autogold.Expect((*schema.Completions)(nil)),
},
{
name: "anthropic completions",
config: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Completions: &schema.Completions{
Enabled: true,
Provider: "anthropic",
ChatModel: "claude-v1",
FastChatModel: "claude-instant-v1",
CompletionModel: "claude-instant-v1",
},
},
want: autogold.Expect(&schema.Completions{
Enabled: true,
ChatModel: "claude-v1",
FastChatModel: "claude-instant-v1",
CompletionModel: "claude-instant-v1",
Provider: "anthropic",
}),
},
{
name: "anthropic completions, with cody.enabled taking precedence over completions.enabled",
config: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Completions: &schema.Completions{
Enabled: false,
Provider: "anthropic",
ChatModel: "claude-v1",
CompletionModel: "claude-instant-v1",
},
},
want: autogold.Expect(&schema.Completions{
ChatModel: "claude-v1", CompletionModel: "claude-instant-v1",
Enabled: true,
FastChatModel: "claude-instant-v1",
Provider: "anthropic",
}),
},
{
name: "zero-config cody gateway completions without license key",
config: schema.SiteConfiguration{
CodyEnabled: &truePtr,
LicenseKey: "",
},
want: autogold.Expect(&schema.Completions{
ChatModel: "anthropic/claude-v1",
CompletionModel: "anthropic/claude-instant-v1",
Enabled: true,
Endpoint: "https://cody-gateway.sourcegraph.com",
Provider: "sourcegraph",
}),
},
{
name: "zero-config cody gateway completions with license key",
config: schema.SiteConfiguration{
CodyEnabled: &truePtr,
LicenseKey: "foobar",
},
want: autogold.Expect(&schema.Completions{
Enabled: true,
AccessToken: "slk_c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
ChatModel: "anthropic/claude-v1",
CompletionModel: "anthropic/claude-instant-v1",
Endpoint: "https://cody-gateway.sourcegraph.com",
Provider: "sourcegraph",
}),
},
{
// Legacy support for completions.enabled
name: "legacy field completions.enabled: zero-config cody gateway completions without license key",
config: schema.SiteConfiguration{
Completions: &schema.Completions{Enabled: true},
LicenseKey: "",
},
want: autogold.Expect(&schema.Completions{
Enabled: true,
ChatModel: "anthropic/claude-v1",
CompletionModel: "anthropic/claude-instant-v1",
Endpoint: "https://cody-gateway.sourcegraph.com",
Provider: "sourcegraph",
}),
},
{
// Legacy support for completions.enabled
name: "legacy field completions.enabled: zero-config cody gateway completions with license key",
config: schema.SiteConfiguration{
Completions: &schema.Completions{Enabled: true},
LicenseKey: "foobar",
},
want: autogold.Expect(&schema.Completions{
Enabled: true,
AccessToken: "slk_c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2",
ChatModel: "anthropic/claude-v1",
CompletionModel: "anthropic/claude-instant-v1",
Endpoint: "https://cody-gateway.sourcegraph.com",
Provider: "sourcegraph",
}),
},
{
name: "app zero-config cody gateway completions with dotcom token",
deployType: deploy.App,
config: schema.SiteConfiguration{
CodyEnabled: &truePtr,
App: &schema.App{
DotcomAuthToken: "TOKEN",
},
},
want: autogold.Expect(&schema.Completions{
AccessToken: "sgd_5df6e0e2761359d30a8275058e299fcc0381534545f55cf43e41983f5d4c9456",
ChatModel: "anthropic/claude-v1",
CompletionModel: "anthropic/claude-instant-v1",
Enabled: true,
Endpoint: "https://cody-gateway.sourcegraph.com",
Provider: "sourcegraph",
}),
},
{
name: "app with custom configuration",
deployType: deploy.App,
config: schema.SiteConfiguration{
CodyEnabled: &truePtr,
Completions: &schema.Completions{
AccessToken: "CUSTOM_TOKEN",
Provider: "anthropic",
ChatModel: "claude-v1",
FastChatModel: "claude-instant-v1",
CompletionModel: "claude-instant-v1",
},
App: &schema.App{
DotcomAuthToken: "TOKEN",
},
},
want: autogold.Expect(&schema.Completions{
AccessToken: "CUSTOM_TOKEN", ChatModel: "claude-v1",
CompletionModel: "claude-instant-v1",
Enabled: true,
FastChatModel: "claude-instant-v1",
Provider: "anthropic",
}),
},
} {
t.Run(tc.name, func(t *testing.T) {
defaultDeploy := deploy.Type()
if tc.deployType != "" {
deploy.Mock(tc.deployType)
}
defer deploy.Mock(defaultDeploy)
got := GetCompletionsConfig(tc.config)
tc.want.Equal(t, got)
})
}
}

7
enterprise/internal/completions/client/codygateway/BUILD.bazel generated
View File

@ -7,10 +7,11 @@ go_library(
importpath = "github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/codygateway",
visibility = ["//enterprise:__subpackages__"],
deps = [
"//enterprise/internal/codygateway",
"//enterprise/internal/completions/client/anthropic",
"//enterprise/internal/completions/client/openai",
"//enterprise/internal/completions/types",
"//internal/codygateway",
"//internal/completions/types",
"//internal/conf/conftypes",
"//internal/httpcli",
"//lib/errors",
"@io_opentelemetry_go_otel//attribute",
@ -23,7 +24,7 @@ go_test(
srcs = ["codygateway_test.go"],
embed = [":codygateway"],
deps = [
"//enterprise/internal/completions/types",
"//internal/completions/types",
"//lib/errors",
"@com_github_hexops_autogold_v2//:autogold",
"@com_github_stretchr_testify//assert",

19
enterprise/internal/completions/client/codygateway/codygateway.go
View File

@ -10,21 +10,15 @@ import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/anthropic"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/openai"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
const (
// ProviderName is 'sourcegraph', since this is a Sourcegraph-provided service,
// backed by Cody Gateway. This is the value accepted in site configuration.
ProviderName = "sourcegraph"
DefaultEndpoint = "https://cody-gateway.sourcegraph.com"
)
// NewClient instantiates a completions provider backed by Sourcegraph's managed
// Cody Gateway service.
func NewClient(cli httpcli.Doer, endpoint, accessToken string) (types.CompletionsClient, error) {
@ -32,9 +26,6 @@ func NewClient(cli httpcli.Doer, endpoint, accessToken string) (types.Completion
if err != nil {
return nil, err
}
if accessToken == "" {
return nil, errors.New("no access token is configured - make sure `licenseKey` is set, or provide an access token in `completions.accessToken`")
}
return &codyGatewayClient{
upstream: cli,
gatewayURL: gatewayURL,
@ -86,9 +77,9 @@ func (c *codyGatewayClient) clientForParams(feature types.CompletionsFeature, re
// Based on the provider, instantiate the appropriate client backed by a
// gatewayDoer that authenticates against the Gateway's API.
switch provider {
case anthropic.ProviderName:
case string(conftypes.CompletionsProviderNameAnthropic):
return anthropic.NewClient(gatewayDoer(c.upstream, feature, c.gatewayURL, c.accessToken, "/v1/completions/anthropic"), "", ""), nil
case openai.ProviderName:
case string(conftypes.CompletionsProviderNameOpenAI):
return openai.NewClient(gatewayDoer(c.upstream, feature, c.gatewayURL, c.accessToken, "/v1/completions/openai"), "", ""), nil
case "":
return nil, errors.Newf("no provider provided in model %s - a model in the format '$PROVIDER/$MODEL_NAME' is expected", model)

2
enterprise/internal/completions/client/codygateway/codygateway_test.go
View File

@ -8,7 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/lib/errors"
)

4
enterprise/internal/completions/client/openai/BUILD.bazel generated
View File

@ -10,7 +10,7 @@ go_library(
importpath = "github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client/openai",
visibility = ["//enterprise:__subpackages__"],
deps = [
"//enterprise/internal/completions/types",
"//internal/completions/types",
"//internal/httpcli",
"//lib/errors",
],
@ -24,7 +24,7 @@ go_test(
],
embed = [":openai"],
deps = [
"//enterprise/internal/completions/types",
"//internal/completions/types",
"@com_github_hexops_autogold_v2//:autogold",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",

9
enterprise/internal/completions/client/openai/openai.go
View File

@ -7,17 +7,12 @@ import (
"net/http"
"strings"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
const ProviderName = "openai"
func NewClient(cli httpcli.Doer, endpoint, accessToken string) types.CompletionsClient {
if endpoint == "" {
endpoint = defaultAPIURL
}
return &openAIChatCompletionStreamClient{
cli: cli,
accessToken: accessToken,
@ -25,8 +20,6 @@ func NewClient(cli httpcli.Doer, endpoint, accessToken string) types.Completions
}
}
const defaultAPIURL = "https://api.openai.com/v1/chat/completions"
type openAIChatCompletionStreamClient struct {
cli httpcli.Doer
accessToken string

2
enterprise/internal/completions/client/openai/openai_test.go
View File

@ -11,7 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
)
type mockDoer struct {

4
enterprise/internal/completions/httpapi/BUILD.bazel generated
View File

@ -14,10 +14,11 @@ go_library(
deps = [
"//enterprise/internal/cody",
"//enterprise/internal/completions/client",
"//enterprise/internal/completions/types",
"//internal/actor",
"//internal/auth",
"//internal/completions/types",
"//internal/conf",
"//internal/conf/conftypes",
"//internal/database",
"//internal/honey",
"//internal/redispool",
@ -25,7 +26,6 @@ go_library(
"//internal/search/streaming/http",
"//internal/trace",
"//lib/errors",
"//schema",
"@com_github_gomodule_redigo//redis",
"@com_github_sourcegraph_log//:log",
],

6
enterprise/internal/completions/httpapi/codecompletion.go
View File

@ -7,11 +7,11 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/redispool"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/schema"
)
// NewCodeCompletionsHandler is an http handler which sends back code completion results
@ -19,7 +19,7 @@ func NewCodeCompletionsHandler(logger log.Logger, db database.DB) http.Handler {
logger = logger.Scoped("code", "code completions handler")
rl := NewRateLimiter(db, redispool.Store, types.CompletionsFeatureCode)
return newCompletionsHandler(rl, "code", func(requestParams types.CodyCompletionRequestParameters, c *schema.Completions) string {
return newCompletionsHandler(rl, "code", func(requestParams types.CodyCompletionRequestParameters, c *conftypes.CompletionsConfig) string {
// No user defined models for now.
// TODO(eseliger): Look into reviving this, but it was unused so far.
return c.CompletionModel

8
enterprise/internal/completions/httpapi/handler.go
View File

@ -10,9 +10,9 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/internal/cody"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/client"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/schema"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
)
// maxRequestDuration is the maximum amount of time a request can take before
@ -22,7 +22,7 @@ const maxRequestDuration = time.Minute
func newCompletionsHandler(
rl RateLimiter,
traceFamily string,
getModel func(types.CodyCompletionRequestParameters, *schema.Completions) string,
getModel func(types.CodyCompletionRequestParameters, *conftypes.CompletionsConfig) string,
handle func(context.Context, types.CompletionRequestParameters, types.CompletionsClient, http.ResponseWriter),
) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -39,7 +39,7 @@ func newCompletionsHandler(
return
}
completionsConfig := client.GetCompletionsConfig(conf.Get().SiteConfig())
completionsConfig := conf.GetCompletionsConfig(conf.Get().SiteConfig())
if completionsConfig == nil {
http.Error(w, "completions are not configured or disabled", http.StatusInternalServerError)
}

12
enterprise/internal/completions/httpapi/limiter.go
View File

@ -7,9 +7,9 @@ import (
"github.com/gomodule/redigo/redis"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/auth"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/redispool"
@ -178,15 +178,15 @@ func getConfiguredLimit(ctx context.Context, db database.DB, scope types.Complet
}
// Otherwise, fall back to the global limit.
cfg := conf.Get()
cfg := conf.GetCompletionsConfig(conf.Get().SiteConfig())
switch scope {
case types.CompletionsFeatureChat:
if cfg.Completions != nil && cfg.Completions.PerUserDailyLimit > 0 {
return cfg.Completions.PerUserDailyLimit, nil
if cfg != nil && cfg.PerUserDailyLimit > 0 {
return cfg.PerUserDailyLimit, nil
}
case types.CompletionsFeatureCode:
if cfg.Completions != nil && cfg.Completions.PerUserCodeCompletionsDailyLimit > 0 {
return cfg.Completions.PerUserCodeCompletionsDailyLimit, nil
if cfg != nil && cfg.PerUserCodeCompletionsDailyLimit > 0 {
return cfg.PerUserCodeCompletionsDailyLimit, nil
}
default:
return 0, errors.Newf("unknown scope: %s", scope)

6
enterprise/internal/completions/httpapi/stream.go
View File

@ -6,12 +6,12 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/redispool"
streamhttp "github.com/sourcegraph/sourcegraph/internal/search/streaming/http"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/schema"
)
// NewChatCompletionsStreamHandler is an http handler which streams back completions results.
@ -19,7 +19,7 @@ func NewChatCompletionsStreamHandler(logger log.Logger, db database.DB) http.Han
logger = logger.Scoped("chat", "chat completions handler")
rl := NewRateLimiter(db, redispool.Store, types.CompletionsFeatureChat)
return newCompletionsHandler(rl, "chat", func(requestParams types.CodyCompletionRequestParameters, c *schema.Completions) string {
return newCompletionsHandler(rl, "chat", func(requestParams types.CodyCompletionRequestParameters, c *conftypes.CompletionsConfig) string {
// No user defined models for now.
if requestParams.Fast {
return c.FastChatModel

2
enterprise/internal/embeddings/background/repo/BUILD.bazel generated
View File

@ -13,7 +13,6 @@ go_library(
deps = [
"//internal/api",
"//internal/conf",
"//internal/conf/conftypes",
"//internal/database",
"//internal/database/basestore",
"//internal/database/dbutil",
@ -43,6 +42,7 @@ go_test(
"//internal/database",
"//internal/database/dbtest",
"//internal/types",
"//lib/pointers",
"//schema",
"@com_github_keegancsmith_sqlf//:sqlf",
"@com_github_sourcegraph_log//logtest",

47
enterprise/internal/embeddings/background/repo/store.go
View File

@ -12,7 +12,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/basestore"
"github.com/sourcegraph/sourcegraph/internal/database/dbutil"
@ -203,51 +202,17 @@ type ListOpts struct {
State *string
}
func init() {
conf.ContributeValidator(embeddingConfigValidator)
}
func embeddingConfigValidator(q conftypes.SiteConfigQuerier) conf.Problems {
embeddingsConf := q.SiteConfig().Embeddings
if embeddingsConf == nil {
return nil
}
minimumIntervalString := embeddingsConf.MinimumInterval
_, err := time.ParseDuration(minimumIntervalString)
if err != nil && minimumIntervalString != "" {
return conf.NewSiteProblems(fmt.Sprintf("Could not parse \"embeddings.minimumInterval: %s\". %s", minimumIntervalString, err))
}
return nil
}
var defaultPolicyRepositoryMatchLimit = 5000
var defaultOpts = EmbeddableRepoOpts{
MinimumInterval: 24 * time.Hour,
PolicyRepositoryMatchLimit: &defaultPolicyRepositoryMatchLimit,
}
func GetEmbeddableRepoOpts() EmbeddableRepoOpts {
opts := defaultOpts
embeddingsConf := conf.Get().Embeddings
embeddingsConf := conf.GetEmbeddingsConfig(conf.Get().SiteConfig())
// Embeddings are disabled, nothing we can do.
if embeddingsConf == nil {
return opts
return EmbeddableRepoOpts{}
}
minimumIntervalString := embeddingsConf.MinimumInterval
d, err := time.ParseDuration(minimumIntervalString)
if err == nil {
opts.MinimumInterval = d
return EmbeddableRepoOpts{
MinimumInterval: embeddingsConf.MinimumInterval,
PolicyRepositoryMatchLimit: embeddingsConf.PolicyRepositoryMatchLimit,
}
if embeddingsConf.PolicyRepositoryMatchLimit != nil {
opts.PolicyRepositoryMatchLimit = embeddingsConf.PolicyRepositoryMatchLimit
}
return opts
}
func (s *repoEmbeddingJobsStore) GetEmbeddableRepos(ctx context.Context, opts EmbeddableRepoOpts) ([]EmbeddableRepo, error) {

24
enterprise/internal/embeddings/background/repo/store_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/database/dbtest"
"github.com/sourcegraph/sourcegraph/internal/types"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"github.com/sourcegraph/sourcegraph/schema"
)
@ -305,18 +306,29 @@ func TestGetEmbeddableReposLimit(t *testing.T) {
func TestGetEmbeddableRepoOpts(t *testing.T) {
conf.Mock(&conf.Unified{})
defer conf.Mock(nil)
conf.Mock(&conf.Unified{SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: "asdf",
}})
opts := GetEmbeddableRepoOpts()
require.Equal(t, defaultOpts.MinimumInterval, opts.MinimumInterval)
require.Equal(t, *defaultOpts.PolicyRepositoryMatchLimit, *opts.PolicyRepositoryMatchLimit)
require.Equal(t, 24*time.Hour, opts.MinimumInterval)
require.Equal(t, 5000, *opts.PolicyRepositoryMatchLimit)
conf.Mock(&conf.Unified{SiteConfiguration: schema.SiteConfiguration{Embeddings: &schema.Embeddings{}}})
opts = GetEmbeddableRepoOpts()
require.Equal(t, defaultOpts.MinimumInterval, opts.MinimumInterval)
require.Equal(t, *defaultOpts.PolicyRepositoryMatchLimit, *opts.PolicyRepositoryMatchLimit)
require.Equal(t, 24*time.Hour, opts.MinimumInterval)
require.Equal(t, 5000, *opts.PolicyRepositoryMatchLimit)
limit := 5
conf.Mock(&conf.Unified{SiteConfiguration: schema.SiteConfiguration{Embeddings: &schema.Embeddings{MinimumInterval: "1h", PolicyRepositoryMatchLimit: &limit}}})
conf.Mock(&conf.Unified{SiteConfiguration: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
Embeddings: &schema.Embeddings{
Provider: "openai",
AccessToken: "asdf",
MinimumInterval: "1h",
PolicyRepositoryMatchLimit: &limit,
}},
})
opts = GetEmbeddableRepoOpts()
require.Equal(t, 1*time.Hour, opts.MinimumInterval)

5
enterprise/internal/embeddings/embed/BUILD.bazel generated
View File

@ -21,9 +21,8 @@ go_library(
"//internal/api",
"//internal/binary",
"//internal/codeintel/types",
"//internal/conf/deploy",
"//internal/conf/conftypes",
"//lib/errors",
"//schema",
"@com_github_sourcegraph_log//:log",
],
)
@ -43,9 +42,7 @@ go_test(
"//enterprise/internal/embeddings/embed/client",
"//internal/api",
"//internal/codeintel/types",
"//internal/conf/deploy",
"//lib/errors",
"//schema",
"@com_github_sourcegraph_log//:log",
"@com_github_stretchr_testify//require",
],

2
enterprise/internal/embeddings/embed/client/openai/BUILD.bazel generated
View File

@ -6,8 +6,8 @@ go_library(
importpath = "github.com/sourcegraph/sourcegraph/enterprise/internal/embeddings/embed/client/openai",
visibility = ["//enterprise:__subpackages__"],
deps = [
"//internal/conf/conftypes",
"//internal/httpcli",
"//lib/errors",
"//schema",
],
)

38
enterprise/internal/embeddings/embed/client/openai/client.go
View File

@ -12,54 +12,28 @@ import (
"strings"
"time"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
)
func NewClient(config *schema.Embeddings) *openaiEmbeddingsClient {
func NewClient(config *conftypes.EmbeddingsConfig) *openaiEmbeddingsClient {
return &openaiEmbeddingsClient{
dimensions: config.Dimensions,
accessToken: config.AccessToken,
model: getModel(config),
url: getURL(config),
model: config.Model,
endpoint: config.Endpoint,
}
}
func getModel(config *schema.Embeddings) string {
if config.Model == "" {
return "text-embedding-ada-002"
}
return strings.ToLower(config.Model)
}
const defaultAPIURL = "https://api.openai.com/v1/embeddings"
func getURL(config *schema.Embeddings) string {
url := config.Endpoint
// Fallback to URL, it's the previous name of the setting.
if url == "" {
url = config.Url
}
// If that is also not set, use a sensible default.
if url == "" {
url = defaultAPIURL
}
return url
}
type openaiEmbeddingsClient struct {
model string
dimensions int
url string
endpoint string
accessToken string
}
func (c *openaiEmbeddingsClient) GetDimensions() (int, error) {
// Use some good default for the only model we supported so far.
if c.dimensions == 0 && strings.EqualFold(c.model, "text-embedding-ada-002") {
return 1536, nil
}
if c.dimensions <= 0 {
return 0, errors.New("invalid config for embeddings.dimensions, must be > 0")
}
@ -119,7 +93,7 @@ func (c *openaiEmbeddingsClient) getEmbeddings(ctx context.Context, texts []stri
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewReader(bodyBytes))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(bodyBytes))
if err != nil {
return nil, err
}

7
enterprise/internal/embeddings/embed/client/sourcegraph/BUILD.bazel generated
View File

@ -6,13 +6,10 @@ go_library(
importpath = "github.com/sourcegraph/sourcegraph/enterprise/internal/embeddings/embed/client/sourcegraph",
visibility = ["//enterprise:__subpackages__"],
deps = [
"//enterprise/internal/codygateway",
"//enterprise/internal/dotcomuser",
"//enterprise/internal/embeddings/embed/client",
"//enterprise/internal/licensing",
"//internal/conf/deploy",
"//internal/codygateway",
"//internal/conf/conftypes",
"//internal/httpcli",
"//lib/errors",
"//schema",
],
)

70
enterprise/internal/embeddings/embed/client/sourcegraph/client.go
View File

@ -13,74 +13,30 @@ import (
"strings"
"time"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codygateway"
"github.com/sourcegraph/sourcegraph/enterprise/internal/dotcomuser"
"github.com/sourcegraph/sourcegraph/enterprise/internal/embeddings/embed/client"
"github.com/sourcegraph/sourcegraph/enterprise/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/codygateway"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
)
const defaultModel = "openai/text-embedding-ada-002"
func NewClient(config *schema.SiteConfiguration) *sourcegraphEmbeddingsClient {
func NewClient(config *conftypes.EmbeddingsConfig) *sourcegraphEmbeddingsClient {
return &sourcegraphEmbeddingsClient{
model: getModel(config),
dimensions: config.Embeddings.Dimensions,
url: getURL(config.Embeddings),
accessToken: getAccessToken(config),
model: config.Model,
dimensions: config.Dimensions,
endpoint: config.Endpoint,
accessToken: config.AccessToken,
}
}
const defaultAPIURL = "https://cody-gateway.sourcegraph.com/v1/embeddings"
func getModel(config *schema.SiteConfiguration) string {
if config.Embeddings == nil || config.Embeddings.Model == "" {
return defaultModel
}
return strings.ToLower(config.Embeddings.Model)
}
func getAccessToken(config *schema.SiteConfiguration) string {
// If an access token is configured, use it.
if config.Embeddings.AccessToken != "" {
return config.Embeddings.AccessToken
}
// App generates a token from the api token the user used to connect app to dotcom.
if deploy.IsApp() && config.App != nil {
return dotcomuser.GenerateDotcomUserGatewayAccessToken(config.App.DotcomAuthToken)
}
// Otherwise, use the current license key to compute an access token.
return licensing.GenerateLicenseKeyBasedAccessToken(config.LicenseKey)
}
func getURL(config *schema.Embeddings) string {
url := config.Endpoint
// Fallback to URL, it's the previous name of the setting.
if url == "" {
url = config.Url
}
// If that is also not set, use a sensible default.
if url == "" {
url = defaultAPIURL
}
return url
}
type sourcegraphEmbeddingsClient struct {
model string
dimensions int
url string
endpoint string
accessToken string
}
func (c *sourcegraphEmbeddingsClient) GetDimensions() (int, error) {
if c.dimensions <= 0 && strings.EqualFold(c.model, defaultModel) {
return 1536, nil
}
// TODO: Later, we should ideally ask the gateway for the dimensionality of the model
// so we don't have to hard-code defaults for all the models and can roll out new models
// to older instances, too.
@ -92,9 +48,11 @@ func (c *sourcegraphEmbeddingsClient) GetDimensions() (int, error) {
}
func (c *sourcegraphEmbeddingsClient) GetModelIdentifier() string {
// Special-case the default model, since it already includes the provider name
if strings.EqualFold(c.model, defaultModel) {
return defaultModel
// Special-case the default model, since it already includes the provider name.
// This ensures we can safely migrate customers from the OpenAI provider to
// Cody Gateway.
if strings.EqualFold(c.model, "openai/text-embedding-ada-002") {
return "openai/text-embedding-ada-002"
}
return fmt.Sprintf("sourcegraph/%s", c.model)
}
@ -148,7 +106,7 @@ func (c *sourcegraphEmbeddingsClient) getEmbeddings(ctx context.Context, texts [
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url, bytes.NewReader(bodyBytes))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(bodyBytes))
if err != nil {
return nil, err
}

38
enterprise/internal/embeddings/embed/embed.go
View File

@ -14,45 +14,21 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/internal/paths"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/codeintel/types"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
)
func NewEmbeddingsClient(siteConfig *schema.SiteConfiguration) (client.EmbeddingsClient, error) {
if !isEmbeddingsEnabled(siteConfig) {
return nil, errors.New("embeddings are not configured or disabled")
}
c := siteConfig.Embeddings
switch c.Provider {
func NewEmbeddingsClient(config *conftypes.EmbeddingsConfig) (client.EmbeddingsClient, error) {
switch config.Provider {
case "sourcegraph":
// TODO(eseliger): Readd empty string defaulting to sourcegraph.
// For a transition period until we have configured S2 and dotcom with the
// new config, this will have to be in here.
return sourcegraph.NewClient(siteConfig), nil
case "openai", "":
return openai.NewClient(c), nil
return sourcegraph.NewClient(config), nil
case "openai":
return openai.NewClient(config), nil
default:
return nil, errors.Newf("invalid provider %q", c.Provider)
return nil, errors.Newf("invalid provider %q", config.Provider)
}
}
func isEmbeddingsEnabled(siteConfig *schema.SiteConfiguration) bool {
c := siteConfig.Embeddings
if c == nil || !c.Enabled {
return false
}
// Additionally Embeddings in App are disabled if there is no dotcom auth token
// and the user hasn't provided their own api token
if deploy.IsApp() {
if (siteConfig.App == nil || len(siteConfig.App.DotcomAuthToken) == 0) && c.AccessToken == "" {
return false
}
}
return true
}
const (
getEmbeddingsMaxRetries = 5
embeddingsBatchSize = 512

99
enterprise/internal/embeddings/embed/embed_test.go
View File

@ -13,108 +13,9 @@ import (
"github.com/sourcegraph/sourcegraph/enterprise/internal/embeddings/embed/client"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/codeintel/types"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/schema"
)
func TestNewEmbeddingsClient(t *testing.T) {
testCases := []struct {
name string
siteConfig *schema.SiteConfiguration
deployType string
wantErr bool
}{
{
name: "Embeddings disabled",
siteConfig: &schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Enabled: false,
},
},
wantErr: true,
},
{
name: "Invalid provider",
siteConfig: &schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Enabled: true,
Provider: "invalid",
},
},
wantErr: true,
},
{
name: "Sourcegraph provider",
siteConfig: &schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Enabled: true,
Provider: "sourcegraph",
},
},
},
{
name: "App with dotcom token",
deployType: deploy.App,
siteConfig: &schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Enabled: true,
Provider: "sourcegraph",
},
App: &schema.App{
DotcomAuthToken: "TOKEN",
},
},
},
{
name: "App with user token",
deployType: deploy.App,
siteConfig: &schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Enabled: true,
Provider: "sourcegraph",
AccessToken: "TOKEN",
},
},
},
{
name: "App without dotcom or user token",
deployType: deploy.App,
siteConfig: &schema.SiteConfiguration{
Embeddings: &schema.Embeddings{
Enabled: true,
Provider: "sourcegraph",
},
},
wantErr: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
defaultDeploy := deploy.Type()
if tc.deployType != "" {
deploy.Mock(tc.deployType)
}
defer deploy.Mock(defaultDeploy)
client, err := NewEmbeddingsClient(tc.siteConfig)
if tc.wantErr {
if err == nil {
t.Fatal("expected error but got nil")
}
} else {
if err != nil {
t.Fatal(err)
}
if client == nil {
t.Fatal("expected client but got nil")
}
}
})
}
}
func mockFile(lines ...string) []byte {
return []byte(strings.Join(lines, "\n"))
}

5
enterprise/internal/licensing/BUILD.bazel generated
View File

@ -4,7 +4,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "licensing",
srcs = [
"access_token.go",
"check.go",
"codygateway.go",
"conf.go",
@ -26,8 +25,8 @@ go_library(
"//internal/env",
"//internal/errcode",
"//internal/goroutine",
"//internal/hashutil",
"//internal/httpcli",
"//internal/licensing",
"//internal/redispool",
"//lib/errors",
"@com_github_derision_test_glock//:glock",
@ -42,7 +41,6 @@ go_test(
name = "licensing_test",
timeout = "short",
srcs = [
"access_token_test.go",
"check_test.go",
"codygateway_test.go",
"features_test.go",
@ -61,7 +59,6 @@ go_test(
"@com_github_derision_test_glock//:glock",
"@com_github_gomodule_redigo//redis",
"@com_github_google_go_cmp//cmp",
"@com_github_hexops_autogold_v2//:autogold",
"@com_github_sourcegraph_log//logtest",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",

3
enterprise/internal/licensing/check.go
View File

@ -15,6 +15,7 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/redispool"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -154,7 +155,7 @@ func StartLicenseCheck(originalCtx context.Context, logger log.Logger, siteID st
ctxWithCancel, cancel = context.WithCancel(originalCtx)
prevLicenseToken, _ := store.Get(prevLicenseTokenKey).String()
licenseToken := GenerateLicenseKeyBasedAccessToken(conf.Get().LicenseKey)
licenseToken := licensing.GenerateLicenseKeyBasedAccessToken(conf.Get().LicenseKey)
var initialWaitInterval time.Duration = 0
if prevLicenseToken == licenseToken {
initialWaitInterval, _ = calcDurationSinceLastCalled(glock.NewRealClock())

12
internal/codygateway/BUILD.bazel generated Normal file
View File

@ -0,0 +1,12 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "codygateway",
srcs = [
"consts.go",
"types.go",
],
importpath = "github.com/sourcegraph/sourcegraph/internal/codygateway",
visibility = ["//:__subpackages__"],
deps = ["//internal/completions/types"],
)

0
enterprise/internal/codygateway/consts.go → internal/codygateway/consts.go
View File

2
enterprise/internal/codygateway/types.go → internal/codygateway/types.go
View File

@ -3,7 +3,7 @@ package codygateway
import (
"time"
"github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types"
"github.com/sourcegraph/sourcegraph/internal/completions/types"
)
type Feature string

6
enterprise/internal/completions/types/BUILD.bazel → internal/completions/types/BUILD.bazel generated
View File

@ -1,5 +1,5 @@
load("//dev:go_defs.bzl", "go_test")
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//dev:go_defs.bzl", "go_test")
go_library(
name = "types",
@ -7,8 +7,8 @@ go_library(
"errors.go",
"types.go",
],
importpath = "github.com/sourcegraph/sourcegraph/enterprise/internal/completions/types",
visibility = ["//enterprise:__subpackages__"],
importpath = "github.com/sourcegraph/sourcegraph/internal/completions/types",
visibility = ["//:__subpackages__"],
deps = ["//lib/errors"],
)

0
enterprise/internal/completions/types/errors.go → internal/completions/types/errors.go
View File

0
enterprise/internal/completions/types/errors_test.go → internal/completions/types/errors_test.go
View File

0
enterprise/internal/completions/types/types.go → internal/completions/types/types.go
View File

6
internal/conf/BUILD.bazel generated
View File

@ -6,6 +6,7 @@ go_library(
srcs = [
"auth.go",
"client.go",
"cody_validators.go",
"computed.go",
"conf.go",
"diff.go",
@ -27,14 +28,17 @@ go_library(
"//internal/conf/confdefaults",
"//internal/conf/conftypes",
"//internal/conf/deploy",
"//internal/dotcomuser",
"//internal/env",
"//internal/extsvc",
"//internal/hashutil",
"//internal/httpcli",
"//internal/jsonc",
"//internal/licensing",
"//internal/src-cli",
"//internal/version",
"//lib/errors",
"//lib/pointers",
"//schema",
"@com_github_getsentry_sentry_go//:sentry-go",
"@com_github_grafana_regexp//:regexp",
@ -61,6 +65,8 @@ go_test(
"//cmd/frontend/envvar",
"//internal/api/internalapi",
"//internal/conf/conftypes",
"//internal/conf/deploy",
"//internal/licensing",
"//lib/errors",
"//lib/pointers",
"//schema",

59
internal/conf/cody_validators.go Normal file
View File

@ -0,0 +1,59 @@
package conf
import (
"fmt"
"time"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
)
func init() {
ContributeValidator(completionsConfigValidator)
ContributeValidator(embeddingsConfigValidator)
}
func completionsConfigValidator(q conftypes.SiteConfigQuerier) Problems {
problems := []string{}
completionsConf := q.SiteConfig().Completions
if completionsConf == nil {
return nil
}
if completionsConf.Provider == "" {
problems = append(problems, "completions.provider is required")
}
if completionsConf.Enabled != nil && q.SiteConfig().CodyEnabled == nil {
problems = append(problems, "completions.enabled has been renamed to cody.enabled, please migrate")
}
if len(problems) > 0 {
return NewSiteProblems(problems...)
}
return nil
}
func embeddingsConfigValidator(q conftypes.SiteConfigQuerier) Problems {
problems := []string{}
embeddingsConf := q.SiteConfig().Embeddings
if embeddingsConf == nil {
return nil
}
if embeddingsConf.Provider == "" {
problems = append(problems, "embeddings.provider is required")
}
minimumIntervalString := embeddingsConf.MinimumInterval
_, err := time.ParseDuration(minimumIntervalString)
if err != nil && minimumIntervalString != "" {
problems = append(problems, fmt.Sprintf("Could not parse \"embeddings.minimumInterval: %s\". %s", minimumIntervalString, err))
}
if len(problems) > 0 {
return NewSiteProblems(problems...)
}
return nil
}

347
internal/conf/computed.go
View File

@ -13,10 +13,13 @@ import (
"github.com/sourcegraph/sourcegraph/internal/conf/confdefaults"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/dotcomuser"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
"github.com/sourcegraph/sourcegraph/internal/licensing"
srccli "github.com/sourcegraph/sourcegraph/internal/src-cli"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/lib/errors"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"github.com/sourcegraph/sourcegraph/schema"
)
@ -194,19 +197,28 @@ func BatchChangesRestrictedToAdmins() bool {
// CodyEnabled returns whether Cody is enabled on this instance.
//
// If `cody.enabled` is not set or set to false, it's not enabled.
// If `cody.enabled` is true but `completions` aren't set, it returns false.
//
// Legacy-support for `completions.enabled`:
// If `cody.enabled` is NOT set, but `completions.enabled` is set, then cody is enabled.
// If `cody.enabled` is set, but `completions.enabled` is set, then cody is enabled based on value of `cody.enabled`.
// If `cody.enabled` is NOT set, but `completions.enabled` is true, then cody is enabled.
// If `cody.enabled` is set, and `completions.enabled` is set to false, cody is disabled.
func CodyEnabled() bool {
enabled := Get().CodyEnabled
completions := Get().Completions
return codyEnabled(Get().SiteConfig())
}
func codyEnabled(siteConfig schema.SiteConfiguration) bool {
enabled := siteConfig.CodyEnabled
completions := siteConfig.Completions
// If the cody.enabled flag is explicitly false, disable all cody features.
if enabled != nil && !*enabled {
return false
}
// Support for Legacy configurations in which `completions` is set to
// `enabled`, but `cody.enabled` is not set.
if enabled == nil && completions != nil && completions.Enabled {
return true
if enabled == nil && completions != nil {
// Unset means false.
return completions.Enabled != nil && *completions.Enabled
}
if enabled == nil {
@ -344,8 +356,7 @@ func CodeIntelRankingStaleResultAge() time.Duration {
}
func EmbeddingsEnabled() bool {
embeddingsConfig := Get().Embeddings
return embeddingsConfig != nil && embeddingsConfig.Enabled
return GetEmbeddingsConfig(Get().SiteConfiguration) != nil
}
func ProductResearchPageEnabled() bool {
@ -630,3 +641,321 @@ func GitMaxConcurrentClones() int {
}
return v
}
// GetCompletionsConfig evaluates a complete completions configuration based on
// site configuration. The configuration may be nil if completions is disabled.
func GetCompletionsConfig(siteConfig schema.SiteConfiguration) (c *conftypes.CompletionsConfig) {
// If cody is disabled, don't use completions.
if !codyEnabled(siteConfig) {
return nil
}
// Additionally, completions in App are disabled if there is no dotcom auth token
// and the user hasn't provided their own api token.
if deploy.IsApp() {
if (siteConfig.App == nil || len(siteConfig.App.DotcomAuthToken) == 0) && (siteConfig.Completions == nil || siteConfig.Completions.AccessToken == "") {
return nil
}
}
completionsConfig := siteConfig.Completions
// If no completions configuration is set at all, but cody is enabled, assume
// a default configuration.
if completionsConfig == nil {
completionsConfig = &schema.Completions{
Provider: string(conftypes.CompletionsProviderNameSourcegraph),
ChatModel: "anthropic/claude-v1",
FastChatModel: "anthropic/claude-instant-v1",
CompletionModel: "anthropic/claude-instant-v1",
}
}
// If after setting defaults for no `completions` config given there still is no
// provider configured.
if completionsConfig.Provider == "" {
return nil
}
// If ChatModel is not set, fall back to the deprecated Model field.
// Note: It might also be empty.
if completionsConfig.ChatModel == "" {
completionsConfig.ChatModel = completionsConfig.Model
}
if completionsConfig.Provider == string(conftypes.CompletionsProviderNameSourcegraph) {
// If no endpoint is configured, use a default value.
if completionsConfig.Endpoint == "" {
completionsConfig.Endpoint = "https://cody-gateway.sourcegraph.com"
}
// Set the access token, either use the configured one, or generate one for the platform.
completionsConfig.AccessToken = getSourcegraphProviderAccessToken(completionsConfig.AccessToken, siteConfig)
// If we weren't able to generate an access token of some sort, authing with
// Cody Gateway is not possible and we cannot use completions.
if completionsConfig.AccessToken == "" {
return nil
}
// Set a default chat model.
if completionsConfig.ChatModel == "" {
completionsConfig.ChatModel = "anthropic/claude-v1"
}
// Set a default fast chat model.
if completionsConfig.FastChatModel == "" {
completionsConfig.FastChatModel = "anthropic/claude-instant-v1"
}
// Set a default completions model.
if completionsConfig.CompletionModel == "" {
completionsConfig.CompletionModel = "anthropic/claude-instant-v1"
}
} else if completionsConfig.Provider == string(conftypes.CompletionsProviderNameOpenAI) {
// If no endpoint is configured, use a default value.
if completionsConfig.Endpoint == "" {
completionsConfig.Endpoint = "https://api.openai.com/v1/chat/completions"
}
// If not access token is set, we cannot talk to OpenAI. Bail.
if completionsConfig.AccessToken == "" {
return nil
}
// Set a default chat model.
if completionsConfig.ChatModel == "" {
completionsConfig.ChatModel = "gpt-4"
}
// Set a default fast chat model.
if completionsConfig.FastChatModel == "" {
completionsConfig.FastChatModel = "gpt-3.5-turbo"
}
// Set a default completions model.
if completionsConfig.CompletionModel == "" {
completionsConfig.CompletionModel = "gpt-3.5-turbo"
}
} else if completionsConfig.Provider == string(conftypes.CompletionsProviderNameAnthropic) {
// If no endpoint is configured, use a default value.
if completionsConfig.Endpoint == "" {
completionsConfig.Endpoint = "https://api.anthropic.com/v1/complete"
}
// If not access token is set, we cannot talk to Anthropic. Bail.
if completionsConfig.AccessToken == "" {
return nil
}
// Set a default chat model.
if completionsConfig.ChatModel == "" {
completionsConfig.ChatModel = "claude-v1"
}
// Set a default fast chat model.
if completionsConfig.FastChatModel == "" {
completionsConfig.FastChatModel = "claude-instant-v1"
}
// Set a default completions model.
if completionsConfig.CompletionModel == "" {
completionsConfig.CompletionModel = "claude-instant-v1"
}
}
// Make sure models are always treated case-insensitive.
completionsConfig.ChatModel = strings.ToLower(completionsConfig.ChatModel)
completionsConfig.FastChatModel = strings.ToLower(completionsConfig.FastChatModel)
completionsConfig.CompletionModel = strings.ToLower(completionsConfig.CompletionModel)
// If after trying to set default we still have not all models configured, completions are
// not available.
if completionsConfig.ChatModel == "" || completionsConfig.FastChatModel == "" || completionsConfig.CompletionModel == "" {
return nil
}
computedConfig := &conftypes.CompletionsConfig{
Provider: conftypes.CompletionsProviderName(completionsConfig.Provider),
AccessToken: completionsConfig.AccessToken,
ChatModel: completionsConfig.ChatModel,
FastChatModel: completionsConfig.FastChatModel,
CompletionModel: completionsConfig.CompletionModel,
Endpoint: completionsConfig.Endpoint,
PerUserDailyLimit: completionsConfig.PerUserDailyLimit,
PerUserCodeCompletionsDailyLimit: completionsConfig.PerUserCodeCompletionsDailyLimit,
}
return computedConfig
}
// GetEmbeddingsConfig evaluates a complete embeddings configuration based on
// site configuration. The configuration may be nil if completions is disabled.
func GetEmbeddingsConfig(siteConfig schema.SiteConfiguration) *conftypes.EmbeddingsConfig {
// If cody is disabled, don't use embeddings.
if !codyEnabled(siteConfig) {
return nil
}
// Additionally Embeddings in App are disabled if there is no dotcom auth token
// and the user hasn't provided their own api token.
if deploy.IsApp() {
if (siteConfig.App == nil || len(siteConfig.App.DotcomAuthToken) == 0) && (siteConfig.Embeddings == nil || siteConfig.Embeddings.AccessToken == "") {
return nil
}
}
// If embeddings are explicitly disabled (legacy flag, TODO: remove after 5.1),
// don't use embeddings either.
if siteConfig.Embeddings != nil && siteConfig.Embeddings.Enabled != nil && !*siteConfig.Embeddings.Enabled {
return nil
}
embeddingsConfig := siteConfig.Embeddings
// If no embeddings configuration is set at all, but cody is enabled, assume
// a default configuration.
if embeddingsConfig == nil {
embeddingsConfig = &schema.Embeddings{
Provider: string(conftypes.EmbeddingsProviderNameSourcegraph),
}
}
// If after setting defaults for no `embeddings` config given there still is no
// provider configured.
// Before, this meant "use OpenAI", but it's easy to accidentally send Cody Gateway
// auth tokens to OpenAI by that, so we want to be explicit going forward.
if embeddingsConfig.Provider == "" {
return nil
}
// The default value for incremental is true.
if embeddingsConfig.Incremental == nil {
embeddingsConfig.Incremental = pointers.Ptr(true)
}
// Set default values for max embeddings counts.
embeddingsConfig.MaxCodeEmbeddingsPerRepo = defaultTo(embeddingsConfig.MaxCodeEmbeddingsPerRepo, defaultMaxCodeEmbeddingsPerRepo)
embeddingsConfig.MaxTextEmbeddingsPerRepo = defaultTo(embeddingsConfig.MaxTextEmbeddingsPerRepo, defaultMaxTextEmbeddingsPerRepo)
// The default value for MinimumInterval is 24h.
if embeddingsConfig.MinimumInterval == "" {
embeddingsConfig.MinimumInterval = defaultMinimumInterval.String()
}
// Set the default for PolicyRepositoryMatchLimit.
if embeddingsConfig.PolicyRepositoryMatchLimit == nil {
v := defaultPolicyRepositoryMatchLimit
embeddingsConfig.PolicyRepositoryMatchLimit = &v
}
// If endpoint is not set, fall back to URL, it's the previous name of the setting.
// Note: It might also be empty.
if embeddingsConfig.Endpoint == "" {
embeddingsConfig.Endpoint = embeddingsConfig.Url
}
if embeddingsConfig.Provider == string(conftypes.EmbeddingsProviderNameSourcegraph) {
// If no endpoint is configured, use a default value.
if embeddingsConfig.Endpoint == "" {
embeddingsConfig.Endpoint = "https://cody-gateway.sourcegraph.com/v1/embeddings"
}
// Set the access token, either use the configured one, or generate one for the platform.
embeddingsConfig.AccessToken = getSourcegraphProviderAccessToken(embeddingsConfig.AccessToken, siteConfig)
// If we weren't able to generate an access token of some sort, authing with
// Cody Gateway is not possible and we cannot use embeddings.
if embeddingsConfig.AccessToken == "" {
return nil
}
// Set a default model.
if embeddingsConfig.Model == "" {
embeddingsConfig.Model = "openai/text-embedding-ada-002"
}
// Make sure models are always treated case-insensitive.
embeddingsConfig.Model = strings.ToLower(embeddingsConfig.Model)
// Set a default for model dimensions if using the default model.
if embeddingsConfig.Dimensions <= 0 && embeddingsConfig.Model == "openai/text-embedding-ada-002" {
embeddingsConfig.Dimensions = 1536
}
} else if embeddingsConfig.Provider == string(conftypes.EmbeddingsProviderNameOpenAI) {
// If no endpoint is configured, use a default value.
if embeddingsConfig.Endpoint == "" {
embeddingsConfig.Endpoint = "https://api.openai.com/v1/embeddings"
}
// If not access token is set, we cannot talk to OpenAI. Bail.
if embeddingsConfig.AccessToken == "" {
return nil
}
// Set a default model.
if embeddingsConfig.Model == "" {
embeddingsConfig.Model = "text-embedding-ada-002"
}
// Make sure models are always treated case-insensitive.
embeddingsConfig.Model = strings.ToLower(embeddingsConfig.Model)
// Set a default for model dimensions if using the default model.
if embeddingsConfig.Dimensions <= 0 && embeddingsConfig.Model == "text-embedding-ada-002" {
embeddingsConfig.Dimensions = 1536
}
} else {
// Unknown provider value.
return nil
}
computedConfig := &conftypes.EmbeddingsConfig{
Provider: conftypes.EmbeddingsProviderName(embeddingsConfig.Provider),
AccessToken: embeddingsConfig.AccessToken,
Model: embeddingsConfig.Model,
Endpoint: embeddingsConfig.Endpoint,
Dimensions: embeddingsConfig.Dimensions,
// This is definitely set at this point.
Incremental: *embeddingsConfig.Incremental,
ExcludedFilePathPatterns: embeddingsConfig.ExcludedFilePathPatterns,
MaxCodeEmbeddingsPerRepo: embeddingsConfig.MaxCodeEmbeddingsPerRepo,
MaxTextEmbeddingsPerRepo: embeddingsConfig.MaxTextEmbeddingsPerRepo,
PolicyRepositoryMatchLimit: embeddingsConfig.PolicyRepositoryMatchLimit,
}
d, err := time.ParseDuration(embeddingsConfig.MinimumInterval)
if err != nil {
computedConfig.MinimumInterval = defaultMinimumInterval
} else {
computedConfig.MinimumInterval = d
}
return computedConfig
}
func getSourcegraphProviderAccessToken(accessToken string, config schema.SiteConfiguration) string {
// If an access token is configured, use it.
if accessToken != "" {
return accessToken
}
// App generates a token from the api token the user used to connect app to dotcom.
if deploy.IsApp() && config.App != nil {
if config.App.DotcomAuthToken == "" {
return ""
}
return dotcomuser.GenerateDotcomUserGatewayAccessToken(config.App.DotcomAuthToken)
}
// Otherwise, use the current license key to compute an access token.
if config.LicenseKey == "" {
return ""
}
return licensing.GenerateLicenseKeyBasedAccessToken(config.LicenseKey)
}
const (
defaultPolicyRepositoryMatchLimit = 5000
defaultMinimumInterval = 24 * time.Hour
defaultMaxCodeEmbeddingsPerRepo = 3_072_000
defaultMaxTextEmbeddingsPerRepo = 512_000
)
func defaultTo(val, def int) int {
if val == 0 {
return def
}
return val
}

533
internal/conf/computed_test.go
View File

@ -7,6 +7,9 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/licensing"
"github.com/sourcegraph/sourcegraph/lib/pointers"
"github.com/sourcegraph/sourcegraph/schema"
)
@ -347,6 +350,11 @@ func TestCodyEnabled(t *testing.T) {
sc: schema.SiteConfiguration{CodyEnabled: pointers.Ptr(true), Completions: &schema.Completions{Model: "foobar"}},
want: true,
},
{
name: "cody disabled, completions enabled",
sc: schema.SiteConfiguration{CodyEnabled: pointers.Ptr(false), Completions: &schema.Completions{Enabled: pointers.Ptr(true), Model: "foobar"}},
want: false,
},
{
name: "cody disabled, completions configured",
sc: schema.SiteConfiguration{CodyEnabled: pointers.Ptr(false), Completions: &schema.Completions{Model: "foobar"}},
@ -361,7 +369,7 @@ func TestCodyEnabled(t *testing.T) {
{
// Legacy support: remove this once completions.enabled is removed
name: "cody.enabled not set, completions configured and enabled",
sc: schema.SiteConfiguration{Completions: &schema.Completions{Enabled: true, Model: "foobar"}},
sc: schema.SiteConfiguration{Completions: &schema.Completions{Enabled: pointers.Ptr(true), Model: "foobar"}},
want: true,
},
}
@ -374,3 +382,526 @@ func TestCodyEnabled(t *testing.T) {
})
}
}
func TestGetCompletionsConfig(t *testing.T) {
licenseKey := "theasdfkey"
licenseAccessToken := licensing.GenerateLicenseKeyBasedAccessToken(licenseKey)
testCases := []struct {
name string
siteConfig schema.SiteConfiguration
deployType string
wantConfig *conftypes.CompletionsConfig
wantDisabled bool
}{
{
name: "Completions disabled",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Completions: &schema.Completions{
Enabled: pointers.Ptr(false),
},
},
wantDisabled: true,
},
{
name: "cody.enabled and empty completions object",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Completions: &schema.Completions{},
},
wantDisabled: true,
},
{
name: "cody.enabled set false",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(false),
Completions: &schema.Completions{},
},
wantDisabled: true,
},
{
name: "no cody config",
siteConfig: schema.SiteConfiguration{
CodyEnabled: nil,
Completions: nil,
},
wantDisabled: true,
},
{
name: "Invalid provider",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Completions: &schema.Completions{
Provider: "invalid",
},
},
wantDisabled: true,
},
{
name: "anthropic completions",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Completions: &schema.Completions{
Enabled: pointers.Ptr(true),
Provider: "anthropic",
AccessToken: "asdf",
},
},
wantConfig: &conftypes.CompletionsConfig{
ChatModel: "claude-v1",
FastChatModel: "claude-instant-v1",
CompletionModel: "claude-instant-v1",
AccessToken: "asdf",
Provider: "anthropic",
Endpoint: "https://api.anthropic.com/v1/complete",
},
},
{
name: "anthropic completions, with only completions.enabled",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Completions: &schema.Completions{
Enabled: pointers.Ptr(false),
Provider: "anthropic",
AccessToken: "asdf",
ChatModel: "claude-v1",
CompletionModel: "claude-instant-v1",
},
},
wantConfig: &conftypes.CompletionsConfig{
ChatModel: "claude-v1",
FastChatModel: "claude-instant-v1",
CompletionModel: "claude-instant-v1",
AccessToken: "asdf",
Provider: "anthropic",
Endpoint: "https://api.anthropic.com/v1/complete",
},
},
{
name: "soucregraph completions defaults",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Completions: &schema.Completions{
Provider: "sourcegraph",
},
},
wantConfig: &conftypes.CompletionsConfig{
ChatModel: "anthropic/claude-v1",
FastChatModel: "anthropic/claude-instant-v1",
CompletionModel: "anthropic/claude-instant-v1",
AccessToken: licenseAccessToken,
Provider: "sourcegraph",
Endpoint: "https://cody-gateway.sourcegraph.com",
},
},
{
name: "OpenAI completions completions",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Completions: &schema.Completions{
Provider: "openai",
AccessToken: "asdf",
},
},
wantConfig: &conftypes.CompletionsConfig{
ChatModel: "gpt-4",
FastChatModel: "gpt-3.5-turbo",
CompletionModel: "gpt-3.5-turbo",
AccessToken: "asdf",
Provider: "openai",
Endpoint: "https://api.openai.com/v1/chat/completions",
},
},
{
name: "zero-config cody gateway completions without license key",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: "",
},
wantDisabled: true,
},
{
name: "zero-config cody gateway completions with license key",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
},
wantConfig: &conftypes.CompletionsConfig{
ChatModel: "anthropic/claude-v1",
FastChatModel: "anthropic/claude-instant-v1",
CompletionModel: "anthropic/claude-instant-v1",
AccessToken: licenseAccessToken,
Provider: "sourcegraph",
Endpoint: "https://cody-gateway.sourcegraph.com",
},
},
{
// Legacy support for completions.enabled
name: "legacy field completions.enabled: zero-config cody gateway completions without license key",
siteConfig: schema.SiteConfiguration{
Completions: &schema.Completions{Enabled: pointers.Ptr(true)},
LicenseKey: "",
},
wantDisabled: true,
},
{
name: "legacy field completions.enabled: zero-config cody gateway completions with license key",
siteConfig: schema.SiteConfiguration{
Completions: &schema.Completions{
Enabled: pointers.Ptr(true),
},
LicenseKey: licenseKey,
},
// Not supported anymore.
wantDisabled: true,
},
{
name: "app zero-config cody gateway completions with dotcom token",
deployType: deploy.App,
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
App: &schema.App{
DotcomAuthToken: "TOKEN",
},
},
wantConfig: &conftypes.CompletionsConfig{
AccessToken: "sgd_5df6e0e2761359d30a8275058e299fcc0381534545f55cf43e41983f5d4c9456",
ChatModel: "anthropic/claude-v1",
FastChatModel: "anthropic/claude-instant-v1",
CompletionModel: "anthropic/claude-instant-v1",
Endpoint: "https://cody-gateway.sourcegraph.com",
Provider: "sourcegraph",
},
},
{
name: "app with custom configuration",
deployType: deploy.App,
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
Completions: &schema.Completions{
AccessToken: "CUSTOM_TOKEN",
Provider: "anthropic",
ChatModel: "claude-v1",
FastChatModel: "claude-instant-v1",
CompletionModel: "claude-instant-v1",
},
App: &schema.App{
DotcomAuthToken: "TOKEN",
},
},
wantConfig: &conftypes.CompletionsConfig{
AccessToken: "CUSTOM_TOKEN",
ChatModel: "claude-v1",
CompletionModel: "claude-instant-v1",
FastChatModel: "claude-instant-v1",
Provider: "anthropic",
Endpoint: "https://api.anthropic.com/v1/complete",
},
},
{
name: "App but no dotcom username",
deployType: deploy.App,
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
App: &schema.App{
DotcomAuthToken: "",
},
},
wantDisabled: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
defaultDeploy := deploy.Type()
if tc.deployType != "" {
deploy.Mock(tc.deployType)
}
t.Cleanup(func() {
deploy.Mock(defaultDeploy)
})
conf := GetCompletionsConfig(tc.siteConfig)
if tc.wantDisabled {
if conf != nil {
t.Fatal("expected nil config but got non-nil")
}
} else {
if conf == nil {
t.Fatal("unexpected nil config returned")
}
if diff := cmp.Diff(tc.wantConfig, conf); diff != "" {
t.Fatalf("unexpected config computed: %s", diff)
}
}
})
}
}
func TestGetEmbeddingsConfig(t *testing.T) {
licenseKey := "theasdfkey"
licenseAccessToken := licensing.GenerateLicenseKeyBasedAccessToken(licenseKey)
testCases := []struct {
name string
siteConfig schema.SiteConfiguration
deployType string
wantConfig *conftypes.EmbeddingsConfig
wantDisabled bool
}{
{
name: "Embeddings disabled",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Embeddings: &schema.Embeddings{
Enabled: pointers.Ptr(false),
},
},
wantDisabled: true,
},
{
name: "cody.enabled and empty embeddings object",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Embeddings: &schema.Embeddings{},
},
wantDisabled: true,
},
{
name: "cody.enabled set false",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(false),
Embeddings: &schema.Embeddings{},
},
wantDisabled: true,
},
{
name: "no cody config",
siteConfig: schema.SiteConfiguration{
CodyEnabled: nil,
Embeddings: nil,
},
wantDisabled: true,
},
{
name: "Invalid provider",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Embeddings: &schema.Embeddings{
Provider: "invalid",
},
},
wantDisabled: true,
},
{
name: "Implicit config with cody.enabled",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
},
wantConfig: &conftypes.EmbeddingsConfig{
Provider: "sourcegraph",
AccessToken: licenseAccessToken,
Model: "openai/text-embedding-ada-002",
Endpoint: "https://cody-gateway.sourcegraph.com/v1/embeddings",
Dimensions: 1536,
Incremental: true,
MinimumInterval: 24 * time.Hour,
MaxCodeEmbeddingsPerRepo: 3_072_000,
MaxTextEmbeddingsPerRepo: 512_000,
PolicyRepositoryMatchLimit: pointers.Ptr(5000),
},
},
{
name: "Sourcegraph provider",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Embeddings: &schema.Embeddings{
Provider: "sourcegraph",
},
},
wantConfig: &conftypes.EmbeddingsConfig{
Provider: "sourcegraph",
AccessToken: licenseAccessToken,
Model: "openai/text-embedding-ada-002",
Endpoint: "https://cody-gateway.sourcegraph.com/v1/embeddings",
Dimensions: 1536,
Incremental: true,
MinimumInterval: 24 * time.Hour,
MaxCodeEmbeddingsPerRepo: 3_072_000,
MaxTextEmbeddingsPerRepo: 512_000,
PolicyRepositoryMatchLimit: pointers.Ptr(5000),
},
},
{
name: "Sourcegraph provider without license",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: "",
Embeddings: &schema.Embeddings{
Provider: "sourcegraph",
},
},
wantDisabled: true,
},
{
name: "OpenAI provider",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Embeddings: &schema.Embeddings{
Provider: "openai",
AccessToken: "asdf",
},
},
wantConfig: &conftypes.EmbeddingsConfig{
Provider: "openai",
AccessToken: "asdf",
Model: "text-embedding-ada-002",
Endpoint: "https://api.openai.com/v1/embeddings",
Dimensions: 1536,
Incremental: true,
MinimumInterval: 24 * time.Hour,
MaxCodeEmbeddingsPerRepo: 3_072_000,
MaxTextEmbeddingsPerRepo: 512_000,
PolicyRepositoryMatchLimit: pointers.Ptr(5000),
},
},
{
name: "OpenAI provider without access token",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
LicenseKey: licenseKey,
Embeddings: &schema.Embeddings{
Provider: "openai",
},
},
wantDisabled: true,
},
{
name: "App default config",
deployType: deploy.App,
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
App: &schema.App{
DotcomAuthToken: "TOKEN",
},
},
wantConfig: &conftypes.EmbeddingsConfig{
Provider: "sourcegraph",
AccessToken: "sgd_5df6e0e2761359d30a8275058e299fcc0381534545f55cf43e41983f5d4c9456",
Model: "openai/text-embedding-ada-002",
Endpoint: "https://cody-gateway.sourcegraph.com/v1/embeddings",
Dimensions: 1536,
Incremental: true,
MinimumInterval: 24 * time.Hour,
MaxCodeEmbeddingsPerRepo: 3_072_000,
MaxTextEmbeddingsPerRepo: 512_000,
PolicyRepositoryMatchLimit: pointers.Ptr(5000),
},
},
{
name: "App but no dotcom username",
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
App: &schema.App{
DotcomAuthToken: "",
},
},
wantDisabled: true,
},
{
name: "App with dotcom token",
deployType: deploy.App,
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
Embeddings: &schema.Embeddings{
Provider: "sourcegraph",
},
App: &schema.App{
DotcomAuthToken: "TOKEN",
},
},
wantConfig: &conftypes.EmbeddingsConfig{
Provider: "sourcegraph",
AccessToken: "sgd_5df6e0e2761359d30a8275058e299fcc0381534545f55cf43e41983f5d4c9456",
Model: "openai/text-embedding-ada-002",
Endpoint: "https://cody-gateway.sourcegraph.com/v1/embeddings",
Dimensions: 1536,
Incremental: true,
MinimumInterval: 24 * time.Hour,
MaxCodeEmbeddingsPerRepo: 3_072_000,
MaxTextEmbeddingsPerRepo: 512_000,
PolicyRepositoryMatchLimit: pointers.Ptr(5000),
},
},
{
name: "App with user token",
deployType: deploy.App,
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
Embeddings: &schema.Embeddings{
Provider: "sourcegraph",
AccessToken: "TOKEN",
},
},
wantConfig: &conftypes.EmbeddingsConfig{
Provider: "sourcegraph",
AccessToken: "TOKEN",
Model: "openai/text-embedding-ada-002",
Endpoint: "https://cody-gateway.sourcegraph.com/v1/embeddings",
Dimensions: 1536,
Incremental: true,
MinimumInterval: 24 * time.Hour,
MaxCodeEmbeddingsPerRepo: 3_072_000,
MaxTextEmbeddingsPerRepo: 512_000,
PolicyRepositoryMatchLimit: pointers.Ptr(5000),
},
},
{
name: "App without dotcom or user token",
deployType: deploy.App,
siteConfig: schema.SiteConfiguration{
CodyEnabled: pointers.Ptr(true),
Embeddings: &schema.Embeddings{
Provider: "sourcegraph",
},
},
wantDisabled: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
defaultDeploy := deploy.Type()
if tc.deployType != "" {
deploy.Mock(tc.deployType)
}
t.Cleanup(func() {
deploy.Mock(defaultDeploy)
})
conf := GetEmbeddingsConfig(tc.siteConfig)
if tc.wantDisabled {
if conf != nil {
t.Fatal("expected nil config but got non-nil")
}
} else {
if conf == nil {
t.Fatal("unexpected nil config returned")
}
if diff := cmp.Diff(tc.wantConfig, conf); diff != "" {
t.Fatalf("unexpected config computed: %s", diff)
}
}
})
}
}

1
internal/conf/conftypes/BUILD.bazel generated
View File

@ -4,6 +4,7 @@ go_library(
name = "conftypes",
srcs = [
"conftypes.go",
"consts.go",
"unified.go",
],
importpath = "github.com/sourcegraph/sourcegraph/internal/conf/conftypes",

Some files were not shown because too many files have changed in this diff Show More