mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 19:21:50 +00:00
config(cody): Various fixes and updating the naming rules for ModelConfig resource IDs (#63436)
The `internal/modelconfig` package provides the schema by which Sourcegraph instances will keep track of LLM providers, models, and various configuration settings. The `cmd/cody-gateway-config` tool writes a JSON file contains the model configuration data for Cody Gateway. This PR fixes an assortment if problems with these components.d8963daf6d- Due to a logic error, we were saving the model ID of "unknown" into `models.json`. We now put the actual `ModelID` from the `ModelRef`.d9baa65534- Updates the model information that gets rendered, so that it matches what is hard-coded into the `sourcegraph/cody` repo. (We were missing any reference to the newly added Google Gemini models.)c28780c4d9- Relaxes the resource ID regular expression so that it is now legal to add periods. So "gemini-1.5-latest" is now considered a valid `ModelID`. ... however, the validation checks were incorrectly passing because there was a bug in the regular expression. And after writing some unit tests for the `validateModelRef` function, I found several other problems with that regular expression 😅 . But we should be much closer to things working as intended now. ## Test plan Added tests
This commit is contained in:
Chris Smith
GitHub
parent
c3ce26f308
commit
09aefd59e6
@ -64,7 +64,7 @@ func getAnthropicModels() []types.Model {
|
||||
// Pro Anthropic models.
|
||||
newModel(
|
||||
modelIdentity{
|
||||
MRef: mRef(anthropic_06_2023, "claude-3-5-sonnet"),
|
||||
MRef: mRef(anthropic_06_2023, "claude-3.5-sonnet"),
|
||||
Name: "claude-3-5-sonnet-20240620",
|
||||
DisplayName: "Claude 3.5 Sonnet",
|
||||
},
|
||||
@ -145,6 +145,43 @@ func getAnthropicModels() []types.Model {
|
||||
}
|
||||
}
|
||||
|
||||
func getGoogleModels() []types.Model {
|
||||
const (
|
||||
// Gemini API versions.
|
||||
// https://ai.google.dev/gemini-api/docs/api-versions
|
||||
geminiV1 = "google::v1"
|
||||
)
|
||||
|
||||
return []types.Model{
|
||||
newModel(
|
||||
modelIdentity{
|
||||
MRef: mRef(geminiV1, "gemini-1.5-pro-latest"),
|
||||
Name: "gemini-1.5-pro-latest",
|
||||
DisplayName: "Gemini 1.5 Pro",
|
||||
},
|
||||
modelMetadata{
|
||||
Capabilities: chatAndEdit,
|
||||
Category: types.ModelCategoryAccuracy,
|
||||
Status: types.ModelStatusStable,
|
||||
Tier: types.ModelTierPro,
|
||||
},
|
||||
expandedCtxWindow),
|
||||
newModel(
|
||||
modelIdentity{
|
||||
MRef: mRef(geminiV1, "gemini-1.5-flash-latest"),
|
||||
Name: "google/gemini-1.5-flash-latest",
|
||||
DisplayName: "Gemini 1.5 Flash",
|
||||
},
|
||||
modelMetadata{
|
||||
Capabilities: chatAndEdit,
|
||||
Category: types.ModelCategorySpeed,
|
||||
Status: types.ModelStatusStable,
|
||||
Tier: types.ModelTierPro,
|
||||
},
|
||||
expandedCtxWindow),
|
||||
}
|
||||
}
|
||||
|
||||
func getMistralModels() []types.Model {
|
||||
// Not sure if there is a canonical API reference, since Mixtral offers 3rd
|
||||
// party LLMs as a service.
|
||||
@ -209,7 +246,7 @@ func getOpenAIModels() []types.Model {
|
||||
},
|
||||
modelMetadata{
|
||||
Capabilities: chatAndEdit,
|
||||
Category: types.ModelCategoryAccuracy,
|
||||
Category: types.ModelCategoryBalanced,
|
||||
Status: types.ModelStatusStable,
|
||||
Tier: types.ModelTierPro,
|
||||
},
|
||||
@ -238,6 +275,7 @@ func GetCodyFreeProModels() ([]types.Model, error) {
|
||||
// ================================================
|
||||
var allModels []types.Model
|
||||
allModels = append(allModels, getAnthropicModels()...)
|
||||
allModels = append(allModels, getGoogleModels()...)
|
||||
allModels = append(allModels, getMistralModels()...)
|
||||
allModels = append(allModels, getOpenAIModels()...)
|
||||
|
||||
|
||||
@ -38,7 +38,7 @@ func newModel(identity modelIdentity, metadata modelMetadata, ctxWindow types.Co
|
||||
// before writing it out.
|
||||
modelID := types.ModelID("unknown")
|
||||
parts := strings.Split(string(identity.MRef), "::")
|
||||
if len(parts) != 3 {
|
||||
if len(parts) == 3 {
|
||||
modelID = types.ModelID(parts[2])
|
||||
}
|
||||
|
||||
|
||||
@ -21,11 +21,11 @@
|
||||
],
|
||||
"models": [
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelId": "claude-3-sonnet",
|
||||
"modelRef": "anthropic::2023-06-01::claude-3-sonnet",
|
||||
"displayName": "Claude 3 Sonnet",
|
||||
"string": "claude-3-sonnet-20240229",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "balanced",
|
||||
"status": "stable",
|
||||
"tier": "free",
|
||||
@ -35,11 +35,11 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelRef": "anthropic::2023-06-01::claude-3-5-sonnet",
|
||||
"modelId": "claude-3.5-sonnet",
|
||||
"modelRef": "anthropic::2023-06-01::claude-3.5-sonnet",
|
||||
"displayName": "Claude 3.5 Sonnet",
|
||||
"string": "claude-3-5-sonnet-20240620",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "accuracy",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
@ -49,11 +49,11 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelId": "claude-3-opus",
|
||||
"modelRef": "anthropic::2023-06-01::claude-3-opus",
|
||||
"displayName": "Claude 3 Opus",
|
||||
"string": "claude-3-opus-20240229",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "accuracy",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
@ -63,11 +63,11 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelId": "claude-3-haiku",
|
||||
"modelRef": "anthropic::2023-06-01::claude-3-haiku",
|
||||
"displayName": "Claude 3 Haiku",
|
||||
"string": "claude-3-haiku-20240307",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "speed",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
@ -77,11 +77,11 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelId": "claude-2.1",
|
||||
"modelRef": "anthropic::2023-01-01::claude-2.1",
|
||||
"displayName": "Claude 2.1",
|
||||
"string": "claude-2.1",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "balanced",
|
||||
"status": "deprecated",
|
||||
"tier": "free",
|
||||
@ -91,11 +91,11 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelId": "claude-2.0",
|
||||
"modelRef": "anthropic::2023-01-01::claude-2.0",
|
||||
"displayName": "Claude 2.0",
|
||||
"string": "claude-2.0",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "balanced",
|
||||
"status": "deprecated",
|
||||
"tier": "free",
|
||||
@ -105,11 +105,11 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelId": "claude-instant-1.2",
|
||||
"modelRef": "anthropic::2023-01-01::claude-instant-1.2",
|
||||
"displayName": "Claude Instant",
|
||||
"string": "claude-instant-1.2",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "balanced",
|
||||
"status": "deprecated",
|
||||
"tier": "free",
|
||||
@ -119,39 +119,11 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelRef": "mistral::v1::mixtral-8x7b-instruct",
|
||||
"displayName": "Mixtral 8x7B",
|
||||
"string": "mixtral-8x7b-instruct",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"category": "speed",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
"contextWindow": {
|
||||
"maxInputTokens": 7000,
|
||||
"maxOutputTokens": 4000
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelRef": "mistral::v1::mixtral-8x22b-instruct",
|
||||
"displayName": "Mixtral 8x22B",
|
||||
"string": "mixtral-8x22b-instruct",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"category": "accuracy",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
"contextWindow": {
|
||||
"maxInputTokens": 7000,
|
||||
"maxOutputTokens": 4000
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelRef": "openai::2024-02-01::gpt-4o",
|
||||
"displayName": "GPT-4o",
|
||||
"string": "gpt-4o",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"modelId": "gemini-1.5-pro-latest",
|
||||
"modelRef": "google::v1::gemini-1.5-pro-latest",
|
||||
"displayName": "Gemini 1.5 Pro",
|
||||
"string": "gemini-1.5-pro-latest",
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "accuracy",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
@ -161,11 +133,39 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelRef": "openai::2024-02-01::gpt-4-turbo",
|
||||
"displayName": "GPT-4 Turbo",
|
||||
"string": "gpt-4-turbo",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"modelId": "gemini-1.5-flash-latest",
|
||||
"modelRef": "google::v1::gemini-1.5-flash-latest",
|
||||
"displayName": "Gemini 1.5 Flash",
|
||||
"string": "google/gemini-1.5-flash-latest",
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "speed",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
"contextWindow": {
|
||||
"maxInputTokens": 30000,
|
||||
"maxOutputTokens": 4000
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "mixtral-8x7b-instruct",
|
||||
"modelRef": "mistral::v1::mixtral-8x7b-instruct",
|
||||
"displayName": "Mixtral 8x7B",
|
||||
"string": "mixtral-8x7b-instruct",
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "speed",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
"contextWindow": {
|
||||
"maxInputTokens": 7000,
|
||||
"maxOutputTokens": 4000
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "mixtral-8x22b-instruct",
|
||||
"modelRef": "mistral::v1::mixtral-8x22b-instruct",
|
||||
"displayName": "Mixtral 8x22B",
|
||||
"string": "mixtral-8x22b-instruct",
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "accuracy",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
@ -175,11 +175,39 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "unknown",
|
||||
"modelId": "gpt-4o",
|
||||
"modelRef": "openai::2024-02-01::gpt-4o",
|
||||
"displayName": "GPT-4o",
|
||||
"string": "gpt-4o",
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "accuracy",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
"contextWindow": {
|
||||
"maxInputTokens": 30000,
|
||||
"maxOutputTokens": 4000
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "gpt-4-turbo",
|
||||
"modelRef": "openai::2024-02-01::gpt-4-turbo",
|
||||
"displayName": "GPT-4 Turbo",
|
||||
"string": "gpt-4-turbo",
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "balanced",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
"contextWindow": {
|
||||
"maxInputTokens": 7000,
|
||||
"maxOutputTokens": 4000
|
||||
}
|
||||
},
|
||||
{
|
||||
"modelId": "gpt-3.5-turbo",
|
||||
"modelRef": "openai::2024-02-01::gpt-3.5-turbo",
|
||||
"displayName": "GPT-3.5 Turbo",
|
||||
"string": "gpt-3.5-turbo",
|
||||
"capabilities": ["chat", "edit"],
|
||||
"capabilities": ["autocomplete", "chat"],
|
||||
"category": "speed",
|
||||
"status": "stable",
|
||||
"tier": "pro",
|
||||
|
||||
@ -16,7 +16,7 @@ import (
|
||||
|
||||
// resourceIDRE is a regular expression for verifying resource IDs are
|
||||
// of a simple format.
|
||||
var resourceIDRE = regexp.MustCompile(`[a-z][a-z-]*[a-z]`)
|
||||
var resourceIDRE = regexp.MustCompile(`^[a-z0-9][a-z0-9_\-\.]*[a-z0-9]$`)
|
||||
|
||||
func validateProvider(p types.Provider) error {
|
||||
if l := len(p.DisplayName); l < 5 || l > 40 {
|
||||
@ -36,17 +36,23 @@ func validateProvider(p types.Provider) error {
|
||||
func validateModelRef(ref types.ModelRef) error {
|
||||
parts := strings.Split(string(ref), "::")
|
||||
if len(parts) != 3 {
|
||||
return errors.New("invalid number of parts")
|
||||
return errors.New("modelRef syntax error")
|
||||
}
|
||||
if !resourceIDRE.MatchString(parts[0]) {
|
||||
return errors.New("invalid ProviderID")
|
||||
}
|
||||
if !resourceIDRE.MatchString(parts[2]) {
|
||||
return errors.New("invalid ModelID")
|
||||
}
|
||||
// We don't impose any constraints on the API Version ID, because
|
||||
// while it's something Sourcegraph manages, there are lots of exotic
|
||||
// but reasonable forms it could take. e.g. "2024-06-01" or
|
||||
// "v1+beta2/with-git-lfs-context-support".
|
||||
if !resourceIDRE.MatchString(parts[2]) {
|
||||
return errors.New("invalid ModelID")
|
||||
//
|
||||
// But we still want to impose some basic standards, defined here.
|
||||
apiVersion := parts[1]
|
||||
if strings.ContainsAny(apiVersion, `:;*% $#\"',!@`) {
|
||||
return errors.New("invalid APIVersionID")
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@ -11,6 +11,61 @@ import (
|
||||
"github.com/sourcegraph/sourcegraph/internal/modelconfig/types"
|
||||
)
|
||||
|
||||
// Tests for various corner cases and regressions.
|
||||
func TestValidationMethods(t *testing.T) {
|
||||
t.Run("ValidateModelRef", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
MRef string
|
||||
// WantErr is the text of the expected error message.
|
||||
// If empty, we expect no error.
|
||||
WantErr string
|
||||
}{
|
||||
// Valid MRefs.
|
||||
{"foo::bar::baz", ""},
|
||||
{"foo-dashed::bar-dashed::baz-dashed-twice", ""},
|
||||
{"foo.dotted::bar.dotted::baz.dotted.twice", ""},
|
||||
{"provider_id::api_id::model_id", ""},
|
||||
|
||||
{"provider::api-version-can-totally/have-slashes::model", ""},
|
||||
|
||||
{"anthropic::2023-06-01::claude-3.5-sonnet", ""},
|
||||
|
||||
// Expected failure with older-style model references.
|
||||
{"claude-2", "modelRef syntax error"},
|
||||
{"anthropic/claude-2", "modelRef syntax error"},
|
||||
|
||||
// Generic validation errors.
|
||||
{"a::b::c::d", "modelRef syntax error"},
|
||||
|
||||
{"provider/v1::api-version::model", "invalid ProviderID"},
|
||||
{"CAPS_PROVIDER::v1::model", "invalid ProviderID"},
|
||||
{"g o o g l e::v1::gemini-1.5", "invalid ProviderID"},
|
||||
|
||||
{"foo::name-with!exclamatnions-should::be-ok", "invalid APIVersionID"},
|
||||
{"google::version one::gemini-1.5", "invalid APIVersionID"},
|
||||
|
||||
{"provider::api-version::model/v1", "invalid ModelID"},
|
||||
{"provider::apiver::CAPS_MODEL", "invalid ModelID"},
|
||||
{"anthropic::2023-01-01::claude instant", "invalid ModelID"},
|
||||
{"google::v1::Gemini 1.5", "invalid ModelID"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
ref := types.ModelRef(test.MRef)
|
||||
gotErr := validateModelRef(ref)
|
||||
|
||||
var gotErrText string
|
||||
if gotErr != nil {
|
||||
gotErrText = gotErr.Error()
|
||||
}
|
||||
|
||||
assert.Equal(
|
||||
t, test.WantErr, gotErrText,
|
||||
"didn't get expected validation error for mref %q", test.MRef)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Confirm that the model data currently in the repo is well-formed and valid.
|
||||
func TestEmbeddedModelConfig(t *testing.T) {
|
||||
loadModelConfig := func(t *testing.T) types.ModelConfiguration {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user