From fcefe5b37260f3169f46d5db8d0f67c4e35e1793 Mon Sep 17 00:00:00 2001 From: Erik Seliger Date: Tue, 4 Apr 2023 16:45:50 +0200 Subject: [PATCH] Add embeddings to server behind env var (#50288) This might be useful for some customers, but definitely will be useful for us to write an E2E pipeline for embeddings. cc @sourcegraph/dev-experience for a quick glance at the debugserver/prometheus part of this. ## Test plan Will build the image locally and see if it works alright with the env var set. --- cmd/server/shared/globals.go | 1 + dev/prometheus/all/prometheus_targets.yml | 5 +++++ dev/prometheus/linux/prometheus_targets.yml | 5 +++++ dev/src-prof-services.json | 1 + enterprise/cmd/server/build.sh | 3 ++- enterprise/cmd/server/main.go | 12 ++++++++++++ enterprise/dev/src-prof-services.json | 3 ++- sg.config.yaml | 1 + 8 files changed, 29 insertions(+), 2 deletions(-) diff --git a/cmd/server/shared/globals.go b/cmd/server/shared/globals.go index 6c476fe83ea..ee484799ab5 100644 --- a/cmd/server/shared/globals.go +++ b/cmd/server/shared/globals.go @@ -19,6 +19,7 @@ var SrcProfServices = []map[string]string{ {"Name": "repo-updater", "Host": "127.0.0.1:6074"}, {"Name": "worker", "Host": "127.0.0.1:6089"}, {"Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088"}, + {"Name": "embeddings", "Host": "127.0.0.1:6099"}, // no executors in server image {"Name": "zoekt-indexserver", "Host": "127.0.0.1:6072"}, {"Name": "zoekt-webserver", "Host": "127.0.0.1:3070", "DefaultPath": "/debug/requests/"}, diff --git a/dev/prometheus/all/prometheus_targets.yml b/dev/prometheus/all/prometheus_targets.yml index ffc19646647..54f105c1a7a 100644 --- a/dev/prometheus/all/prometheus_targets.yml +++ b/dev/prometheus/all/prometheus_targets.yml @@ -68,3 +68,8 @@ targets: # opentelemetry collector - host.docker.internal:8888 +- labels: + job: embeddings + targets: + # embeddings + - host.docker.internal:6099 diff --git a/dev/prometheus/linux/prometheus_targets.yml b/dev/prometheus/linux/prometheus_targets.yml index 8f94e01133c..cd4b8a165cc 100644 --- a/dev/prometheus/linux/prometheus_targets.yml +++ b/dev/prometheus/linux/prometheus_targets.yml @@ -68,3 +68,8 @@ targets: # opentelemetry collector - host.docker.internal:8888 +- labels: + job: embeddings + targets: + # embeddings + - 127.0.0.1:6099 diff --git a/dev/src-prof-services.json b/dev/src-prof-services.json index 0192a2f87ba..c73aa5ff4f2 100644 --- a/dev/src-prof-services.json +++ b/dev/src-prof-services.json @@ -11,6 +11,7 @@ { "Name": "executor-batches", "Host": "127.0.0.1:6093" }, { "Name": "zoekt-indexserver-0", "Host": "127.0.0.1:6072" }, { "Name": "zoekt-indexserver-1", "Host": "127.0.0.1:6073" }, + { "Name": "embeddings", "Host": "127.0.0.1:6099" }, { "Name": "zoekt-webserver-0", "Host": "127.0.0.1:3070", "DefaultPath": "/debug/requests/" }, { "Name": "zoekt-webserver-1", "Host": "127.0.0.1:3071", "DefaultPath": "/debug/requests/" } ] diff --git a/enterprise/cmd/server/build.sh b/enterprise/cmd/server/build.sh index 1c3b27ef08e..40ad1842fa7 100755 --- a/enterprise/cmd/server/build.sh +++ b/enterprise/cmd/server/build.sh @@ -13,4 +13,5 @@ export SERVER_PKG=${SERVER_PKG:-github.com/sourcegraph/sourcegraph/enterprise/cm github.com/sourcegraph/sourcegraph/enterprise/cmd/migrator \ github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater \ github.com/sourcegraph/sourcegraph/enterprise/cmd/symbols \ - github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker + github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker \ + github.com/sourcegraph/sourcegraph/enterprise/cmd/embeddings diff --git a/enterprise/cmd/server/main.go b/enterprise/cmd/server/main.go index 170572f8ccc..bb8b678bc7f 100644 --- a/enterprise/cmd/server/main.go +++ b/enterprise/cmd/server/main.go @@ -24,5 +24,17 @@ func main() { map[string]string{"Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088"}, ) + enableEmbeddings, _ := strconv.ParseBool(os.Getenv("SRC_ENABLE_EMBEDDINGS")) + if enableEmbeddings { + shared.ProcfileAdditions = append( + shared.ProcfileAdditions, + `embeddings: embeddings`, + ) + shared.SrcProfServices = append( + shared.SrcProfServices, + map[string]string{"Name": "embeddings", "Host": "127.0.0.1:6099"}, + ) + } + shared.Main() } diff --git a/enterprise/dev/src-prof-services.json b/enterprise/dev/src-prof-services.json index eca7b983df9..6f679260d13 100644 --- a/enterprise/dev/src-prof-services.json +++ b/enterprise/dev/src-prof-services.json @@ -6,5 +6,6 @@ { "Name": "repo-updater", "Host": "127.0.0.1:6074" }, { "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" }, { "Name": "worker", "Host": "127.0.0.1:6089" }, - { "Name": "worker-executors", "Host": "127.0.0.1:6969" } + { "Name": "worker-executors", "Host": "127.0.0.1:6969" }, + { "Name": "embeddings", "Host": "127.0.0.1:6099" } ] diff --git a/sg.config.yaml b/sg.config.yaml index d2e5ba278ac..7098138ee58 100644 --- a/sg.config.yaml +++ b/sg.config.yaml @@ -64,6 +64,7 @@ env: { "Name": "github-proxy", "Host": "127.0.0.1:6090" }, { "Name": "worker", "Host": "127.0.0.1:6089" }, { "Name": "worker-executors", "Host": "127.0.0.1:6996" }, + { "Name": "embeddings", "Host": "127.0.0.1:6099" }, { "Name": "zoekt-index-0", "Host": "127.0.0.1:6072" }, { "Name": "zoekt-index-1", "Host": "127.0.0.1:6073" }, { "Name": "zoekt-web-0", "Host": "127.0.0.1:3070", "DefaultPath": "/debug/requests/" },