codeintel: Create auto-indexer service skeleton (#10884)

cmd/precise-code-intel-indexer/Dockerfile

@@ -0,0 +1,19 @@
+FROM sourcegraph/alpine:3.10@sha256:4d05cd5669726fc38823e92320659a6d1ef7879e62268adec5df658a0bacf65c
+
+ARG COMMIT_SHA="unknown"
+ARG DATE="unknown"
+ARG VERSION="unknown"
+
+LABEL org.opencontainers.image.revision=${COMMIT_SHA}
+LABEL org.opencontainers.image.created=${DATE}
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}
+
+# hadolint ignore=DL3018
+RUN apk update && apk add --no-cache \
+    tini
+
+USER sourcegraph
+EXPOSE 3189
+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/precise-code-intel-api-indexer"]
+COPY precise-code-intel-api-indexer /usr/local/bin/

cmd/precise-code-intel-indexer/build.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script builds the precise-code-intel-api-indexer docker image.
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.."
+set -eu
+
+OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
+cleanup() {
+  rm -rf "$OUTPUT"
+}
+trap cleanup EXIT
+
+# Environment for building linux binaries
+export GO111MODULE=on
+export GOARCH=amd64
+export GOOS=linux
+export CGO_ENABLED=0
+
+echo "--- go build"
+pkg="github.com/sourcegraph/sourcegraph/cmd/precise-code-intel-api-indexer"
+go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg"
+
+echo "--- docker build"
+docker build -f cmd/precise-code-intel-api-indexer/Dockerfile -t "$IMAGE" "$OUTPUT" \
+  --progress=plain \
+  --build-arg COMMIT_SHA \
+  --build-arg DATE \
+  --build-arg VERSION

cmd/precise-code-intel-indexer/internal/server/handler.go

@@ -0,0 +1,15 @@
+package server
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+)
+
+func (s *Server) handler() http.Handler {
+	mux := mux.NewRouter()
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+	return mux
+}

cmd/precise-code-intel-indexer/internal/server/server.go

@@ -0,0 +1,52 @@
+package server
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"sync"
+
+	"github.com/inconshreveable/log15"
+	"github.com/sourcegraph/sourcegraph/internal/env"
+	"github.com/sourcegraph/sourcegraph/internal/trace/ot"
+)
+
+const Port = 3189
+
+type Server struct {
+	server *http.Server
+	once   sync.Once
+}
+
+func New() *Server {
+	host := ""
+	if env.InsecureDev {
+		host = "127.0.0.1"
+	}
+
+	s := &Server{}
+
+	s.server = &http.Server{
+		Addr:    net.JoinHostPort(host, strconv.FormatInt(int64(Port), 10)),
+		Handler: ot.Middleware(s.handler()),
+	}
+
+	return s
+}
+
+func (s *Server) Start() {
+	if err := s.server.ListenAndServe(); err != http.ErrServerClosed {
+		log15.Error("Failed to start server", "error", err)
+		os.Exit(1)
+	}
+}
+
+func (s *Server) Stop() {
+	s.once.Do(func() {
+		if err := s.server.Shutdown(context.Background()); err != nil {
+			log15.Error("Failed to shutdown server", "error", err)
+		}
+	})
+}

cmd/precise-code-intel-indexer/main.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/sourcegraph/sourcegraph/cmd/precise-code-intel-indexer/internal/server"
+	"github.com/sourcegraph/sourcegraph/internal/debugserver"
+	"github.com/sourcegraph/sourcegraph/internal/env"
+	"github.com/sourcegraph/sourcegraph/internal/tracer"
+)
+
+func main() {
+	env.Lock()
+	env.HandleHelpFlag()
+	tracer.Init()
+
+	server := server.New()
+
+	go server.Start()
+	go debugserver.Start()
+
+	// Attempt to clean up after first shutdown signal
+	signals := make(chan os.Signal, 2)
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGHUP)
+	<-signals
+
+	go func() {
+		// Insta-shutdown on a second signal
+		<-signals
+		os.Exit(0)
+	}()
+
+	server.Stop()
+}

dev/Procfile

@@ -20,6 +20,7 @@ jaeger: ./dev/jaeger.sh
 docsite: ./dev/docsite.sh -config doc/docsite.json serve -http=localhost:5080
 precise-code-intel-bundle-manager: precise-code-intel-bundle-manager
 precise-code-intel-worker: precise-code-intel-worker
+precise-code-intel-indexer: precise-code-intel-indexer
 prometheus: ./dev/prometheus.sh
 grafana: ./dev/grafana.sh
 postgres_exporter: ./dev/postgres_exporter.sh

dev/go-install.sh

@@ -6,7 +6,7 @@
 # This will install binaries into the `.bin` directory under the repository root.
 #
 
-all_oss_commands=" gitserver query-runner github-proxy searcher replacer frontend repo-updater symbols precise-code-intel-bundle-manager precise-code-intel-worker "
+all_oss_commands=" gitserver query-runner github-proxy searcher replacer frontend repo-updater symbols precise-code-intel-bundle-manager precise-code-intel-indexer precise-code-intel-worker "
 
 # handle options
 verbose=false

dev/prometheus/all/prometheus_targets.yml

@@ -44,6 +44,11 @@
   targets:
     # precise-code-intel-worker
     - host.docker.internal:6088
+- labels:
+    job: precise-code-intel-indexer
+  targets:
+    # precise-code-intel-indexer
+    - host.docker.internal:6089
 - labels:
     job: postgres_exporter
   targets:

dev/prometheus/linux/prometheus_targets.yml

@@ -44,6 +44,11 @@
   targets:
     # precise-code-intel-worker
     - 127.0.0.1:6088
+- labels:
+    job: precise-code-intel-indexer
+  targets:
+    # precise-code-intel-indexer
+    - 127.0.0.1:6089
 - labels:
     job: postgres_exporter
   targets:

dev/src-prof-services.json

@@ -8,6 +8,7 @@
   { "Name": "replacer", "Host": "127.0.0.1:6076" },
   { "Name": "precise-code-intel-bundle-manager", "Host": "127.0.0.1:6087" },
   { "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" },
+  { "Name": "precise-code-intel-indexer", "Host": "127.0.0.1:6089" },
   { "Name": "zoekt-indexserver", "Host": "127.0.0.1:6072" },
   { "Name": "zoekt-webserver", "Host": "127.0.0.1:3070", "DefaultPath": "/debug/requests/" }
 ]

doc/admin/observability/alert_solutions.md

@@ -458,6 +458,59 @@ for assistance.
 - **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`.
 - **Docker Compose:** Consider increasing `cpus:` of the precise-code-intel-worker container in `docker-compose.yml`.
 
+# precise-code-intel-indexer: frontend_internal_api_error_responses
+
+**Descriptions:**
+
+- _precise-code-intel-indexer: 5+ frontend-internal API error responses every 5m by route_
+
+**Possible solutions:**
+
+- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing.
+- **Kubernetes:**
+	- Confirm that `kubectl get pods` shows the `frontend` pods are healthy.
+	- Check `kubectl logs precise-code-intel-indexer` for logs indicate request failures to `frontend` or `frontend-internal`.
+- **Docker Compose:**
+	- Confirm that `docker ps` shows the `frontend-internal` container is healthy.
+	- Check `docker logs precise-code-intel-indexer` for logs indicating request failures to `frontend` or `frontend-internal`.
+
+# precise-code-intel-indexer: container_restarts
+
+**Descriptions:**
+
+- _precise-code-intel-indexer: 1+ container restarts every 5m by instance (not available on k8s or server)_
+
+**Possible solutions:**
+
+- **Kubernetes:**
+	- Determine if the pod was OOM killed using `kubectl describe pod precise-code-intel-indexer` (look for `OOMKilled: true`) and, if so, consider increasing the memory limit in the relevant `Deployment.yaml`.
+	- Check the logs before the container restarted to see if there are `panic:` messages or similar using `kubectl logs -p precise-code-intel-indexer`.
+- **Docker Compose:**
+	- Determine if the pod was OOM killed using `docker inspect -f '{{json .State}}' precise-code-intel-indexer` (look for `"OOMKilled":true`) and, if so, consider increasing the memory limit of the precise-code-intel-indexer container in `docker-compose.yml`.
+	- Check the logs before the container restarted to see if there are `panic:` messages or similar using `docker logs precise-code-intel-indexer` (note this will include logs from the previous and currently running container).
+
+# precise-code-intel-indexer: container_memory_usage
+
+**Descriptions:**
+
+- _precise-code-intel-indexer: 90%+ container memory usage by instance (not available on k8s or server)_
+
+**Possible solutions:**
+
+- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`.
+- **Docker Compose:** Consider increasing `memory:` of precise-code-intel-indexer container in `docker-compose.yml`.
+
+# precise-code-intel-indexer: container_cpu_usage
+
+**Descriptions:**
+
+- _precise-code-intel-indexer: 90%+ container cpu usage total (5m average) across all cores by instance (not available on k8s or server)_
+
+**Possible solutions:**
+
+- **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`.
+- **Docker Compose:** Consider increasing `cpus:` of the precise-code-intel-indexer container in `docker-compose.yml`.
+
 # query-runner: frontend_internal_api_error_responses
 
 **Descriptions:**

enterprise/dev/ci/ci/pipeline-steps.go

@@ -21,6 +21,7 @@ var allDockerImages = []string{
 	"symbols",
 	"precise-code-intel-bundle-manager",
 	"precise-code-intel-worker",
+	"precise-code-intel-indexer",
 
 	// Images under docker-images/
 	"cadvisor",

monitoring/generator.go

@@ -825,6 +825,7 @@ func main() {
 		GitHubProxy(),
 		PreciseCodeIntelBundleManager(),
 		PreciseCodeIntelWorker(),
+		PreciseCodeIntelIndexer(),
 		QueryRunner(),
 		Replacer(),
 		RepoUpdater(),

monitoring/precise_code_intel_indexer.go

@@ -0,0 +1,31 @@
+package main
+
+func PreciseCodeIntelIndexer() *Container {
+	return &Container{
+		Name:        "precise-code-intel-indexer",
+		Title:       "Precise Code Intel Indexer",
+		Description: "Automatically indexes from popular, active Go repositories.",
+		Groups: []Group{
+			{
+				Title:  "Internal service requests",
+				Hidden: true,
+				Rows: []Row{
+					{
+						sharedFrontendInternalAPIErrorResponses("precise-code-intel-indexer"),
+					},
+				},
+			},
+			{
+				Title:  "Container monitoring (not available on k8s or server)",
+				Hidden: true,
+				Rows: []Row{
+					{
+						sharedContainerRestarts("precise-code-intel-indexer"),
+						sharedContainerMemoryUsage("precise-code-intel-indexer"),
+						sharedContainerCPUUsage("precise-code-intel-indexer"),
+					},
+				},
+			},
+		},
+	}
+}