worker: Add skeleton service (#21768)

cmd/server/shared/shared.go

@@ -141,6 +141,7 @@ func Main() {
 		`symbols: symbols`,
 		`searcher: searcher`,
 		`github-proxy: github-proxy`,
+		`worker: worker`,
 		`repo-updater: repo-updater`,
 		`syntect_server: sh -c 'env QUIET=true ROCKET_ENV=production ROCKET_PORT=9238 ROCKET_LIMITS='"'"'{json=10485760}'"'"' ROCKET_SECRET_KEY='"'"'SeerutKeyIsI7releuantAndknvsuZPluaseIgnorYA='"'"' ROCKET_KEEP_ALIVE=0 ROCKET_ADDRESS='"'"'"127.0.0.1"'"'"' syntect_server | grep -v "Rocket has launched" | grep -v "Warning: environment is"' | grep -v 'Configured for production'`,
 		postgresExporterLine,

cmd/worker/CODENOTIFY

@@ -0,0 +1,3 @@
+# See https://github.com/sourcegraph/codenotify for documentation.
+
+**/* @efritz

cmd/worker/Dockerfile

@@ -0,0 +1,19 @@
+FROM sourcegraph/alpine:3.12@sha256:ce099fbcd3cf70b338fc4cb2a4e1fa9ae847de21afdb0a849a393b87d94fb174
+
+ARG COMMIT_SHA="unknown"
+ARG DATE="unknown"
+ARG VERSION="unknown"
+
+LABEL org.opencontainers.image.revision=${COMMIT_SHA}
+LABEL org.opencontainers.image.created=${DATE}
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}
+
+# hadolint ignore=DL3018
+RUN apk update && apk add --no-cache \
+    tini
+
+USER sourcegraph
+EXPOSE 3189
+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/worker"]
+COPY worker /usr/local/bin/

cmd/worker/README.md

@@ -0,0 +1,5 @@
+# Worker
+
+The worker service is a collection of the background tasks performed by a Sourcegraph instance. Tasks registered to the worker will run periodically or in response to some event read from the database.
+
+Currently, no tasks are registered to the worker.

cmd/worker/build.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script builds the worker docker image.
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../.."
+set -eu
+
+OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
+cleanup() {
+  rm -rf "$OUTPUT"
+}
+trap cleanup EXIT
+
+# Environment for building linux binaries
+export GO111MODULE=on
+export GOARCH=amd64
+export GOOS=linux
+export CGO_ENABLED=0
+
+echo "--- go build"
+pkg="github.com/sourcegraph/sourcegraph/cmd/worker"
+go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg"
+
+echo "--- docker build"
+docker build -f cmd/worker/Dockerfile -t "$IMAGE" "$OUTPUT" \
+  --progress=plain \
+  --build-arg COMMIT_SHA \
+  --build-arg DATE \
+  --build-arg VERSION

cmd/worker/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/sourcegraph/sourcegraph/cmd/worker/shared"
+
+func main() {
+	shared.Main()
+}

cmd/worker/shared/main.go

@@ -0,0 +1,42 @@
+package shared
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/sourcegraph/sourcegraph/internal/debugserver"
+	"github.com/sourcegraph/sourcegraph/internal/env"
+	"github.com/sourcegraph/sourcegraph/internal/goroutine"
+	"github.com/sourcegraph/sourcegraph/internal/httpserver"
+	"github.com/sourcegraph/sourcegraph/internal/logging"
+	"github.com/sourcegraph/sourcegraph/internal/trace"
+	"github.com/sourcegraph/sourcegraph/internal/tracer"
+)
+
+const addr = ":3189"
+
+func Main() {
+	env.Lock()
+	env.HandleHelpFlag()
+	logging.Init()
+	tracer.Init()
+	trace.Init(true)
+
+	// Start debug server
+	ready := make(chan struct{})
+	go debugserver.NewServerRoutine(ready).Start()
+
+	var allRoutines []goroutine.BackgroundRoutine
+
+	// Initialize health server
+	server := httpserver.NewFromAddr(addr, &http.Server{
+		ReadTimeout:  75 * time.Second,
+		WriteTimeout: 10 * time.Minute,
+		Handler:      httpserver.NewHandler(nil),
+	})
+	allRoutines = append(allRoutines, server)
+
+	close(ready)
+	goroutine.MonitorBackgroundRoutines(context.Background(), allRoutines...)
+}

dev/Procfile

@@ -1,5 +1,6 @@
 gitserver: env HOSTNAME=$SRC_GIT_SERVER_1 gitserver
 query-runner: query-runner
+worker: worker
 repo-updater: repo-updater
 searcher: searcher
 symbols: symbols

dev/prometheus/all/prometheus_targets.yml

@@ -46,6 +46,11 @@
   targets:
     # precise-code-intel-worker
     - host.docker.internal:6088
+- labels:
+    job: worker
+  targets:
+    # worker
+    - host.docker.internal:6089
 - labels:
     job: executor-queue
   targets:

dev/prometheus/linux/prometheus_targets.yml

@@ -47,6 +47,10 @@
     # precise-code-intel-worker
     - 127.0.0.1:6088
 - labels:
+    job: worker
+  targets:
+    # worker
+    - 127.0.0.1:6089
 - labels:
     job: executor-queue
   targets:

dev/src-prof-services.json

@@ -6,6 +6,7 @@
   { "Name": "repo-updater", "Host": "127.0.0.1:6074" },
   { "Name": "query-runner", "Host": "127.0.0.1:6067" },
   { "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" },
+  { "Name": "worker", "Host": "127.0.0.1:6089" },
   { "Name": "executor-queue", "Host": "127.0.0.1:6091" },
   { "Name": "executor", "Host": "127.0.0.1:6092" },
   { "Name": "zoekt-indexserver-0", "Host": "127.0.0.1:6072" },

doc/admin/observability/alert_solutions.md

@@ -3128,6 +3128,247 @@ To learn more about Sourcegraph's alerting and how to set up alerts, see [our al
 
 <br />
 
+## worker: frontend_internal_api_error_responses
+
+<p class="subtitle">frontend-internal API error responses every 5m by route</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 2%+ frontend-internal API error responses every 5m by route for 5m0s
+
+**Possible solutions**
+
+- **Single-container deployments:** Check `docker logs $CONTAINER_ID` for logs starting with `repo-updater` that indicate requests to the frontend service are failing.
+- **Kubernetes:**
+	- Confirm that `kubectl get pods` shows the `frontend` pods are healthy.
+	- Check `kubectl logs worker` for logs indicate request failures to `frontend` or `frontend-internal`.
+- **Docker Compose:**
+	- Confirm that `docker ps` shows the `frontend-internal` container is healthy.
+	- Check `docker logs worker` for logs indicating request failures to `frontend` or `frontend-internal`.
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_frontend_internal_api_error_responses"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: container_cpu_usage
+
+<p class="subtitle">container cpu usage total (1m average) across all cores by instance</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 99%+ container cpu usage total (1m average) across all cores by instance
+
+**Possible solutions**
+
+- **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`.
+- **Docker Compose:** Consider increasing `cpus:` of the worker container in `docker-compose.yml`.
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_container_cpu_usage"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: container_memory_usage
+
+<p class="subtitle">container memory usage by instance</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 99%+ container memory usage by instance
+
+**Possible solutions**
+
+- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`.
+- **Docker Compose:** Consider increasing `memory:` of worker container in `docker-compose.yml`.
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_container_memory_usage"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: provisioning_container_cpu_usage_long_term
+
+<p class="subtitle">container cpu usage total (90th percentile over 1d) across all cores by instance</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 80%+ container cpu usage total (90th percentile over 1d) across all cores by instance for 336h0m0s
+
+**Possible solutions**
+
+- **Kubernetes:** Consider increasing CPU limits in the `Deployment.yaml` for the worker service.
+- **Docker Compose:** Consider increasing `cpus:` of the worker container in `docker-compose.yml`.
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_provisioning_container_cpu_usage_long_term"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: provisioning_container_memory_usage_long_term
+
+<p class="subtitle">container memory usage (1d maximum) by instance</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 80%+ container memory usage (1d maximum) by instance for 336h0m0s
+
+**Possible solutions**
+
+- **Kubernetes:** Consider increasing memory limits in the `Deployment.yaml` for the worker service.
+- **Docker Compose:** Consider increasing `memory:` of the worker container in `docker-compose.yml`.
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_provisioning_container_memory_usage_long_term"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: provisioning_container_cpu_usage_short_term
+
+<p class="subtitle">container cpu usage total (5m maximum) across all cores by instance</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 90%+ container cpu usage total (5m maximum) across all cores by instance for 30m0s
+
+**Possible solutions**
+
+- **Kubernetes:** Consider increasing CPU limits in the the relevant `Deployment.yaml`.
+- **Docker Compose:** Consider increasing `cpus:` of the worker container in `docker-compose.yml`.
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_provisioning_container_cpu_usage_short_term"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: provisioning_container_memory_usage_short_term
+
+<p class="subtitle">container memory usage (5m maximum) by instance</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 90%+ container memory usage (5m maximum) by instance
+
+**Possible solutions**
+
+- **Kubernetes:** Consider increasing memory limit in relevant `Deployment.yaml`.
+- **Docker Compose:** Consider increasing `memory:` of worker container in `docker-compose.yml`.
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_provisioning_container_memory_usage_short_term"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: go_goroutines
+
+<p class="subtitle">maximum active goroutines</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 10000+ maximum active goroutines for 10m0s
+
+**Possible solutions**
+
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_go_goroutines"
+]
+```
+
+> NOTE: More help interpreting this metric is available in the [dashboards reference](./dashboards.md#worker-go-goroutines).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: go_gc_duration_seconds
+
+<p class="subtitle">maximum go garbage collection duration</p>
+
+**Descriptions**
+
+- <span class="badge badge-warning">warning</span> worker: 2s+ maximum go garbage collection duration
+
+**Possible solutions**
+
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "warning_worker_go_gc_duration_seconds"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+## worker: pods_available_percentage
+
+<p class="subtitle">percentage pods available</p>
+
+**Descriptions**
+
+- <span class="badge badge-critical">critical</span> worker: less than 90% percentage pods available for 10m0s
+
+**Possible solutions**
+
+- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
+
+```json
+"observability.silenceAlerts": [
+  "critical_worker_pods_available_percentage"
+]
+```
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
 ## repo-updater: frontend_internal_api_error_responses
 
 <p class="subtitle">frontend-internal API error responses every 5m by route</p>

doc/admin/observability/dashboards.md

@@ -1847,6 +1847,140 @@ This panel indicates percentage pods available.
 
 <br />
 
+## Worker
+
+<p class="subtitle">Manages background processes.</p>
+
+### Worker: Internal service requests
+
+#### worker: frontend_internal_api_error_responses
+
+This panel indicates frontend-internal API error responses every 5m by route.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-frontend-internal-api-error-responses).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+### Worker: Container monitoring (not available on server)
+
+#### worker: container_cpu_usage
+
+This panel indicates container cpu usage total (1m average) across all cores by instance.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-container-cpu-usage).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+#### worker: container_memory_usage
+
+This panel indicates container memory usage by instance.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-container-memory-usage).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+#### worker: container_missing
+
+This panel indicates container missing.
+
+This value is the number of times a container has not been seen for more than one minute. If you observe this
+value change independent of deployment events (such as an upgrade), it could indicate pods are being OOM killed or terminated for some other reasons.
+
+- **Kubernetes:**
+	- Determine if the pod was OOM killed using `kubectl describe pod worker` (look for `OOMKilled: true`) and, if so, consider increasing the memory limit in the relevant `Deployment.yaml`.
+	- Check the logs before the container restarted to see if there are `panic:` messages or similar using `kubectl logs -p worker`.
+- **Docker Compose:**
+	- Determine if the pod was OOM killed using `docker inspect -f '{{json .State}}' worker` (look for `"OOMKilled":true`) and, if so, consider increasing the memory limit of the worker container in `docker-compose.yml`.
+	- Check the logs before the container restarted to see if there are `panic:` messages or similar using `docker logs worker` (note this will include logs from the previous and currently running container).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+### Worker: Provisioning indicators (not available on server)
+
+#### worker: provisioning_container_cpu_usage_long_term
+
+This panel indicates container cpu usage total (90th percentile over 1d) across all cores by instance.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-provisioning-container-cpu-usage-long-term).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+#### worker: provisioning_container_memory_usage_long_term
+
+This panel indicates container memory usage (1d maximum) by instance.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-provisioning-container-memory-usage-long-term).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+#### worker: provisioning_container_cpu_usage_short_term
+
+This panel indicates container cpu usage total (5m maximum) across all cores by instance.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-provisioning-container-cpu-usage-short-term).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+#### worker: provisioning_container_memory_usage_short_term
+
+This panel indicates container memory usage (5m maximum) by instance.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-provisioning-container-memory-usage-short-term).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+### Worker: Golang runtime monitoring
+
+#### worker: go_goroutines
+
+This panel indicates maximum active goroutines.
+
+A high value here indicates a possible goroutine leak.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-go-goroutines).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+#### worker: go_gc_duration_seconds
+
+This panel indicates maximum go garbage collection duration.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-go-gc-duration-seconds).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
+### Worker: Kubernetes monitoring (only available on Kubernetes)
+
+#### worker: pods_available_percentage
+
+This panel indicates percentage pods available.
+
+> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./alert_solutions.md#worker-pods-available-percentage).
+
+<sub>*Managed by the [Sourcegraph Code-intelligence team](https://about.sourcegraph.com/handbook/engineering/code-intelligence).*</sub>
+
+<br />
+
 ## Repo Updater
 
 <p class="subtitle">Manages interaction with code hosts, instructs Gitserver to update repositories.</p>

enterprise/cmd/server/build.sh

@@ -8,5 +8,6 @@ export SERVER_PKG=${SERVER_PKG:-github.com/sourcegraph/sourcegraph/enterprise/cm
 
 ./cmd/server/build.sh \
   github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend \
+  github.com/sourcegraph/sourcegraph/enterprise/cmd/worker \
   github.com/sourcegraph/sourcegraph/enterprise/cmd/repo-updater \
   github.com/sourcegraph/sourcegraph/enterprise/cmd/precise-code-intel-worker

enterprise/cmd/worker/CODENOTIFY

@@ -0,0 +1,3 @@
+# See https://github.com/sourcegraph/codenotify for documentation.
+
+**/* @efritz

enterprise/cmd/worker/Dockerfile

@@ -0,0 +1,19 @@
+FROM sourcegraph/alpine:3.12@sha256:ce099fbcd3cf70b338fc4cb2a4e1fa9ae847de21afdb0a849a393b87d94fb174
+
+ARG COMMIT_SHA="unknown"
+ARG DATE="unknown"
+ARG VERSION="unknown"
+
+LABEL org.opencontainers.image.revision=${COMMIT_SHA}
+LABEL org.opencontainers.image.created=${DATE}
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL com.sourcegraph.github.url=https://github.com/sourcegraph/sourcegraph/commit/${COMMIT_SHA}
+
+# hadolint ignore=DL3018
+RUN apk update && apk add --no-cache \
+    tini
+
+USER sourcegraph
+EXPOSE 3189
+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/worker"]
+COPY worker /usr/local/bin/

enterprise/cmd/worker/README.md

@@ -0,0 +1,3 @@
+# Enterprise worker
+
+The enterprise worker supplements the OSS worker with additional enterprise-relevant background tasks.

enterprise/cmd/worker/build.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# This script builds the enterprise worker docker image.
+
+cd "$(dirname "${BASH_SOURCE[0]}")/../../.."
+set -eu
+
+OUTPUT=$(mktemp -d -t sgdockerbuild_XXXXXXX)
+cleanup() {
+  rm -rf "$OUTPUT"
+}
+trap cleanup EXIT
+
+# Environment for building linux binaries
+export GO111MODULE=on
+export GOARCH=amd64
+export GOOS=linux
+export CGO_ENABLED=0
+
+echo "--- go build"
+pkg="github.com/sourcegraph/sourcegraph/enterprise/cmd/worker"
+go build -trimpath -ldflags "-X github.com/sourcegraph/sourcegraph/internal/version.version=$VERSION -X github.com/sourcegraph/sourcegraph/internal/version.timestamp=$(date +%s)" -buildmode exe -tags dist -o "$OUTPUT/$(basename $pkg)" "$pkg"
+
+echo "--- docker build"
+docker build -f enterprise/cmd/worker/Dockerfile -t "$IMAGE" "$OUTPUT" \
+  --progress=plain \
+  --build-arg COMMIT_SHA \
+  --build-arg DATE \
+  --build-arg VERSION

enterprise/cmd/worker/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"log"
+	"os"
+	"strconv"
+
+	"github.com/sourcegraph/sourcegraph/cmd/worker/shared"
+)
+
+func main() {
+	debug, _ := strconv.ParseBool(os.Getenv("DEBUG"))
+	if debug {
+		log.Println("enterprise edition")
+	}
+
+	shared.Main()
+}

enterprise/dev/ci/images/images.go

@@ -37,6 +37,7 @@ var SourcegraphDockerImages = []string{
 	"gitserver",
 	"query-runner",
 	"repo-updater",
+	"worker",
 	"searcher",
 	"symbols",
 	"precise-code-intel-worker",

enterprise/dev/src-prof-services.json

@@ -5,5 +5,6 @@
   { "Name": "symbols", "Host": "127.0.0.1:6071" },
   { "Name": "repo-updater", "Host": "127.0.0.1:6074" },
   { "Name": "query-runner", "Host": "127.0.0.1:6067" },
-  { "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" }
+  { "Name": "precise-code-intel-worker", "Host": "127.0.0.1:6088" },
+  { "Name": "worker", "Host": "127.0.0.1:6089" }
 ]

monitoring/definitions/worker.go

@@ -0,0 +1,71 @@
+package definitions
+
+import (
+	"github.com/sourcegraph/sourcegraph/monitoring/definitions/shared"
+	"github.com/sourcegraph/sourcegraph/monitoring/monitoring"
+)
+
+func Worker() *monitoring.Container {
+	return &monitoring.Container{
+		Name:        "worker",
+		Title:       "Worker",
+		Description: "Manages background processes.",
+		Groups: []monitoring.Group{
+			{
+				Title:  "Internal service requests",
+				Hidden: true,
+				Rows: []monitoring.Row{
+					{
+						shared.FrontendInternalAPIErrorResponses("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+					},
+				},
+			},
+			{
+				Title:  shared.TitleContainerMonitoring,
+				Hidden: true,
+				Rows: []monitoring.Row{
+					{
+						shared.ContainerCPUUsage("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+						shared.ContainerMemoryUsage("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+					},
+					{
+						shared.ContainerMissing("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+					},
+				},
+			},
+			{
+				Title:  shared.TitleProvisioningIndicators,
+				Hidden: true,
+				Rows: []monitoring.Row{
+					{
+						shared.ProvisioningCPUUsageLongTerm("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+						shared.ProvisioningMemoryUsageLongTerm("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+					},
+					{
+						shared.ProvisioningCPUUsageShortTerm("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+						shared.ProvisioningMemoryUsageShortTerm("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+					},
+				},
+			},
+			{
+				Title:  shared.TitleGolangMonitoring,
+				Hidden: true,
+				Rows: []monitoring.Row{
+					{
+						shared.GoGoroutines("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+						shared.GoGcDuration("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+					},
+				},
+			},
+			{
+				Title:  shared.TitleKubernetesMonitoring,
+				Hidden: true,
+				Rows: []monitoring.Row{
+					{
+						shared.KubernetesPodsAvailable("worker", monitoring.ObservableOwnerCodeIntel).Observable(),
+					},
+				},
+			},
+		},
+	}
+}

monitoring/main.go

@@ -38,6 +38,7 @@ func main() {
 		definitions.Postgres(),
 		definitions.PreciseCodeIntelWorker(),
 		definitions.QueryRunner(),
+		definitions.Worker(),
 		definitions.RepoUpdater(),
 		definitions.Searcher(),
 		definitions.Symbols(),

sg.config.yaml

@@ -139,6 +139,24 @@ commands:
       - internal
       - cmd/github-proxy
 
+  worker:
+    cmd: ulimit -n 10000 && .bin/worker
+    install: go build -o .bin/worker github.com/sourcegraph/sourcegraph/cmd/worker
+    watch:
+      - lib
+      - internal
+      - cmd/worker
+
+  enterprise-worker:
+    cmd: ulimit -n 10000 && .bin/worker
+    install: go build -o .bin/worker github.com/sourcegraph/sourcegraph/enterprise/cmd/worker
+    watch:
+      - lib
+      - internal
+      - enterprise/internal
+      - cmd/worker
+      - enterprise/cmd/worker
+
   repo-updater:
     cmd: .bin/repo-updater
     install: go build -o .bin/repo-updater github.com/sourcegraph/sourcegraph/cmd/repo-updater
@@ -442,6 +460,7 @@ commandsets:
 
   default:
     - frontend
+    - worker
     - repo-updater
     - gitserver
     - searcher
@@ -459,6 +478,7 @@ commandsets:
 
   enterprise:
     - enterprise-frontend
+    - enterprise-worker
     - enterprise-repo-updater
     - enterprise-web
     - gitserver
@@ -476,6 +496,7 @@ commandsets:
 
   enterprise-codeintel:
     - enterprise-frontend
+    - enterprise-worker
     - enterprise-repo-updater
     - enterprise-web
     - gitserver
@@ -504,6 +525,7 @@ commandsets:
     #     DISABLE_CODE_INSIGHTS: false
     #
     - enterprise-frontend
+    - enterprise-worker
     - enterprise-repo-updater
     - enterprise-web
     - gitserver
@@ -522,6 +544,7 @@ commandsets:
 
   api-only:
     - enterprise-frontend
+    - enterprise-worker
     - enterprise-repo-updater
     - gitserver
     - searcher