dev: enable OpenTelemetry by default in dev, improve tracer logging (#40946)

Since observability components are not run by default in sg start, we also amend the error logging of the exporter to suppress issues unless observability.tracing.debug is enabled.

doc/dev/how-to/otel_local_dev.md

@@ -6,7 +6,8 @@ General OpenTelemetry export configuration is done via environment variables acc
 
 ## Collector
 
-`sg start otel` -> runs `otel-collector` and `jaeger`. Additional configuration can be provided to export to different destinations - for example, to configure a simple Honeycomb exporter, add the following to your `sg.config.overwrite.yaml`:
+`sg start otel` runs `otel-collector` and `jaeger`, and configures `otel-collector` to forward traces to Jaeger.
+Additional configuration can be provided to export to different destinations - for example, to configure a simple Honeycomb exporter, add the following to your `sg.config.overwrite.yaml`:
 
 ```yaml
 commands:
@@ -20,18 +21,7 @@ To learn more, see [`docker-images/opentelemetry-collector`](https://github.com/
 
 ## Configuration
 
-Set `dev-private` site config to use `"observability.tracing": { "type": "opentelemetry" }` to enable OpenTelemetry export for most services.
-
-To enable Zoekt OpenTelemetry, add the following to your `sg.config.overwrite.yaml`:
-
-```yaml
-commands:
-  zoekt-web-0: &zoekt_otel
-    JAEGER_DISABLED: true
-    OPENTELEMETRY_DISABLED: false
-  zoekt-web-1:
-    <<: *zoekt_otel
-```
+Set `dev-private` site config to use `"observability.tracing": { "type": "opentelemetry" }` to enable OpenTelemetry export for most services - this should be set by default in the latest versions of `dev-private`.
 
 ## Testing
 

internal/tracer/otel.go

@@ -63,10 +63,22 @@ func newOTelBridgeTracer(logger log.Logger, exporter oteltracesdk.SpanExporter,
 	otBridgeTracer.SetTextMapPropagator(compositePropagator)
 
 	// Set up logging
-	otelLogger := logger.AddCallerSkip(2) // no additional scope needed, this is already otel scope
-	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) { otelLogger.Warn("error encountered", log.Error(err)) }))
+	otelLogger := logger.AddCallerSkip(2).Scoped("otel", "OpenTelemetry library")
+	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
+		if debug {
+			otelLogger.Warn("error encountered", log.Error(err))
+		} else {
+			otelLogger.Debug("error encountered", log.Error(err))
+		}
+	}))
 	bridgeLogger := logger.AddCallerSkip(2).Scoped("bridge", "OpenTracing to OpenTelemetry compatibility layer")
-	otBridgeTracer.SetWarningHandler(func(msg string) { bridgeLogger.Debug(msg) })
+	otBridgeTracer.SetWarningHandler(func(msg string) {
+		if debug {
+			bridgeLogger.Warn(msg)
+		} else {
+			bridgeLogger.Debug(msg)
+		}
+	})
 
 	// Done
 	return otBridgeTracer, otelTracerProvider, &otelBridgeCloser{provider}, nil

sg.config.yaml

@@ -420,7 +420,8 @@ commands:
       env GOBIN="${PWD}/.bin" go install github.com/sourcegraph/zoekt/cmd/zoekt-webserver
     checkBinary: .bin/zoekt-webserver
     env:
-      JAEGER_DISABLED: false
+      JAEGER_DISABLED: true
+      OPENTELEMETRY_DISABLED: false
       GOGC: 25
 
   zoekt-web-0:
@@ -804,6 +805,7 @@ commandsets:
       - minio
       - codeintel-worker
       - codeintel-executor
+      - otel-collector
       - jaeger
       - grafana
       - prometheus
@@ -903,6 +905,7 @@ commandsets:
       - docker
     commands:
       - jaeger
+      - otel-collector
       - prometheus
       - grafana
       - postgres_exporter