observation: disable trace logs in dev (#34703)

Explicit logging of trace logs was added because Datadog does not support OT/OTel trace logs/events at INFO level because we do not want these logs to be lost if we have no other way of seeing trace log events. This PR disables it for dev environments, and optionally for other environments, because it can get a bit noisy.

internal/observation/observation.go

@@ -64,6 +64,7 @@ package observation
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	otlog "github.com/opentracing/opentracing-go/log"
@@ -80,6 +81,11 @@ import (
 	"github.com/sourcegraph/sourcegraph/lib/log"
 )
 
+// enableTraceLog toggles whether TraceLogger.Log events should be logged at info level.
+// This is useful in dev environments, or in environments where OpenTracing/OpenTelemetry
+// logs/events are supported (i.e. not Datadog).
+var enableTraceLog = os.Getenv("SRC_TRACE_LOG") != "false"
+
 // Context carries context about where to send logs, trace spans, and register
 // metrics. It should be created once on service startup, and passed around to
 // any location that wants to use it for observing operations.
@@ -226,9 +232,11 @@ func (t *traceLogger) Log(fields ...otlog.Field) {
 	if t.trace != nil {
 		t.trace.LogFields(fields...)
 	}
-	t.Logger.
-		AddCallerSkip(1). // Log() -> Logger
-		Info("trace.log", toLogFields(fields)...)
+	if enableTraceLog {
+		t.Logger.
+			AddCallerSkip(1). // Log() -> Logger
+			Info("trace.log", toLogFields(fields)...)
+	}
 }
 
 func (t *traceLogger) Tag(fields ...otlog.Field) {

sg.config.yaml

@@ -10,6 +10,7 @@ env:
   SRC_REPOS_DIR: $HOME/.sourcegraph/repos
   SRC_LOG_LEVEL: info
   SRC_LOG_FORMAT: condensed
+  SRC_TRACE_LOG: false
   # Set this to true to show an iTerm link to the file:line where the log message came from
   SRC_LOG_SOURCE_LINK: false
   SRC_GIT_SERVER_1: 127.0.0.1:3178