mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 15:51:43 +00:00
2c7d217730
Closes https://linear.app/sourcegraph/issue/SRC-369/finalize-memory-tracking-in-gitserver-on-linux This PR adds a memory tracking feature to all gitserver command, using the utilities introduced in https://app.graphite.dev/github/pr/sourcegraph/sourcegraph/62803. When the GITSERVER_MEMORY_OBSERVATION_ENABLED env var is set, an observer will be spawned that keeps track of the memory usage of the git invocation. This information is available in traces. Additionally, we'll print WARN logs if the commands uses more than 100 MB. ## Test plan Run sg start, and manually tweak the high memory usage threshold down from 500 MB to 1MB. ```diff diff --git a/cmd/gitserver/internal/git/gitcli/command.go b/cmd/gitserver/internal/git/gitcli/command.go index 393fa3c91e6..cf2630fc830 100644 --- a/cmd/gitserver/internal/git/gitcli/command.go +++ b/cmd/gitserver/internal/git/gitcli/command.go @@ -297,7 +297,7 @@ func (rc *cmdReader) waitCmd() error { return rc.err } -const highMemoryUsageThreshold = 500 * 1024 * 1024 // 500 MiB +const highMemoryUsageThreshold = 1 * 1024 * 1024 // 500 MiB func (rc *cmdReader) trace() { duration := time.Since(rc.cmdStart) ``` See log lines similar to the following when I run `sg start` (note the memory usage information in the log fields): ``` [ gitserver-1] WARN gitserver gitcli/command.go:363 High memory usage exec request {"TraceId": "5881f40e3bbbe4d26ec0f32b7f64f535", "SpanId": "a68818dc9f1f9ee2", "ev.Fields": {"cmd": "config", "cmd_ru_maxrss_kib": "5056", "cmd_ru_maxrss_human_readable": "5.2 MB", "traceID": "5881f40e3bbbe4d26ec0f32b7f64f535", "cmd_ru_inblock": "0", "args": "[git config --unset-all gc.auto]", "actor": "0", "cmd_duration_ms": "6", "user_time_ms": "2", "system_time_ms": "2", "repo": "github.com/sgtest/typescript-deps", "error": "git command [git config --unset-all gc.auto] failed with status code 5 (output: \"\")", "cmd_ru_majflt": "10", "cmd_ru_oublock": "0", "trace": "https://sourcegraph.test:3443/-/debug/jaeger/trace/5881f40e3bbbe4d26ec0f32b7f64f535", "exit_status": "5", "cmd_ru_minflt": "494"}} [ gitserver-1] WARN gitserver gitcli/command.go:363 High memory usage exec request {"TraceId": "93cb7e9b3a42cfc085f570b8ad4a2ded", "SpanId": "c35ecddb6ef89e24", "ev.Fields": {"cmd_duration_ms": "6", "system_time_ms": "2", "cmd_ru_maxrss_kib": "5072", "cmd_ru_maxrss_human_readable": "5.2 MB", "cmd_ru_minflt": "495", "cmd_ru_majflt": "10", "actor": "0", "exit_status": "5", "user_time_ms": "2", "cmd_ru_oublock": "0", "traceID": "93cb7e9b3a42cfc085f570b8ad4a2ded", "trace": "https://sourcegraph.test:3443/-/debug/jaeger/trace/93cb7e9b3a42cfc085f570b8ad4a2ded", "repo": "github.com/sgtest/update-ids-test-base", "cmd": "config", "args": "[git config --unset-all gc.auto]", "error": "git command [git config --unset-all gc.auto] failed with status code 5 (output: \"\")", "cmd_ru_inblock": "0"}} [ gitserver-1] WARN gitserver gitcli/command.go:363 High memory usage exec request {"TraceId": "346f1d04dc869f069b04bcabadec0665", "SpanId": "ec43228229e5f531", "ev.Fields": {"actor": "0", "error": "git command [git config --unset-all gc.auto] failed with status code 5 (output: \"\")", "cmd_ru_maxrss_kib": "4960", "trace": "https://sourcegraph.test:3443/-/debug/jaeger/trace/346f1d04dc869f069b04bcabadec0665", "args": "[git config --unset-all gc.auto]", "exit_status": "5", "cmd_duration_ms": "7", "system_time_ms": "2", "repo": "github.com/sgtest/utf8_test", "user_time_ms": "2", "cmd_ru_maxrss_human_readable": "5.1 MB", "cmd_ru_minflt": "487", "cmd_ru_inblock": "0", "cmd_ru_oublock": "0", "traceID": "346f1d04dc869f069b04bcabadec0665", "cmd": "config", "cmd_ru_majflt": "10"}} [ gitserver-1] WARN gitserver gitcli/command.go:363 High memory usage exec request {"TraceId": "1b6a65835e3f75e7e83c8fe7355baeb2", "SpanId": "d525ac1f8c077184", "ev.Fields": {"cmd_ru_oublock": "0", "repo": "github.com/sourcegraph/about", "args": "[git config --unset-all gc.auto]", "cmd_duration_ms": "7", "system_time_ms": "2", "cmd_ru_inblock": "0", "exit_status": "5", "error": "git command [git config --unset-all gc.auto] failed with status code 5 (output: \"\")", "cmd_ru_maxrss_kib": "4912", "cmd_ru_maxrss_human_readable": "5.0 MB", "cmd_ru_minflt": "483", "cmd": "config", "user_time_ms": "2", "trace": "https://sourcegraph.test:3443/-/debug/jaeger/trace/1b6a65835e3f75e7e83c8fe7355baeb2", "actor": "0", "cmd_ru_majflt": "10", "traceID": "1b6a65835e3f75e7e83c8fe7355baeb2"}} [ gitserver-1] WARN gitserver gitcli/command.go:363 High memory usage exec request {"TraceId": "39bbc0cc8b99292e6d3b6dfc067d4049", "SpanId": "60e4d69a25a3074b", "ev.Fields": {"args": "[git config --unset-all gc.auto]", "cmd_duration_ms": "7", "cmd_ru_maxrss_kib": "5056", "cmd_ru_minflt": "492", "trace": "https://sourcegraph.test:3443/-/debug/jaeger/trace/39bbc0cc8b99292e6d3b6dfc067d4049", "actor": "0", "system_time_ms": "2", "cmd_ru_inblock": "0", "cmd": "config", "exit_status": "5", "error": "git command [git config --unset-all gc.auto] failed with status code 5 (output: \"\")", "user_time_ms": "2", "cmd_ru_maxrss_human_readable": "5.2 MB", "cmd_ru_oublock": "0", "repo": "github.com/sourcegraph-testing/etcd", "cmd_ru_majflt": "10", "traceID": "39bbc0cc8b99292e6d3b6dfc067d4049"}} [ ``` Note that I have not tested this e2e in a linux system, but I think it's fine to test it on sourcegraph.com since: 1) we have the benchmarks / integration tests from the previous PR 2) it's behind a environment variable that is off by default ## Changelog Adds a new experimental feature to enable track of `git` command memory invocations when the `GITSERVER_MEMORY_OBSERVATION_ENABLED` environment variable is true (off by default).
97 lines
2.8 KiB
Python
97 lines
2.8 KiB
Python
load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
|
load("//dev:go_defs.bzl", "go_test")
|
|
|
|
go_library(
|
|
name = "gitcli",
|
|
srcs = [
|
|
"archivereader.go",
|
|
"blame.go",
|
|
"clibackend.go",
|
|
"command.go",
|
|
"commitlog.go",
|
|
"commits.go",
|
|
"config.go",
|
|
"contributors.go",
|
|
"diff.go",
|
|
"exec.go",
|
|
"head.go",
|
|
"mergebase.go",
|
|
"metrics.go",
|
|
"object.go",
|
|
"odb.go",
|
|
"refs.go",
|
|
"resolverevision.go",
|
|
"revattime.go",
|
|
"util.go",
|
|
],
|
|
importpath = "github.com/sourcegraph/sourcegraph/cmd/gitserver/internal/git/gitcli",
|
|
tags = [TAG_PLATFORM_SOURCE],
|
|
visibility = ["//cmd/gitserver:__subpackages__"],
|
|
deps = [
|
|
"//cmd/gitserver/internal/common",
|
|
"//cmd/gitserver/internal/git",
|
|
"//internal/actor",
|
|
"//internal/api",
|
|
"//internal/bytesize",
|
|
"//internal/byteutils",
|
|
"//internal/env",
|
|
"//internal/fileutil",
|
|
"//internal/gitserver/gitdomain",
|
|
"//internal/honey",
|
|
"//internal/lazyregexp",
|
|
"//internal/memcmd",
|
|
"//internal/trace",
|
|
"//internal/wrexec",
|
|
"//lib/errors",
|
|
"@com_github_dustin_go_humanize//:go-humanize",
|
|
"@com_github_go_git_go_git_v5//plumbing/format/config",
|
|
"@com_github_hashicorp_golang_lru_v2//:golang-lru",
|
|
"@com_github_prometheus_client_golang//prometheus",
|
|
"@com_github_prometheus_client_golang//prometheus/promauto",
|
|
"@com_github_sourcegraph_log//:log",
|
|
"@io_opentelemetry_go_otel//attribute",
|
|
],
|
|
)
|
|
|
|
go_test(
|
|
name = "gitcli_test",
|
|
srcs = [
|
|
"archivereader_test.go",
|
|
"blame_test.go",
|
|
"command_test.go",
|
|
"commitlog_test.go",
|
|
"commits_test.go",
|
|
"config_test.go",
|
|
"contributors_test.go",
|
|
"diff_test.go",
|
|
"exec_test.go",
|
|
"head_test.go",
|
|
"mergebase_test.go",
|
|
"object_test.go",
|
|
"odb_test.go",
|
|
"refs_test.go",
|
|
"resolverevision_test.go",
|
|
"revattime_test.go",
|
|
"util_test.go",
|
|
],
|
|
embed = [":gitcli"],
|
|
embedsrcs = ["fixtures/git-diff-java-langserver/output.hex"],
|
|
tags = [TAG_PLATFORM_SOURCE],
|
|
deps = [
|
|
"//cmd/gitserver/internal/common",
|
|
"//cmd/gitserver/internal/git",
|
|
"//internal/api",
|
|
"//internal/fileutil",
|
|
"//internal/gitserver",
|
|
"//internal/gitserver/gitdomain",
|
|
"//internal/wrexec",
|
|
"//lib/errors",
|
|
"@com_github_go_git_go_git_v5//plumbing/format/config",
|
|
"@com_github_google_go_cmp//cmp",
|
|
"@com_github_google_go_cmp//cmp/cmpopts",
|
|
"@com_github_sourcegraph_log//logtest",
|
|
"@com_github_stretchr_testify//assert",
|
|
"@com_github_stretchr_testify//require",
|
|
],
|
|
)
|