artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 18:11:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

monitoring: fix alerting docs, clean up shared observables API (#16971)

Browse Source 
* Fix the alerting guide in adding monitoring
* Reduce the API surface of the monitoring lib by moving shared observable helpers (`WithWarning`, etc.) into a separate type specific to the shared observables

Co-authored-by: uwedeportivo <534011+uwedeportivo@users.noreply.github.com>
This commit is contained in:
Robert Lin 2020-12-29 20:51:00 +08:00 committed by GitHub
parent 8f0064a2fe
commit 00ca7b1b81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 259 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/dev/background-information/observability/index.md
View File

@ -24,7 +24,7 @@ Observability at Sourcegraph includes:
- [How to find monitoring](../../how-to/find_monitoring.md)
- [How to add monitoring](../../how-to/add_monitoring.md)
- [Set up local monitoring development](../../how-to/monitoring_local_dev.md)
- How to add observability (coming soon)
- How to add debugging (coming soon)
## Components

21
doc/dev/how-to/add_monitoring.md
View File

@ -94,7 +94,7 @@ The primary thing you'll use is to change the Grafana display from plain numbers
Name: "some_metric_behaviour",
Description: "some behaviour of a metric over 5m",
Query: `histogram_quantile(0.99, sum by (le)(rate(search_request_duration{status="success}[5m])))`,
+ PanelOptions: PanelOptions().LegendFormat("duration").Unit(Seconds),
+ PanelOptions: monitoring.PanelOptions().LegendFormat("duration").Unit(monitoring.Seconds),
}
```
@ -103,22 +103,23 @@ The primary thing you'll use is to change the Grafana display from plain numbers
Alerts can be defined at two levels: warning, and critical.
They are used to provide Sourcegraph health notifications for site administrators.
This step is optional, but highly recommended.
If you opt not to include an alert, you must explicitly set `NoAlert: true` and [provide relevant documentation for this observable](#add-documentation).
To get started, make a guess about what a good or bad value for your query is.
It's OK if this isn't perfect, just do your best.
Then add an alert to your Observable, for example:
To get started, refer to [understanding alerts](../../admin/observability/alerting.md#understanding-alerts) for what your alert should indicate.
Then make a guess about what a good or bad value for your query is - it's OK if this isn't perfect, just do your best.
You can then use the [ObservableAlertDefinition](https://sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/monitoring/monitoring/README.md#type-observablealertdefinition) to add an alert to your Observable, for example:
```diff
{
Name: "some_metric_behaviour",
Description: "some behaviour of a metric over 5m",
Query: `histogram_quantile(0.99, sum by (le)(rate(search_request_duration{status="success}[5m])))`,
PanelOptions: PanelOptions().LegendFormat("duration").Unit(Seconds),
+ Warning: Alert{GreaterOrEqual: 20},
PanelOptions: monitoring.PanelOptions().LegendFormat("duration").Unit(monitoring.Seconds),
+ Warning: monitoring.Alert().GreaterOrEqual(20),
}
```
This step is optional - if you opt not to include an alert, you must explicitly set `NoAlert: true` and provide [relevant documentation for this observable](#add-documentation).
Options like only alerting after a certain duration (`.For(time.Duration)`) are also available - refer to the [monitoring library reference](https://sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/monitoring/monitoring/README.md#type-observablealertdefinition).
### Add documentation
@ -129,8 +130,8 @@ It's best if you also add some Markdown documentation with your best guess of wh
Name: "some_metric_behaviour",
Description: "some behaviour of a metric over 5m",
Query: `histogram_quantile(0.99, sum by (le)(rate(search_request_duration{status="success}[5m])))`,
Warning: Alert{GreaterOrEqual: 20},
PanelOptions: PanelOptions().LegendFormat("duration").Unit(Seconds),
Warning: monitoring.Alert().GreaterOrEqual(20),
PanelOptions: monitoring.PanelOptions().LegendFormat("duration").Unit(monitoring.Seconds),
+ PossibleSolutions: `
+ - Look at 'SERVICE' logs for details on the slow search queries.
+ `,
@ -143,7 +144,7 @@ It's best if you also add some Markdown documentation with your best guess of wh
Description: "some behaviour of a metric over 5m",
Query: `histogram_quantile(0.99, sum by (le)(rate(search_request_duration{status="success}[5m])))`,
NoAlert: true,
PanelOptions: PanelOptions().LegendFormat("duration").Unit(Seconds),
PanelOptions: monitoring.PanelOptions().LegendFormat("duration").Unit(monitoring.Seconds),
+ Interpretation: `
+ This value might be high under X, Y, and Z conditions.
+ `,

24
monitoring/definitions/executor_queue.go
View File

@ -102,7 +102,7 @@ func ExecutorQueue() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.FrontendInternalAPIErrorResponses("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.FrontendInternalAPIErrorResponses("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -111,12 +111,12 @@ func ExecutorQueue() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ContainerMemoryUsage("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ContainerCPUUsage("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerMemoryUsage("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ContainerRestarts("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ContainerFsInodes("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ContainerRestarts("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerFsInodes("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -125,12 +125,12 @@ func ExecutorQueue() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageLongTerm("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageLongTerm("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageLongTerm("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageShortTerm("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageShortTerm("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageShortTerm("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -139,8 +139,8 @@ func ExecutorQueue() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.GoGcDuration("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.GoGoroutines("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.GoGcDuration("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -149,7 +149,7 @@ func ExecutorQueue() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("executor-queue", monitoring.ObservableOwnerCodeIntel),
shared.KubernetesPodsAvailable("executor-queue", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},

22
monitoring/definitions/frontend.go
View File

@ -760,12 +760,12 @@ func Frontend() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("frontend", monitoring.ObservableOwnerCloud),
shared.ContainerMemoryUsage("frontend", monitoring.ObservableOwnerCloud),
shared.ContainerCPUUsage("frontend", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerMemoryUsage("frontend", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ContainerRestarts("frontend", monitoring.ObservableOwnerCloud),
shared.ContainerFsInodes("frontend", monitoring.ObservableOwnerCloud),
shared.ContainerRestarts("frontend", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerFsInodes("frontend", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -774,12 +774,12 @@ func Frontend() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("frontend", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageLongTerm("frontend", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageLongTerm("frontend", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageLongTerm("frontend", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("frontend", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageShortTerm("frontend", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageShortTerm("frontend", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageShortTerm("frontend", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -788,8 +788,8 @@ func Frontend() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("frontend", monitoring.ObservableOwnerCloud),
shared.GoGcDuration("frontend", monitoring.ObservableOwnerCloud),
shared.GoGoroutines("frontend", monitoring.ObservableOwnerCloud).Observable(),
shared.GoGcDuration("frontend", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -798,7 +798,7 @@ func Frontend() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("frontend", monitoring.ObservableOwnerCloud),
shared.KubernetesPodsAvailable("frontend", monitoring.ObservableOwnerCloud).Observable(),
},
},
},

32
monitoring/definitions/git_server.go
View File

@ -96,7 +96,7 @@ func GitServer() *monitoring.Container {
- **Kubernetes and Docker Compose:** Check that you are running a similar number of git server replicas and that their CPU/memory limits are allocated according to what is shown in the [Sourcegraph resource estimator](../install/resource_estimator.md).
`,
},
shared.FrontendInternalAPIErrorResponses("gitserver", monitoring.ObservableOwnerCloud),
shared.FrontendInternalAPIErrorResponses("gitserver", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -105,12 +105,12 @@ func GitServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("gitserver", monitoring.ObservableOwnerCloud),
shared.ContainerMemoryUsage("gitserver", monitoring.ObservableOwnerCloud),
shared.ContainerCPUUsage("gitserver", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerMemoryUsage("gitserver", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ContainerRestarts("gitserver", monitoring.ObservableOwnerCloud),
shared.ContainerFsInodes("gitserver", monitoring.ObservableOwnerCloud),
shared.ContainerRestarts("gitserver", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerFsInodes("gitserver", monitoring.ObservableOwnerCloud).Observable(),
},
{
{
@ -131,16 +131,16 @@ func GitServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("gitserver", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageLongTerm("gitserver", monitoring.ObservableOwnerCloud).WithNoAlerts(`
Git Server is expected to use up all the memory it is provided.
`),
shared.ProvisioningCPUUsageLongTerm("gitserver", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageLongTerm("gitserver", monitoring.ObservableOwnerCloud).
WithNoAlerts(`Git Server is expected to use up all the memory it is provided.`).
Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("gitserver", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageShortTerm("gitserver", monitoring.ObservableOwnerCloud).WithNoAlerts(`
Git Server is expected to use up all the memory it is provided.
`),
shared.ProvisioningCPUUsageShortTerm("gitserver", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageShortTerm("gitserver", monitoring.ObservableOwnerCloud).
WithNoAlerts(`Git Server is expected to use up all the memory it is provided.`).
Observable(),
},
},
},
@ -149,8 +149,8 @@ func GitServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("gitserver", monitoring.ObservableOwnerCloud),
shared.GoGcDuration("gitserver", monitoring.ObservableOwnerCloud),
shared.GoGoroutines("gitserver", monitoring.ObservableOwnerCloud).Observable(),
shared.GoGcDuration("gitserver", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -159,7 +159,7 @@ func GitServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("gitserver", monitoring.ObservableOwnerCloud),
shared.KubernetesPodsAvailable("gitserver", monitoring.ObservableOwnerCloud).Observable(),
},
},
},

22
monitoring/definitions/github_proxy.go
View File

@ -37,12 +37,12 @@ func GitHubProxy() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("github-proxy", monitoring.ObservableOwnerCloud),
shared.ContainerMemoryUsage("github-proxy", monitoring.ObservableOwnerCloud),
shared.ContainerCPUUsage("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerMemoryUsage("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ContainerRestarts("github-proxy", monitoring.ObservableOwnerCloud),
shared.ContainerFsInodes("github-proxy", monitoring.ObservableOwnerCloud),
shared.ContainerRestarts("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerFsInodes("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -51,12 +51,12 @@ func GitHubProxy() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("github-proxy", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageLongTerm("github-proxy", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageLongTerm("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageLongTerm("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("github-proxy", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageShortTerm("github-proxy", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageShortTerm("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageShortTerm("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -65,8 +65,8 @@ func GitHubProxy() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("github-proxy", monitoring.ObservableOwnerCloud),
shared.GoGcDuration("github-proxy", monitoring.ObservableOwnerCloud),
shared.GoGoroutines("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
shared.GoGcDuration("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -75,7 +75,7 @@ func GitHubProxy() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("github-proxy", monitoring.ObservableOwnerCloud),
shared.KubernetesPodsAvailable("github-proxy", monitoring.ObservableOwnerCloud).Observable(),
},
},
},

16
monitoring/definitions/postgres.go
View File

@ -102,20 +102,20 @@ func Postgres() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm(dbSourcegraph, monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageLongTerm(dbSourcegraph, monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageLongTerm(dbSourcegraph, monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageLongTerm(dbSourcegraph, monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm(dbSourcegraph, monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageShortTerm(dbSourcegraph, monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageShortTerm(dbSourcegraph, monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageShortTerm(dbSourcegraph, monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ProvisioningCPUUsageLongTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageLongTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageLongTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageLongTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageShortTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageShortTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageShortTerm(dbCodeIntel, monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},

22
monitoring/definitions/precise_code_intel_indexer.go
View File

@ -151,12 +151,12 @@ func PreciseCodeIntelIndexer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerMemoryUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerCPUUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerMemoryUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ContainerRestarts("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerFsInodes("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerRestarts("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerFsInodes("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -165,12 +165,12 @@ func PreciseCodeIntelIndexer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -179,8 +179,8 @@ func PreciseCodeIntelIndexer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.GoGcDuration("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.GoGoroutines("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.GoGcDuration("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -189,7 +189,7 @@ func PreciseCodeIntelIndexer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.KubernetesPodsAvailable("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},

24
monitoring/definitions/precise_code_intel_worker.go
View File

@ -207,7 +207,7 @@ func PreciseCodeIntelWorker() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.FrontendInternalAPIErrorResponses("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.FrontendInternalAPIErrorResponses("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -216,12 +216,12 @@ func PreciseCodeIntelWorker() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerMemoryUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerCPUUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerMemoryUsage("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ContainerRestarts("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerFsInodes("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ContainerRestarts("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerFsInodes("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -230,12 +230,12 @@ func PreciseCodeIntelWorker() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageLongTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageShortTerm("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -244,8 +244,8 @@ func PreciseCodeIntelWorker() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.GoGcDuration("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.GoGoroutines("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.GoGcDuration("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -254,7 +254,7 @@ func PreciseCodeIntelWorker() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel),
shared.KubernetesPodsAvailable("precise-code-intel-worker", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},

18
monitoring/definitions/prometheus.go
View File

@ -50,12 +50,12 @@ func Prometheus() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("prometheus", monitoring.ObservableOwnerDistribution),
shared.ContainerMemoryUsage("prometheus", monitoring.ObservableOwnerDistribution),
shared.ContainerCPUUsage("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
shared.ContainerMemoryUsage("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
},
{
shared.ContainerRestarts("prometheus", monitoring.ObservableOwnerDistribution),
shared.ContainerFsInodes("prometheus", monitoring.ObservableOwnerDistribution),
shared.ContainerRestarts("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
shared.ContainerFsInodes("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
},
},
},
@ -64,12 +64,12 @@ func Prometheus() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("prometheus", monitoring.ObservableOwnerDistribution),
shared.ProvisioningMemoryUsageLongTerm("prometheus", monitoring.ObservableOwnerDistribution),
shared.ProvisioningCPUUsageLongTerm("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
shared.ProvisioningMemoryUsageLongTerm("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("prometheus", monitoring.ObservableOwnerDistribution),
shared.ProvisioningMemoryUsageShortTerm("prometheus", monitoring.ObservableOwnerDistribution),
shared.ProvisioningCPUUsageShortTerm("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
shared.ProvisioningMemoryUsageShortTerm("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
},
},
},
@ -78,7 +78,7 @@ func Prometheus() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("prometheus", monitoring.ObservableOwnerDistribution),
shared.KubernetesPodsAvailable("prometheus", monitoring.ObservableOwnerDistribution).Observable(),
},
},
},

24
monitoring/definitions/query_runner.go
View File

@ -15,7 +15,7 @@ func QueryRunner() *monitoring.Container {
Title: "General",
Rows: []monitoring.Row{
{
shared.FrontendInternalAPIErrorResponses("query-runner", monitoring.ObservableOwnerSearch),
shared.FrontendInternalAPIErrorResponses("query-runner", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -24,12 +24,12 @@ func QueryRunner() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerMemoryUsage("query-runner", monitoring.ObservableOwnerSearch),
shared.ContainerCPUUsage("query-runner", monitoring.ObservableOwnerSearch),
shared.ContainerMemoryUsage("query-runner", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerCPUUsage("query-runner", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ContainerRestarts("query-runner", monitoring.ObservableOwnerSearch),
shared.ContainerFsInodes("query-runner", monitoring.ObservableOwnerSearch),
shared.ContainerRestarts("query-runner", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerFsInodes("query-runner", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -38,12 +38,12 @@ func QueryRunner() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("query-runner", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageLongTerm("query-runner", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageLongTerm("query-runner", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageLongTerm("query-runner", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("query-runner", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageShortTerm("query-runner", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageShortTerm("query-runner", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageShortTerm("query-runner", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -52,8 +52,8 @@ func QueryRunner() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("query-runner", monitoring.ObservableOwnerSearch),
shared.GoGcDuration("query-runner", monitoring.ObservableOwnerSearch),
shared.GoGoroutines("query-runner", monitoring.ObservableOwnerSearch).Observable(),
shared.GoGcDuration("query-runner", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -62,7 +62,7 @@ func QueryRunner() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("query-runner", monitoring.ObservableOwnerSearch),
shared.KubernetesPodsAvailable("query-runner", monitoring.ObservableOwnerSearch).Observable(),
},
},
},

25
monitoring/definitions/repo_updater.go
View File

@ -21,7 +21,7 @@ func RepoUpdater() *monitoring.Container {
Title: "General",
Rows: []monitoring.Row{
{
shared.FrontendInternalAPIErrorResponses("repo-updater", monitoring.ObservableOwnerCloud),
shared.FrontendInternalAPIErrorResponses("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -424,14 +424,15 @@ func RepoUpdater() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("repo-updater", monitoring.ObservableOwnerCloud),
shared.ContainerCPUUsage("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerMemoryUsage("repo-updater", monitoring.ObservableOwnerCloud).
WithWarning(nil).
WithCritical(monitoring.Alert().GreaterOrEqual(90)),
WithCritical(monitoring.Alert().GreaterOrEqual(90)).
Observable(),
},
{
shared.ContainerRestarts("repo-updater", monitoring.ObservableOwnerCloud),
shared.ContainerFsInodes("repo-updater", monitoring.ObservableOwnerCloud),
shared.ContainerRestarts("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerFsInodes("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -440,12 +441,12 @@ func RepoUpdater() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("repo-updater", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageLongTerm("repo-updater", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageLongTerm("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageLongTerm("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("repo-updater", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageShortTerm("repo-updater", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageShortTerm("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageShortTerm("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -454,8 +455,8 @@ func RepoUpdater() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("repo-updater", monitoring.ObservableOwnerCloud),
shared.GoGcDuration("repo-updater", monitoring.ObservableOwnerCloud),
shared.GoGoroutines("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
shared.GoGcDuration("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -464,7 +465,7 @@ func RepoUpdater() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("repo-updater", monitoring.ObservableOwnerCloud),
shared.KubernetesPodsAvailable("repo-updater", monitoring.ObservableOwnerCloud).Observable(),
},
},
},

24
monitoring/definitions/searcher.go
View File

@ -36,7 +36,7 @@ func Searcher() *monitoring.Container {
Owner: monitoring.ObservableOwnerSearch,
PossibleSolutions: "none",
},
shared.FrontendInternalAPIErrorResponses("searcher", monitoring.ObservableOwnerSearch),
shared.FrontendInternalAPIErrorResponses("searcher", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -45,12 +45,12 @@ func Searcher() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("searcher", monitoring.ObservableOwnerSearch),
shared.ContainerMemoryUsage("searcher", monitoring.ObservableOwnerSearch),
shared.ContainerCPUUsage("searcher", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerMemoryUsage("searcher", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ContainerRestarts("searcher", monitoring.ObservableOwnerSearch),
shared.ContainerFsInodes("searcher", monitoring.ObservableOwnerSearch),
shared.ContainerRestarts("searcher", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerFsInodes("searcher", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -59,12 +59,12 @@ func Searcher() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("searcher", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageLongTerm("searcher", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageLongTerm("searcher", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageLongTerm("searcher", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("searcher", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageShortTerm("searcher", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageShortTerm("searcher", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageShortTerm("searcher", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -73,8 +73,8 @@ func Searcher() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("searcher", monitoring.ObservableOwnerSearch),
shared.GoGcDuration("searcher", monitoring.ObservableOwnerSearch),
shared.GoGoroutines("searcher", monitoring.ObservableOwnerSearch).Observable(),
shared.GoGcDuration("searcher", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -83,7 +83,7 @@ func Searcher() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("searcher", monitoring.ObservableOwnerSearch),
shared.KubernetesPodsAvailable("searcher", monitoring.ObservableOwnerSearch).Observable(),
},
},
},

16
monitoring/definitions/shared/container.go
View File

@ -11,8 +11,8 @@ import (
// More granular resource usage warnings are provided by the provisioning observables.
var (
ContainerRestarts sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ContainerRestarts sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "container_restarts",
Description: "container restarts every 5m by instance",
Query: fmt.Sprintf(`increase(cadvisor_container_restart_count{%s}[5m])`, CadvisorNameMatcher(containerName)),
@ -31,8 +31,8 @@ var (
}
}
ContainerMemoryUsage sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ContainerMemoryUsage sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "container_memory_usage",
Description: "container memory usage by instance",
Query: fmt.Sprintf(`cadvisor_container_memory_usage_percentage_total{%s}`, CadvisorNameMatcher(containerName)),
@ -47,8 +47,8 @@ var (
}
}
ContainerCPUUsage sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ContainerCPUUsage sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "container_cpu_usage",
Description: "container cpu usage total (1m average) across all cores by instance",
Query: fmt.Sprintf(`cadvisor_container_cpu_usage_percentage_total{%s}`, CadvisorNameMatcher(containerName)),
@ -63,8 +63,8 @@ var (
}
}
ContainerFsInodes sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ContainerFsInodes sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "fs_inodes_used",
Description: "fs inodes in use by instance",
Query: fmt.Sprintf(`sum by (name)(container_fs_inodes_total{%s})`, CadvisorNameMatcher(containerName)),

4
monitoring/definitions/shared/frontend.go
View File

@ -8,8 +8,8 @@ import (
"github.com/sourcegraph/sourcegraph/monitoring/monitoring"
)
var FrontendInternalAPIErrorResponses sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
var FrontendInternalAPIErrorResponses sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "frontend_internal_api_error_responses",
Description: "frontend-internal API error responses every 5m by route",
Query: fmt.Sprintf(`sum by (category)(increase(src_frontend_internal_request_duration_seconds_count{job="%[1]s",code!~"2.."}[5m])) / ignoring(category) group_left sum(increase(src_frontend_internal_request_duration_seconds_count{job="%[1]s"}[5m]))`, containerName),

8
monitoring/definitions/shared/go.go
View File

@ -10,8 +10,8 @@ import (
// Golang monitoring overviews
var (
GoGoroutines sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
GoGoroutines sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "go_goroutines",
Description: "maximum active goroutines",
Query: fmt.Sprintf(`max by(instance) (go_goroutines{job=~".*%s"})`, containerName),
@ -23,8 +23,8 @@ var (
}
}
GoGcDuration sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
GoGcDuration sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "go_gc_duration_seconds",
Description: "maximum go garbage collection duration",
Query: fmt.Sprintf(`max by(instance) (go_gc_duration_seconds{job=~".*%s"})`, containerName),

4
monitoring/definitions/shared/kubernetes.go
View File

@ -10,8 +10,8 @@ import (
// Kubernetes monitoring overviews
var (
KubernetesPodsAvailable sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
KubernetesPodsAvailable sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "pods_available_percentage",
Description: "percentage pods available",
Query: fmt.Sprintf(`sum by(app) (up{app=~".*%[1]s"}) / count by (app) (up{app=~".*%[1]s"}) * 100`, containerName),

16
monitoring/definitions/shared/provisioning.go
View File

@ -11,8 +11,8 @@ import (
// Warn that instances might need more/less resources if usage is high on various time scales.
var (
ProvisioningCPUUsageLongTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ProvisioningCPUUsageLongTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "provisioning_container_cpu_usage_long_term",
Description: "container cpu usage total (90th percentile over 1d) across all cores by instance",
Query: fmt.Sprintf(`quantile_over_time(0.9, cadvisor_container_cpu_usage_percentage_total{%s}[1d])`, CadvisorNameMatcher(containerName)),
@ -27,8 +27,8 @@ var (
}
}
ProvisioningMemoryUsageLongTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ProvisioningMemoryUsageLongTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "provisioning_container_memory_usage_long_term",
Description: "container memory usage (1d maximum) by instance",
Query: fmt.Sprintf(`max_over_time(cadvisor_container_memory_usage_percentage_total{%s}[1d])`, CadvisorNameMatcher(containerName)),
@ -43,8 +43,8 @@ var (
}
}
ProvisioningCPUUsageShortTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ProvisioningCPUUsageShortTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "provisioning_container_cpu_usage_short_term",
Description: "container cpu usage total (5m maximum) across all cores by instance",
Query: fmt.Sprintf(`max_over_time(cadvisor_container_cpu_usage_percentage_total{%s}[5m])`, CadvisorNameMatcher(containerName)),
@ -59,8 +59,8 @@ var (
}
}
ProvisioningMemoryUsageShortTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable {
return monitoring.Observable{
ProvisioningMemoryUsageShortTerm sharedObservable = func(containerName string, owner monitoring.ObservableOwner) Observable {
return Observable{
Name: "provisioning_container_memory_usage_short_term",
Description: "container memory usage (5m maximum) by instance",
Query: fmt.Sprintf(`max_over_time(cadvisor_container_memory_usage_percentage_total{%s}[5m])`, CadvisorNameMatcher(containerName)),

31
monitoring/definitions/shared/shared.go
View File

@ -28,7 +28,36 @@ import (
"github.com/sourcegraph/sourcegraph/monitoring/monitoring"
)
type sharedObservable func(containerName string, owner monitoring.ObservableOwner) monitoring.Observable
// Observable is a variant of normal Observables that offer convenience functions for
// customizing shared observables.
type Observable monitoring.Observable
// Observable is a convenience adapter that casts this SharedObservable as an normal Observable.
func (o Observable) Observable() monitoring.Observable { return monitoring.Observable(o) }
// WithWarning overrides this Observable's warning-level alert with the given alert.
func (o Observable) WithWarning(a *monitoring.ObservableAlertDefinition) Observable {
o.Warning = a
return o
}
// WithCritical overrides this Observable's critical-level alert with the given alert.
func (o Observable) WithCritical(a *monitoring.ObservableAlertDefinition) Observable {
o.Critical = a
return o
}
// WithNoAlerts disables alerting on this Observable and sets the given interpretation instead.
func (o Observable) WithNoAlerts(interpretation string) Observable {
o.Warning = nil
o.Critical = nil
o.NoAlert = true
o.PossibleSolutions = ""
o.Interpretation = interpretation
return o
}
type sharedObservable func(containerName string, owner monitoring.ObservableOwner) Observable
// CadvisorNameMatcher generates Prometheus matchers that capture metrics that match the given container name
// while excluding some irrelevant series

24
monitoring/definitions/symbols.go
View File

@ -37,7 +37,7 @@ func Symbols() *monitoring.Container {
},
},
{
shared.FrontendInternalAPIErrorResponses("symbols", monitoring.ObservableOwnerCodeIntel),
shared.FrontendInternalAPIErrorResponses("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -46,12 +46,12 @@ func Symbols() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ContainerMemoryUsage("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ContainerCPUUsage("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerMemoryUsage("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ContainerRestarts("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ContainerFsInodes("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ContainerRestarts("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ContainerFsInodes("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -60,12 +60,12 @@ func Symbols() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageLongTerm("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageLongTerm("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageLongTerm("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningMemoryUsageShortTerm("symbols", monitoring.ObservableOwnerCodeIntel),
shared.ProvisioningCPUUsageShortTerm("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.ProvisioningMemoryUsageShortTerm("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -74,8 +74,8 @@ func Symbols() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.GoGoroutines("symbols", monitoring.ObservableOwnerCodeIntel),
shared.GoGcDuration("symbols", monitoring.ObservableOwnerCodeIntel),
shared.GoGoroutines("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
shared.GoGcDuration("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},
@ -84,7 +84,7 @@ func Symbols() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("symbols", monitoring.ObservableOwnerCodeIntel),
shared.KubernetesPodsAvailable("symbols", monitoring.ObservableOwnerCodeIntel).Observable(),
},
},
},

18
monitoring/definitions/syntect_server.go
View File

@ -65,12 +65,12 @@ func SyntectServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("syntect-server", monitoring.ObservableOwnerCloud),
shared.ContainerMemoryUsage("syntect-server", monitoring.ObservableOwnerCloud),
shared.ContainerCPUUsage("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerMemoryUsage("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ContainerRestarts("syntect-server", monitoring.ObservableOwnerCloud),
shared.ContainerFsInodes("syntect-server", monitoring.ObservableOwnerCloud),
shared.ContainerRestarts("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
shared.ContainerFsInodes("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -79,12 +79,12 @@ func SyntectServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("syntect-server", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageLongTerm("syntect-server", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageLongTerm("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageLongTerm("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("syntect-server", monitoring.ObservableOwnerCloud),
shared.ProvisioningMemoryUsageShortTerm("syntect-server", monitoring.ObservableOwnerCloud),
shared.ProvisioningCPUUsageShortTerm("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
shared.ProvisioningMemoryUsageShortTerm("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
},
},
},
@ -93,7 +93,7 @@ func SyntectServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.KubernetesPodsAvailable("syntect-server", monitoring.ObservableOwnerCloud),
shared.KubernetesPodsAvailable("syntect-server", monitoring.ObservableOwnerCloud).Observable(),
},
},
},

18
monitoring/definitions/zoekt_index_server.go
View File

@ -36,12 +36,12 @@ func ZoektIndexServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ContainerMemoryUsage("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ContainerCPUUsage("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerMemoryUsage("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ContainerRestarts("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ContainerFsInodes("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ContainerRestarts("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerFsInodes("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
},
{
{
@ -62,12 +62,12 @@ func ZoektIndexServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageLongTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageLongTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageLongTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageShortTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageShortTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageShortTerm("zoekt-indexserver", monitoring.ObservableOwnerSearch).Observable(),
},
},
},
@ -79,7 +79,7 @@ func ZoektIndexServer() *monitoring.Container {
// zoekt_index_server, zoekt_web_server are deployed together
// as part of the indexed-search service, so only show pod
// availability here.
shared.KubernetesPodsAvailable("indexed-search", monitoring.ObservableOwnerSearch),
shared.KubernetesPodsAvailable("indexed-search", monitoring.ObservableOwnerSearch).Observable(),
},
},
},

16
monitoring/definitions/zoekt_web_server.go
View File

@ -36,12 +36,12 @@ func ZoektWebServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ContainerCPUUsage("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ContainerMemoryUsage("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ContainerCPUUsage("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerMemoryUsage("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ContainerRestarts("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ContainerFsInodes("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ContainerRestarts("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ContainerFsInodes("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
},
{
{
@ -62,12 +62,12 @@ func ZoektWebServer() *monitoring.Container {
Hidden: true,
Rows: []monitoring.Row{
{
shared.ProvisioningCPUUsageLongTerm("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageLongTerm("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageLongTerm("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageLongTerm("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
},
{
shared.ProvisioningCPUUsageShortTerm("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningMemoryUsageShortTerm("zoekt-webserver", monitoring.ObservableOwnerSearch),
shared.ProvisioningCPUUsageShortTerm("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
shared.ProvisioningMemoryUsageShortTerm("zoekt-webserver", monitoring.ObservableOwnerSearch).Observable(),
},
},
},

57
monitoring/monitoring/README.md
View File

@ -19,9 +19,6 @@ To learn more about the generator\, see the top\-level program: https://github.c
- [type GenerateOptions](<#type-generateoptions>)
- [type Group](<#type-group>)
- [type Observable](<#type-observable>)
- [func (o Observable) WithCritical(a *ObservableAlertDefinition) Observable](<#func-observable-withcritical>)
- [func (o Observable) WithNoAlerts(interpretation string) Observable](<#func-observable-withnoalerts>)
- [func (o Observable) WithWarning(a *ObservableAlertDefinition) Observable](<#func-observable-withwarning>)
- [type ObservableAlertDefinition](<#type-observablealertdefinition>)
- [func Alert() *ObservableAlertDefinition](<#func-alert>)
- [func (a *ObservableAlertDefinition) For(d time.Duration) *ObservableAlertDefinition](<#func-observablealertdefinition-for>)
@ -244,27 +241,7 @@ type Observable struct {
}
```
### func \(Observable\) [WithCritical](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L649>)
```go
func (o Observable) WithCritical(a *ObservableAlertDefinition) Observable
```
### func \(Observable\) [WithNoAlerts](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L655>)
```go
func (o Observable) WithNoAlerts(interpretation string) Observable
```
WithNoAlerts disables alerting on this Observable and sets the given interpretation instead\.
### func \(Observable\) [WithWarning](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L644>)
```go
func (o Observable) WithWarning(a *ObservableAlertDefinition) Observable
```
## type [ObservableAlertDefinition](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L717-L729>)
## type [ObservableAlertDefinition](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L697-L701>)
ObservableAlertDefinition defines when an alert would be considered firing\.
@ -274,7 +251,7 @@ type ObservableAlertDefinition struct {
}
```
### func [Alert](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L712>)
### func [Alert](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L692>)
```go
func Alert() *ObservableAlertDefinition
@ -282,24 +259,30 @@ func Alert() *ObservableAlertDefinition
Alert provides a builder for defining alerting on an Observable\.
### func \(\*ObservableAlertDefinition\) [For](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L741>)
### func \(\*ObservableAlertDefinition\) [For](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L719>)
```go
func (a *ObservableAlertDefinition) For(d time.Duration) *ObservableAlertDefinition
```
### func \(\*ObservableAlertDefinition\) [GreaterOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L731>)
For indicates how long the given thresholds must be exceeded for this alert to be considered firing\. Defaults to 0s \(immediately alerts when threshold is exceeded\)\.
### func \(\*ObservableAlertDefinition\) [GreaterOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L705>)
```go
func (a *ObservableAlertDefinition) GreaterOrEqual(f float64) *ObservableAlertDefinition
```
### func \(\*ObservableAlertDefinition\) [LessOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L736>)
GreaterOrEqual\, when non\-zero\, indicates the alert should fire when greater or equal to this value\.
### func \(\*ObservableAlertDefinition\) [LessOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L712>)
```go
func (a *ObservableAlertDefinition) LessOrEqual(f float64) *ObservableAlertDefinition
```
LessOrEqual\, when non\-zero\, indicates the alert should fire when less than or equal to this value\.
## type [ObservableOwner](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L509>)
ObservableOwner denotes a team that owns an Observable\. The current teams are described in the handbook: https://about.sourcegraph.com/company/team/org_chart#engineering
@ -320,7 +303,7 @@ const (
)
```
## type [ObservablePanelOptions](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L796-L802>)
## type [ObservablePanelOptions](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L774-L780>)
ObservablePanelOptions declares options for visualizing an Observable\.
@ -330,7 +313,7 @@ type ObservablePanelOptions struct {
}
```
### func [PanelOptions](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L805>)
### func [PanelOptions](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L783>)
```go
func PanelOptions() ObservablePanelOptions
@ -338,7 +321,7 @@ func PanelOptions() ObservablePanelOptions
PanelOptions provides a builder for customizing an Observable visualization\.
### func \(ObservablePanelOptions\) [Interval](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L842>)
### func \(ObservablePanelOptions\) [Interval](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L820>)
```go
func (p ObservablePanelOptions) Interval(ms int) ObservablePanelOptions
@ -346,7 +329,7 @@ func (p ObservablePanelOptions) Interval(ms int) ObservablePanelOptions
Interval declares the panel's interval in milliseconds\.
### func \(ObservablePanelOptions\) [LegendFormat](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L830>)
### func \(ObservablePanelOptions\) [LegendFormat](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L808>)
```go
func (p ObservablePanelOptions) LegendFormat(format string) ObservablePanelOptions
@ -354,7 +337,7 @@ func (p ObservablePanelOptions) LegendFormat(format string) ObservablePanelOptio
LegendFormat sets the panel's legend format\, which may use Go template strings to select labels from the Prometheus query\.
### func \(ObservablePanelOptions\) [Max](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L823>)
### func \(ObservablePanelOptions\) [Max](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L801>)
```go
func (p ObservablePanelOptions) Max(max float64) ObservablePanelOptions
@ -362,7 +345,7 @@ func (p ObservablePanelOptions) Max(max float64) ObservablePanelOptions
Max sets the maximum value of the Y axis on the panel\. The default is auto\.
### func \(ObservablePanelOptions\) [Min](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L808>)
### func \(ObservablePanelOptions\) [Min](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L786>)
```go
func (p ObservablePanelOptions) Min(min float64) ObservablePanelOptions
@ -370,7 +353,7 @@ func (p ObservablePanelOptions) Min(min float64) ObservablePanelOptions
Min sets the minimum value of the Y axis on the panel\. The default is zero\.
### func \(ObservablePanelOptions\) [MinAuto](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L817>)
### func \(ObservablePanelOptions\) [MinAuto](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L795>)
```go
func (p ObservablePanelOptions) MinAuto() ObservablePanelOptions
@ -380,7 +363,7 @@ Min sets the minimum value of the Y axis on the panel to auto\, instead of the d
This is generally only useful if trying to show negative numbers\.
### func \(ObservablePanelOptions\) [Unit](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L836>)
### func \(ObservablePanelOptions\) [Unit](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L814>)
```go
func (p ObservablePanelOptions) Unit(t UnitType) ObservablePanelOptions
@ -398,7 +381,7 @@ These correspond to a row of Grafana graphs\.
type Row []Observable
```
## type [UnitType](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L751>)
## type [UnitType](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L729>)
UnitType for controlling the unit type display on graphs\.

38
monitoring/monitoring/monitoring.go
View File

@ -641,26 +641,6 @@ type Observable struct {
PanelOptions ObservablePanelOptions
}
func (o Observable) WithWarning(a *ObservableAlertDefinition) Observable {
o.Warning = a
return o
}
func (o Observable) WithCritical(a *ObservableAlertDefinition) Observable {
o.Critical = a
return o
}
// WithNoAlerts disables alerting on this Observable and sets the given interpretation instead.
func (o Observable) WithNoAlerts(interpretation string) Observable {
o.Warning = nil
o.Critical = nil
o.NoAlert = true
o.PossibleSolutions = ""
o.Interpretation = interpretation
return o
}
func (o Observable) validate() error {
if strings.Contains(o.Name, " ") || strings.ToLower(o.Name) != o.Name {
return fmt.Errorf("Name must be in lower_snake_case; found \"%s\"", o.Name)
@ -715,29 +695,27 @@ func Alert() *ObservableAlertDefinition {
// ObservableAlertDefinition defines when an alert would be considered firing.
type ObservableAlertDefinition struct {
// GreaterOrEqual, when non-zero, indicates the alert should fire when
// greater or equal to this value.
greaterOrEqual *float64
// LessOrEqual, when non-zero, indicates the alert should fire when less
// than or equal to this value.
lessOrEqual *float64
// For indicates how long the given thresholds must be exceeded for this
// alert to be considered firing. Defaults to 0s.
duration time.Duration
lessOrEqual *float64
duration time.Duration
}
// GreaterOrEqual, when non-zero, indicates the alert should fire when greater or equal
// to this value.
func (a *ObservableAlertDefinition) GreaterOrEqual(f float64) *ObservableAlertDefinition {
a.greaterOrEqual = &f
return a
}
// LessOrEqual, when non-zero, indicates the alert should fire when less than or equal to
// this value.
func (a *ObservableAlertDefinition) LessOrEqual(f float64) *ObservableAlertDefinition {
a.lessOrEqual = &f
return a
}
// For indicates how long the given thresholds must be exceeded for this alert to be
// considered firing. Defaults to 0s (immediately alerts when threshold is exceeded).
func (a *ObservableAlertDefinition) For(d time.Duration) *ObservableAlertDefinition {
a.duration = d
return a