artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 17:51:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

monitoring: explicitly point out team that owns alerts/panels (#17471)

Browse Source
This commit is contained in:
Robert Lin 2021-01-20 23:24:10 +08:00 committed by GitHub
parent b5d979d6c8
commit 40b8f30aea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 2473 additions and 1362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2107
doc/admin/observability/alert_solutions.md
View File

File diff suppressed because it is too large Load Diff

1665
doc/admin/observability/dashboards.md
View File

File diff suppressed because it is too large Load Diff

16
monitoring/monitoring/README.md generated
View File

@ -125,7 +125,7 @@ type Group struct {
}
```
## type [Observable](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L422-L534>)
## type [Observable](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L437-L549>)
Observable describes a metric about a container that can be observed\. For example\, memory usage\.
@ -247,7 +247,7 @@ type Observable struct {
}
```
## type [ObservableAlertDefinition](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L598-L604>)
## type [ObservableAlertDefinition](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L613-L619>)
ObservableAlertDefinition defines when an alert would be considered firing\.
@ -257,7 +257,7 @@ type ObservableAlertDefinition struct {
}
```
### func [Alert](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L593>)
### func [Alert](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L608>)
```go
func Alert() *ObservableAlertDefinition
@ -265,7 +265,7 @@ func Alert() *ObservableAlertDefinition
Alert provides a builder for defining alerting on an Observable\.
### func \(\*ObservableAlertDefinition\) [For](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L636>)
### func \(\*ObservableAlertDefinition\) [For](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L651>)
```go
func (a *ObservableAlertDefinition) For(d time.Duration) *ObservableAlertDefinition
@ -273,7 +273,7 @@ func (a *ObservableAlertDefinition) For(d time.Duration) *ObservableAlertDefinit
For indicates how long the given thresholds must be exceeded for this alert to be considered firing\. Defaults to 0s \(immediately alerts when threshold is exceeded\)\.
### func \(\*ObservableAlertDefinition\) [Greater](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L621>)
### func \(\*ObservableAlertDefinition\) [Greater](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L636>)
```go
func (a *ObservableAlertDefinition) Greater(f float64) *ObservableAlertDefinition
@ -281,7 +281,7 @@ func (a *ObservableAlertDefinition) Greater(f float64) *ObservableAlertDefinitio
Greater indicates the alert should fire when strictly greater to this value\.
### func \(\*ObservableAlertDefinition\) [GreaterOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L607>)
### func \(\*ObservableAlertDefinition\) [GreaterOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L622>)
```go
func (a *ObservableAlertDefinition) GreaterOrEqual(f float64) *ObservableAlertDefinition
@ -289,7 +289,7 @@ func (a *ObservableAlertDefinition) GreaterOrEqual(f float64) *ObservableAlertDe
GreaterOrEqual indicates the alert should fire when greater or equal the given value\.
### func \(\*ObservableAlertDefinition\) [Less](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L628>)
### func \(\*ObservableAlertDefinition\) [Less](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L643>)
```go
func (a *ObservableAlertDefinition) Less(f float64) *ObservableAlertDefinition
@ -297,7 +297,7 @@ func (a *ObservableAlertDefinition) Less(f float64) *ObservableAlertDefinition
Less indicates the alert should fire when strictly less than this value\.
### func \(\*ObservableAlertDefinition\) [LessOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L614>)
### func \(\*ObservableAlertDefinition\) [LessOrEqual](<https://github.com/sourcegraph/sourcegraph/blob/main/monitoring/monitoring/monitoring.go#L629>)
```go
func (a *ObservableAlertDefinition) LessOrEqual(f float64) *ObservableAlertDefinition

32
monitoring/monitoring/documentation.go
View File

@ -36,6 +36,7 @@ To learn more about Sourcegraph's metrics and how to view these dashboards, see
`
// fprintSubtitle prints subtitle-class text
func fprintSubtitle(w io.Writer, text string) {
fmt.Fprintf(w, "<p class=\"subtitle\">%s</p>\n\n", text)
}
@ -48,6 +49,11 @@ func fprintObservableHeader(w io.Writer, c *Container, o *Observable, headerLeve
fmt.Fprintf(w, " %s: %s\n\n", c.Name, o.Name)
}
// fprintOwnedBy prints information about who owns a particular monitoring definition.
func fprintOwnedBy(w io.Writer, owner ObservableOwner) {
fmt.Fprintf(w, "<sub>*Managed by the %s.*</sub>\n", owner.toMarkdown())
}
// Create an anchor link that matches `fprintObservableHeader`
//
// Must match Prometheus template in `docker-images/prometheus/cmd/prom-wrapper/receivers.go`
@ -101,11 +107,11 @@ func (d *documentation) renderAlertSolutionEntry(c *Container, o Observable) err
}
fprintObservableHeader(&d.alertSolutions, c, &o, 2)
fprintSubtitle(&d.alertSolutions, fmt.Sprintf(`%s (%s)`, o.Description, o.Owner))
fprintSubtitle(&d.alertSolutions, o.Description)
var prometheusAlertNames []string // collect names for silencing configuration
// Render descriptions of various levels of this alert
fmt.Fprintf(&d.alertSolutions, "**Descriptions:**\n\n")
fmt.Fprintf(&d.alertSolutions, "**Descriptions**\n\n")
for _, alert := range []struct {
level string
threshold *ObservableAlertDefinition
@ -127,29 +133,31 @@ func (d *documentation) renderAlertSolutionEntry(c *Container, o Observable) err
fmt.Fprint(&d.alertSolutions, "\n")
// Render solutions for dealing with this alert
fmt.Fprintf(&d.alertSolutions, "**Possible solutions:**\n\n")
fmt.Fprintf(&d.alertSolutions, "**Possible solutions**\n\n")
if o.PossibleSolutions != "none" {
possibleSolutions, _ := toMarkdown(o.PossibleSolutions, true)
fmt.Fprintf(&d.alertSolutions, "%s\n", possibleSolutions)
}
// add link to panel information IF there are additional details available
if o.Interpretation != "" && o.Interpretation != "none" {
fmt.Fprintf(&d.alertSolutions, "- **Refer to the [dashboards reference](./%s#%s)** for more help interpreting this alert and metric.\n",
dashboardsDocsFile, observableDocAnchor(c, o))
}
// add silencing configuration as another solution
fmt.Fprintf(&d.alertSolutions, "- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:\n\n")
fmt.Fprintf(&d.alertSolutions, "```json\n%s\n```\n\n", fmt.Sprintf(`"observability.silenceAlerts": [
%s
]`, strings.Join(prometheusAlertNames, ",\n")))
// add link to panel information IF there are additional details available
if o.Interpretation != "" && o.Interpretation != "none" {
fmt.Fprintf(&d.alertSolutions, "> NOTE: More help interpreting this metric is available in the [dashboards reference](./%s#%s).\n\n",
dashboardsDocsFile, observableDocAnchor(c, o))
}
// add owner
fprintOwnedBy(&d.alertSolutions, o.Owner)
// render break for readability
fmt.Fprint(&d.alertSolutions, "<br />\n\n")
fmt.Fprint(&d.alertSolutions, "\n<br />\n\n")
return nil
}
func (d *documentation) renderDashboardPanelEntry(c *Container, o Observable) error {
fprintObservableHeader(&d.dashboards, c, &o, 4)
fmt.Fprintf(&d.dashboards, "This %s panel indicates %s.\n\n", o.Owner, o.Description)
fmt.Fprintf(&d.dashboards, "This panel indicates %s.\n\n", o.Description)
// render interpretation reference if available
if o.Interpretation != "" && o.Interpretation != "none" {
interpretation, _ := toMarkdown(o.Interpretation, false)
@ -157,9 +165,11 @@ func (d *documentation) renderDashboardPanelEntry(c *Container, o Observable) er
}
// add link to alert solutions IF there is an alert attached
if !o.NoAlert {
fmt.Fprintf(&d.dashboards, "Refer to the [alert solutions reference](./%s#%s) for relevant alerts.\n",
fmt.Fprintf(&d.dashboards, "> NOTE: Alerts related to this panel are documented in the [alert solutions reference](./%s#%s).\n\n",
alertSolutionsFile, observableDocAnchor(c, o))
}
// add owner
fprintOwnedBy(&d.dashboards, o.Owner)
// render break for readability
fmt.Fprint(&d.dashboards, "\n<br />\n\n")
return nil

15
monitoring/monitoring/monitoring.go
View File

@ -416,6 +416,21 @@ const (
ObservableOwnerCloud ObservableOwner = "cloud"
)
// toMarkdown returns a Markdown string that also links to the owner's team page
func (o ObservableOwner) toMarkdown() string {
var teamName string
// special cases for differences in how a team is named in ObservableOwner and how
// they are named in the handbook.
// see https://about.sourcegraph.com/company/team/org_chart#engineering
switch o {
case ObservableOwnerCodeIntel:
teamName = "code-intelligence"
default:
teamName = string(o)
}
return fmt.Sprintf("[Sourcegraph %s team](https://about.sourcegraph.com/handbook/engineering/%s)", upperFirst(teamName), teamName)
}
// Observable describes a metric about a container that can be observed. For example, memory usage.
//
// These correspond to Grafana graphs.