monitoring: encourage silencing, render entry for alerts w/o solutions (#12731)

monitoring/generator.go

@@ -791,6 +791,8 @@ This document contains possible solutions for when you find alerts are firing in
 If your alert isn't mentioned here, or if the solution doesn't help, [contact us](mailto:support@sourcegraph.com)
 for assistance.
 
+To learn more about Sourcegraph's alerting, see [our alerting documentation](https://docs.sourcegraph.com/admin/observability/alerting).
+
 <!-- DO NOT EDIT: generated via: go generate ./monitoring -->
 
 `)
@@ -798,13 +800,10 @@ for assistance.
 		for _, g := range c.Groups {
 			for _, r := range g.Rows {
 				for _, o := range r {
-					if o.PossibleSolutions == "none" {
-						continue
-					}
-
-					fmt.Fprintf(&b, "# %s: %s\n\n", c.Name, o.Name)
+					fmt.Fprintf(&b, "## %s: %s\n\n", c.Name, o.Name)
 
 					fmt.Fprintf(&b, "**Descriptions:**\n")
+					var prometheusAlertNames []string
 					for _, alert := range []struct {
 						level     string
 						threshold Alert
@@ -815,14 +814,22 @@ for assistance.
 						if alert.threshold.isEmpty() {
 							continue
 						}
-						fmt.Fprintf(&b, "\n- _%s_ (`%s`)\n\n",
-							c.alertDescription(o, alert.threshold),
-							prometheusAlertName(alert.level, c.Name, o.Name))
+						fmt.Fprintf(&b, "\n- _%s_\n", c.alertDescription(o, alert.threshold))
+						prometheusAlertNames = append(prometheusAlertNames,
+							fmt.Sprintf("  \"%s\"", prometheusAlertName(alert.level, c.Name, o.Name)))
 					}
+					fmt.Fprint(&b, "\n")
 
 					fmt.Fprintf(&b, "**Possible solutions:**\n\n")
-					possibleSolutions, _ := goMarkdown(o.PossibleSolutions)
-					fmt.Fprintf(&b, "%s\n\n", possibleSolutions)
+					if o.PossibleSolutions != "none" {
+						possibleSolutions, _ := goMarkdown(o.PossibleSolutions)
+						fmt.Fprintf(&b, "%s\n", possibleSolutions)
+					}
+					// add silencing configuration as another solution
+					fmt.Fprintf(&b, "- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:\n\n")
+					fmt.Fprintf(&b, "```json\n%s\n```\n\n", fmt.Sprintf(`"observability.silenceAlerts": [
+%s
+]`, strings.Join(prometheusAlertNames, ",\n")))
 				}
 			}
 		}
@@ -857,6 +864,12 @@ func goMarkdown(m string) (string, error) {
 		}
 		m = strings.Join(lines[:len(lines)-1], "\n")
 	}
+
+	// If result is not a list, make it a list, so we can add items.
+	if !strings.HasPrefix(m, "-") && !strings.HasPrefix(m, "*") {
+		m = fmt.Sprintf("- %s", m)
+	}
+
 	return m, nil
 }
 

monitoring/shared.go

@@ -127,8 +127,7 @@ var sharedContainerFsInodes sharedObservable = func(containerName string) Observ
 		Owner:           ObservableOwnerDistribution,
 		PossibleSolutions: `
 			- Refer to your OS or cloud provider's documentation for how to increase inodes.
-			- **Kubernetes:** consider provisioning more machines with less resources.
-`,
+			- **Kubernetes:** consider provisioning more machines with less resources.`,
 	}
 }