msp/monitoring: document when the maxcount alert is created, fix alert (#59994)

dev/managedservicesplatform/managedservicesplatform.go

@@ -134,13 +134,8 @@ func (r *Renderer) RenderEnvironment(
 		EnvironmentID:       env.ID,
 		Alerting:            pointers.DerefZero(env.Alerting),
 
-		Monitoring: monitoringSpec,
-		MaxInstanceCount: func() *int {
-			if env.Instances.Scaling != nil {
-				return env.Instances.Scaling.MaxCount
-			}
-			return nil
-		}(),
+		Monitoring:            monitoringSpec,
+		MaxInstanceCount:      env.Instances.Scaling.GetMaxCount(), // returns nil if not relevant
 		ExternalDomain:        pointers.DerefZero(env.EnvironmentServiceSpec).Domain,
 		ServiceAuthentication: pointers.DerefZero(env.EnvironmentServiceSpec).Authentication,
 		DiagnosticsSecret:     cloudrunOutput.DiagnosticsSecret,

dev/managedservicesplatform/spec/environment.go

@@ -400,12 +400,23 @@ type EnvironmentInstancesScalingSpec struct {
 	// times. Set this to >0 to avoid service warm-up delays.
 	MinCount int `yaml:"minCount"`
 	// MaxCount is the maximum number of instances that Cloud Run is allowed to
-	// scale up to.
+	// scale up to. When this value is >= the default of 5, then we also provision
+	// an alert that fires when Cloud Run scaling approaches the max instance
+	// count.
 	//
 	// If not provided, the default is 5.
 	MaxCount *int `yaml:"maxCount,omitempty"`
 }
 
+// GetMaxCount returns nil if no scaling options are relevant, or the default,
+// or the max value.
+func (e *EnvironmentInstancesScalingSpec) GetMaxCount() *int {
+	if e == nil {
+		return nil
+	}
+	return pointers.Ptr(pointers.Deref(e.MaxCount, 5)) // builder.DefaultMaxInstances
+}
+
 type EnvironmentServiceAuthenticationSpec struct {
 	// Sourcegraph enables access to everyone in the sourcegraph.com GSuite
 	// domain.

dev/managedservicesplatform/stacks/monitoring/service.go

@@ -19,8 +19,9 @@ func createServiceAlerts(
 	vars Variables,
 	channels []monitoringnotificationchannel.MonitoringNotificationChannel,
 ) error {
-	// Only provision if MaxCount is specified above 5
-	if pointers.Deref(vars.MaxInstanceCount, 0) > 5 {
+	// Only provision if MaxCount is specified greater or equal 5 (the default).
+	// If nil, it doesn't matter
+	if vars.MaxInstanceCount != nil && *vars.MaxInstanceCount >= 5 {
 		if _, err := alertpolicy.New(stack, id, &alertpolicy.Config{
 			Service:       vars.Service,
 			EnvironmentID: vars.EnvironmentID,
@@ -36,6 +37,9 @@ func createServiceAlerts(
 				Aligner: alertpolicy.MonitoringAlignMax,
 				Reducer: alertpolicy.MonitoringReduceMax,
 				Period:  "60s",
+				// Fire when we are 1 instance away from hitting the limit.
+				Threshold:  float64(*vars.MaxInstanceCount - 1),
+				Comparison: alertpolicy.ComparisonGT,
 			},
 			NotificationChannels: channels,
 		}); err != nil {