msp: Provision Sentry project for MSP Services (#59458)

Provision a Sentry project and Alert Issue rules to send error notifications to Slack

---------

Co-authored-by: Robert Lin <robert@bobheadxi.dev>

deps.bzl

@@ -5687,6 +5687,13 @@ def go_dependencies():
         sum = "h1:N0OxHqeujHxvVU666KQY9whauLyw4s3BJGBLxx6gKR0=",
         version = "v0.0.0-20230822024612-edb48c530722",
     )
+    go_repository(
+        name = "com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry",
+        sum = "h1:wHEDTUur95a2HtUUodA8OXQi0Uefwa60CvhN4fLIuhU=",
+        version = "v0.0.0-20240109225336-01188650a68d",
+    )
     go_repository(
         name = "com_github_sourcegraph_managed_services_platform_cdktf_gen_slack",
         build_file_proto_mode = "disable_global",

dev/managedservicesplatform/googlesecretsmanager/googlesecretsmanager.go

@@ -34,6 +34,11 @@ const (
 	//
 	// The current bot user is https://api.slack.com/apps/A06C4TF6YF7/oauth
 	SecretSlackOperatorOAuthToken = "SLACK_OPERATOR_BOT_OAUTH_TOKEN"
+	// SecretSentryAuthToken is a Sentry internal integration auth token with
+	// Project permissions
+	//
+	// The integration is configured in https://sourcegraph.sentry.io/settings/developer-settings/managed-services-platform-fbf7cc/
+	SecretSentryAuthToken = "TFC_MSP_SENTRY_INTEGRATION"
 
 	// SecretSourcegraphWildcardKey and SecretSourcegraphWildcardCert are used
 	// for configuring Cloudflare TLS.

dev/managedservicesplatform/internal/resource/sentryalert/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//dev:go_defs.bzl", "go_test")
+
+go_library(
+    name = "sentryalert",
+    srcs = ["sentryalert.go"],
+    importpath = "github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/sentryalert",
+    visibility = ["//dev/managedservicesplatform:__subpackages__"],
+    deps = [
+        "//dev/managedservicesplatform/internal/resourceid",
+        "//lib/errors",
+        "//lib/pointers",
+        "@com_github_aws_constructs_go_constructs_v10//:constructs",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//issuealert",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//project",
+    ],
+)
+
+go_test(
+    name = "sentryalert_test",
+    srcs = ["sentryalert_test.go"],
+    embed = [":sentryalert"],
+    deps = ["@com_github_hexops_autogold_v2//:autogold"],
+)

dev/managedservicesplatform/internal/resource/sentryalert/sentryalert.go

@@ -0,0 +1,241 @@
+package sentryalert
+
+import (
+	"encoding/json"
+
+	"github.com/aws/constructs-go/constructs/v10"
+
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+
+	sentryproject "github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/project"
+
+	"github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/issuealert"
+
+	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resourceid"
+	"github.com/sourcegraph/sourcegraph/lib/pointers"
+)
+
+type Output struct{}
+
+// Config for a Sentry issue alert.
+//
+// [Sentry] and [Terraform] document each field
+//
+// [Sentry]: https://docs.sentry.io/api/alerts/create-an-issue-alert-rule-for-a-project/
+// [Terraform]: https://registry.terraform.io/providers/jianyuan/sentry/latest/docs/resources/issue_alert
+type Config struct {
+	// ID of the issue alert. Must be unique
+	ID string
+	// SentryProject is the project to set the alert on
+	SentryProject sentryproject.Project
+	// AlertConfig is the configuration for the Sentry issue alert rule
+	AlertConfig AlertConfig
+}
+
+// AlertConfig is the configuration for the Sentry issue alert rule
+type AlertConfig struct {
+	// Name the name of the alert
+	Name string
+	// Frequency determines how often to perform the actions for an issue, in minutes. (valid range 5-43200)
+	Frequency float64
+	// ActionMatch trigger actions when an event is captured by Sentry and `ActionMatch` of the
+	// specified `Conditions` happen
+	ActionMatch ActionMatch
+	// Conditions which must be satisfied for actions to trigger.
+	// `ActionMatch` determines whether all or any must be true.
+	Conditions []Condition
+	// Actions the list of actions to run when the conditions match
+	Actions []Action
+
+	// FilterMatch (optional) determines which filters need to be true before actions are executed.
+	// Required when a value is provided for `Filters`
+	FilterMatch FilterMatch
+	// Filters determines if a rule fires after the necessary conditions have been met
+	Filters []Filter
+}
+
+func (a AlertConfig) Validate() error {
+	if a.Name == "" {
+		return errors.New("Name is required")
+	}
+
+	if a.ActionMatch == "" {
+		return errors.New("ActionMatch is required")
+	}
+
+	// TODO(jac): allow Conditions to be nil after provider issue is fixed
+	// https://github.com/jianyuan/terraform-provider-sentry/issues/366
+	if len(a.Conditions) == 0 {
+		return errors.New("Conditions is required with at least one condition specified")
+	}
+
+	if len(a.Actions) == 0 {
+		return errors.New("Actions is required with at least one action specified")
+	}
+
+	if a.Frequency == 0 {
+		return errors.New("Frequency is required and must be a value between 5-43200")
+	}
+
+	if a.Frequency < 5 || a.Frequency > 43200 {
+		return errors.New("Frequency must be between 5-43200")
+	}
+
+	if a.Filters != nil && a.FilterMatch == "" {
+		return errors.New("FilterMatch is required when Filters are provided")
+	}
+
+	return nil
+}
+
+type ActionMatch string
+
+const (
+	ActionMatchAll ActionMatch = "all"
+	ActionMatchAny ActionMatch = "any"
+)
+
+type ConditionID string
+
+const (
+	// FirstSeenEventCondition A new issue is created
+	FirstSeenEventCondition ConditionID = "sentry.rules.conditions.first_seen_event.FirstSeenEventCondition"
+	// RegressionEventCondition The issue changes state from resolved to unresolved
+	RegressionEventCondition ConditionID = "sentry.rules.conditions.regression_event.RegressionEventCondition"
+	// EventFrequencyCondition The issue is seen more than `value` times in `interval` (valid values are 1m, 5m, 15m, 1h, 1d, 1w and 30d)
+	EventFrequencyCondition ConditionID = "sentry.rules.conditions.event_frequency.EventFrequencyCondition"
+	// EventUniqueUserFrequencyCondition The issue is seen by more than `value` users in `interval` (valid values are 1m, 5m, 15m, 1h, 1d, 1w and 30d)
+	EventUniqueUserFrequencyCondition ConditionID = "sentry.rules.conditions.event_frequency.EventUniqueUserFrequencyCondition"
+	// EventFrequencyPercentCondition The issue affects more than `value` percent (integer 0 to 100) of sessions in `interval` (valid values are 5m, 10m, 30m, and 1h)
+	EventFrequencyPercentCondition ConditionID = "sentry.rules.conditions.event_frequency.EventFrequencyPercentCondition"
+)
+
+// Condition checked `WHEN` an event is captured by Sentry.
+//
+// Multiple Conditions can be composed together in an alert requiring `all` or `any` to pass
+type Condition struct {
+	// ID represents the type of condition
+	ID ConditionID `json:"id"`
+	// Value is an integer threshold used by certain conditions
+	Value *int `json:"value,omitempty"`
+	// Interval is a threshold used by certain conditions.
+	Interval *string `json:"interval,omitempty"`
+}
+
+type ActionID string
+
+const (
+	// Send a Slack notification
+	SlackNotifyServiceAction ActionID = "sentry.integrations.slack.notify_action.SlackNotifyServiceAction"
+	// Send an Opsgenie notification
+	OpsgenieNotifyTeamAction ActionID = "sentry.integrations.opsgenie.notify_action.OpsgenieNotifyTeamAction"
+)
+
+type Action struct {
+	// ID represents the type of action
+	ID ActionID
+	// ActionParameters define parameters unique to specific actions documented here [body parameters > actions]
+	//
+	// [body parameters > actions]: https://docs.sentry.io/api/alerts/create-an-issue-alert-rule-for-a-project/
+	ActionParameters map[string]any
+}
+
+// Custom marshalling to flatten Action struct
+func (a Action) MarshalJSON() ([]byte, error) {
+	// Create a new map to hold the flattened JSON representation
+	flattened := make(map[string]interface{})
+
+	// Copy the fields from the Action struct to the flattened map
+	flattened["id"] = a.ID
+	for key, value := range a.ActionParameters {
+		flattened[key] = value
+	}
+
+	// Marshal the flattened map to JSON
+	return json.Marshal(flattened)
+}
+
+type FilterMatch string
+
+const (
+	FilterMatchAll  FilterMatch = "all"
+	FilterMatchAny  FilterMatch = "any"
+	FilterMatchNone FilterMatch = "none"
+)
+
+type FilterID string
+
+const (
+	// AgeComparisonFilter the issue `comparison_type` (older, newer) than `value` of `time`
+	AgeComparisonFilter FilterID = "sentry.rules.filters.age_comparison.AgeComparisonFilter"
+)
+
+type Filter struct {
+	// ID represents the type of filter
+	ID FilterID
+
+	// FilterParameters define parameters unique to specific filters documented here [body parameters > filters]
+	//
+	// [body parameters > filters]: https://docs.sentry.io/api/alerts/create-an-issue-alert-rule-for-a-project/
+	FilterParameters map[string]any
+}
+
+// Custom marshalling to flatten Filter struct
+func (f Filter) MarshalJSON() ([]byte, error) {
+	// Create a new map to hold the flattened JSON representation
+	flattened := make(map[string]interface{})
+
+	// Copy the fields from the Filter struct to the flattened map
+	flattened["id"] = f.ID
+	for key, value := range f.FilterParameters {
+		flattened[key] = value
+	}
+
+	// Marshal the flattened map to JSON
+	return json.Marshal(flattened)
+}
+
+func New(scope constructs.Construct, id resourceid.ID, config Config) (*Output, error) {
+	if err := config.AlertConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "sentry alert validation failed")
+	}
+
+	conditions, err := json.Marshal(config.AlertConfig.Conditions)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marshal AlertConfig.Conditions")
+	}
+
+	actions, err := json.Marshal(config.AlertConfig.Actions)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marshal AlertConfig.Actions")
+	}
+
+	// Filters is optional
+	filters := []byte{}
+	if len(config.AlertConfig.Filters) > 0 {
+		filters, err = json.Marshal(config.AlertConfig.Filters)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to marshal AlertConfig.Filters")
+		}
+	}
+
+	_ = issuealert.NewIssueAlert(scope, id.TerraformID("alert"), &issuealert.IssueAlertConfig{
+		Organization: config.SentryProject.Organization(),
+		Project:      config.SentryProject.Slug(),
+		Name:         pointers.Ptr(config.AlertConfig.Name),
+		ActionMatch:  pointers.Ptr(string(config.AlertConfig.ActionMatch)),
+		Conditions:   pointers.Ptr(string(conditions)),
+		Actions:      pointers.Ptr(string(actions)),
+		Frequency:    pointers.Ptr(config.AlertConfig.Frequency),
+		FilterMatch: func() *string {
+			// Sentry will default to All when none is set which causes issues with the Terraform provider
+			if config.AlertConfig.FilterMatch == "" {
+				return pointers.Ptr(string(FilterMatchAll))
+			}
+			return pointers.Ptr(string(config.AlertConfig.FilterMatch))
+		}(),
+		Filters: pointers.NonZeroPtr(string(filters)),
+	})
+
+	return &Output{}, nil
+}

dev/managedservicesplatform/internal/resource/sentryalert/sentryalert_test.go

@@ -0,0 +1,61 @@
+package sentryalert
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/hexops/autogold/v2"
+)
+
+func TestActionMarshal(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		config Action
+		want   autogold.Value
+	}{
+		{
+			name: "Slack Action",
+			config: Action{
+				ID: SlackNotifyServiceAction,
+				ActionParameters: map[string]any{
+					"workspace":  12345,
+					"channel":    "test-channel",
+					"channel_id": 67890,
+					"tags":       "test-service",
+				},
+			},
+			want: autogold.Expect(`{"channel":"test-channel","channel_id":67890,"id":"sentry.integrations.slack.notify_action.SlackNotifyServiceAction","tags":"test-service","workspace":12345}`),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got, _ := json.Marshal(tc.config)
+			tc.want.Equal(t, string(got))
+		})
+	}
+}
+
+func TestFilterMarshal(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		config Filter
+		want   autogold.Value
+	}{
+		{
+			name: "Age Filter",
+			config: Filter{
+				ID: AgeComparisonFilter,
+				FilterParameters: map[string]any{
+					"comparison_type": "older",
+					"value":           3,
+					"time":            "week",
+				},
+			},
+			want: autogold.Expect(`{"comparison_type":"older","id":"sentry.rules.filters.age_comparison.AgeComparisonFilter","time":"week","value":3}`),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got, _ := json.Marshal(tc.config)
+			tc.want.Equal(t, string(got))
+		})
+	}
+}

dev/managedservicesplatform/internal/stack/options/sentryprovider/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "sentryprovider",
+    srcs = ["sentryprovider.go"],
+    importpath = "github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/sentryprovider",
+    visibility = ["//dev/managedservicesplatform:__subpackages__"],
+    deps = [
+        "//dev/managedservicesplatform/internal/resource/gsmsecret",
+        "//dev/managedservicesplatform/internal/resourceid",
+        "//dev/managedservicesplatform/internal/stack",
+        "//lib/pointers",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//provider",
+    ],
+)

dev/managedservicesplatform/internal/stack/options/sentryprovider/sentryprovider.go

@@ -0,0 +1,21 @@
+package sentryprovider
+
+import (
+	sentry "github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/provider"
+
+	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/gsmsecret"
+	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resourceid"
+	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack"
+	"github.com/sourcegraph/sourcegraph/lib/pointers"
+)
+
+// With configures a stack to be able to use Sentry resources.
+func With(sentryToken gsmsecret.DataConfig) stack.NewStackOption {
+	return func(s stack.Stack) error {
+		_ = sentry.NewSentryProvider(s.Stack, pointers.Ptr("sentry"),
+			&sentry.SentryProviderConfig{
+				Token: &gsmsecret.Get(s.Stack, resourceid.New("sentry-provider-token"), sentryToken).Value,
+			})
+		return nil
+	}
+}

dev/managedservicesplatform/managedservicesplatform.go

@@ -144,6 +144,7 @@ func (r *Renderer) RenderEnvironment(
 		DiagnosticsSecret:     cloudrunOutput.DiagnosticsSecret,
 		RedisInstanceID:       cloudrunOutput.RedisInstanceID,
 		ServiceHealthProbes:   pointers.DerefZero(env.EnvironmentServiceSpec).HealthProbes,
+		SentryProject:         cloudrunOutput.SentryProject,
 	}); err != nil {
 		return nil, errors.Wrap(err, "failed to create monitoring stack")
 	}

dev/managedservicesplatform/stacks/cloudrun/BUILD.bazel

@@ -24,6 +24,7 @@ go_library(
         "//dev/managedservicesplatform/internal/stack/options/dynamicvariables",
         "//dev/managedservicesplatform/internal/stack/options/googleprovider",
         "//dev/managedservicesplatform/internal/stack/options/randomprovider",
+        "//dev/managedservicesplatform/internal/stack/options/sentryprovider",
         "//dev/managedservicesplatform/spec",
         "//dev/managedservicesplatform/stacks/cloudrun/internal/builder",
         "//dev/managedservicesplatform/stacks/cloudrun/internal/builder/job",
@@ -32,6 +33,10 @@ go_library(
         "//lib/errors",
         "//lib/pointers",
         "@com_github_hashicorp_terraform_cdk_go_cdktf//:cdktf",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//datasentryorganization",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//datasentryteam",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//key",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//project",
         "@org_golang_x_exp//maps",
         "@org_golang_x_exp//slices",
     ],

dev/managedservicesplatform/stacks/cloudrun/cloudrun.go

@@ -12,6 +12,11 @@ import (
 
 	"github.com/hashicorp/terraform-cdk-go/cdktf"
 
+	"github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/datasentryorganization"
+	"github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/datasentryteam"
+	"github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/key"
+	sentryproject "github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/project"
+
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/googlesecretsmanager"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/bigquery"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/cloudsql"
@@ -27,6 +32,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/dynamicvariables"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/googleprovider"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/randomprovider"
+	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/sentryprovider"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/spec"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/stacks/cloudrun/internal/builder"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/stacks/cloudrun/internal/builder/job"
@@ -39,6 +45,7 @@ import (
 type CrossStackOutput struct {
 	DiagnosticsSecret *random.Output
 	RedisInstanceID   *string
+	SentryProject     sentryproject.Project
 }
 
 type Variables struct {
@@ -69,6 +76,8 @@ var (
 
 const tfVarKeyResolvedImageTag = "resolved_image_tag"
 
+const SentryOrganization = "sourcegraph"
+
 // NewStack instantiates the MSP cloudrun stack, which is currently a pretty
 // monolithic stack that encompasses all the core components of an MSP service,
 // including networking and dependencies like Redis.
@@ -83,6 +92,10 @@ func NewStack(stacks *stack.Set, vars Variables) (crossStackOutput *CrossStackOu
 		dynamicvariables.With(vars.StableGenerate, func() (stack.TFVars, error) {
 			resolvedImageTag, err := vars.Environment.Deploy.ResolveTag(vars.Image)
 			return stack.TFVars{tfVarKeyResolvedImageTag: resolvedImageTag}, err
+		}),
+		sentryprovider.With(gsmsecret.DataConfig{
+			Secret:    googlesecretsmanager.SecretSentryAuthToken,
+			ProjectID: googlesecretsmanager.SharedSecretsProjectID,
 		}))
 	if err != nil {
 		return nil, err
@@ -116,6 +129,9 @@ func NewStack(stacks *stack.Set, vars Variables) (crossStackOutput *CrossStackOu
 		cloudRunBuilder.AddEnv("EXTERNAL_DNS_NAME", dnsName)
 	}
 
+	// Add environment ID env var
+	cloudRunBuilder.AddEnv("ENVIRONMENT_ID", vars.Environment.ID)
+
 	// Add user-configured env vars
 	if err := addContainerEnvVars(cloudRunBuilder, vars.Environment.Env, vars.Environment.SecretEnv, envVariablesData{
 		ProjectID:      vars.ProjectID,
@@ -261,6 +277,43 @@ func NewStack(stacks *stack.Set, vars Variables) (crossStackOutput *CrossStackOu
 		}
 	}
 
+	// Sentry
+	var sentryProject sentryproject.Project
+	{
+		id := id.Group("sentry")
+		// Get the Sentry organization
+		organization := datasentryorganization.NewDataSentryOrganization(stack, id.TerraformID("organization"), &datasentryorganization.DataSentryOrganizationConfig{
+			Slug: pointers.Ptr(SentryOrganization),
+		})
+
+		// Get the Sourcegraph team - we don't use individual owner teams
+		// because it's hard to tell whether they already exist or not, and
+		// it's not really important enough to force operators to create a
+		// team by hand. We depend on Opsgenie teams for concrete ownership
+		// instead.
+		sentryTeam := datasentryteam.NewDataSentryTeam(stack, id.TerraformID("team"), &datasentryteam.DataSentryTeamConfig{
+			Organization: organization.Id(),
+			Slug:         pointers.Ptr("sourcegraph"),
+		})
+
+		// Create the project
+		sentryProject = sentryproject.NewProject(stack, id.TerraformID("project"), &sentryproject.ProjectConfig{
+			Organization: organization.Id(),
+			Name:         pointers.Stringf("%s - %s", vars.Service.GetName(), vars.Environment.ID),
+			Slug:         pointers.Stringf("%s-%s", vars.Service.ID, vars.Environment.ID),
+			Teams:        &[]*string{sentryTeam.Slug()},
+		})
+
+		// Create a DSN
+		key := key.NewKey(stack, id.TerraformID("dsn"), &key.KeyConfig{
+			Organization: organization.Id(),
+			Project:      sentryProject.Slug(),
+			Name:         pointers.Ptr("Managed Servcies Platform"),
+		})
+
+		cloudRunBuilder.AddEnv("SENTRY_DSN", *key.DsnPublic())
+	}
+
 	// Finalize output of builder
 	cloudRunBuilder.AddEnv("SSL_CERT_DIR", strings.Join(sslCertDirs, ":"))
 	cloudRunResource, err := cloudRunBuilder.Build(stack, builder.Variables{
@@ -292,6 +345,7 @@ func NewStack(stacks *stack.Set, vars Variables) (crossStackOutput *CrossStackOu
 	locals.Add("image_tag", *imageTag.StringValue,
 		"Resolved tag of service image to deploy")
 	return &CrossStackOutput{
+		SentryProject:     sentryProject,
 		DiagnosticsSecret: diagnosticsSecret,
 		RedisInstanceID:   redisInstanceID,
 	}, nil

dev/managedservicesplatform/stacks/monitoring/BUILD.bazel

@@ -13,10 +13,12 @@ go_library(
         "//dev/managedservicesplatform/internal/resource/alertpolicy",
         "//dev/managedservicesplatform/internal/resource/gsmsecret",
         "//dev/managedservicesplatform/internal/resource/random",
+        "//dev/managedservicesplatform/internal/resource/sentryalert",
         "//dev/managedservicesplatform/internal/resourceid",
         "//dev/managedservicesplatform/internal/stack",
         "//dev/managedservicesplatform/internal/stack/options/googleprovider",
         "//dev/managedservicesplatform/internal/stack/options/opsgenieprovider",
+        "//dev/managedservicesplatform/internal/stack/options/sentryprovider",
         "//dev/managedservicesplatform/internal/stack/options/slackprovider",
         "//dev/managedservicesplatform/spec",
         "//lib/errors",
@@ -26,6 +28,9 @@ go_library(
         "@com_github_sourcegraph_managed_services_platform_cdktf_gen_google//monitoringuptimecheckconfig",
         "@com_github_sourcegraph_managed_services_platform_cdktf_gen_opsgenie//apiintegration",
         "@com_github_sourcegraph_managed_services_platform_cdktf_gen_opsgenie//dataopsgenieteam",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//datasentryorganizationintegration",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//notificationaction",
+        "@com_github_sourcegraph_managed_services_platform_cdktf_gen_sentry//project",
         "@com_github_sourcegraph_managed_services_platform_cdktf_gen_slack//conversation",
     ],
 )

dev/managedservicesplatform/stacks/monitoring/monitoring.go

@@ -10,16 +10,21 @@ import (
 	"github.com/sourcegraph/managed-services-platform-cdktf/gen/google/monitoringuptimecheckconfig"
 	opsgenieintegration "github.com/sourcegraph/managed-services-platform-cdktf/gen/opsgenie/apiintegration"
 	"github.com/sourcegraph/managed-services-platform-cdktf/gen/opsgenie/dataopsgenieteam"
+	"github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/datasentryorganizationintegration"
+	"github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/notificationaction"
+	sentryproject "github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry/project"
 	slackconversation "github.com/sourcegraph/managed-services-platform-cdktf/gen/slack/conversation"
 
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/googlesecretsmanager"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/alertpolicy"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/gsmsecret"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/random"
+	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resource/sentryalert"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/resourceid"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/googleprovider"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/opsgenieprovider"
+	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/sentryprovider"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/internal/stack/options/slackprovider"
 	"github.com/sourcegraph/sourcegraph/dev/managedservicesplatform/spec"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
@@ -82,8 +87,7 @@ type Variables struct {
 	EnvironmentCategory spec.EnvironmentCategory
 	// EnvironmentID is the name of the service environment.
 	EnvironmentID string
-
-	Monitoring spec.MonitoringSpec
+	Monitoring    spec.MonitoringSpec
 	// MaxInstanceCount informs service scaling alerts.
 	MaxInstanceCount *int
 	// ExternalDomain informs external health checks on the service domain.
@@ -98,6 +102,8 @@ type Variables struct {
 	// ServiceHealthProbes is used to determine the threshold for service
 	// startup latency alerts.
 	ServiceHealthProbes *spec.EnvironmentServiceHealthProbesSpec
+	// SentryProject is the project in Sentry for the service environment
+	SentryProject sentryproject.Project
 }
 
 const StackName = "monitoring"
@@ -114,6 +120,10 @@ func NewStack(stacks *stack.Set, vars Variables) (*CrossStackOutput, error) {
 		slackprovider.With(gsmsecret.DataConfig{
 			Secret:    googlesecretsmanager.SecretSlackOperatorOAuthToken,
 			ProjectID: googlesecretsmanager.SharedSecretsProjectID,
+		}),
+		sentryprovider.With(gsmsecret.DataConfig{
+			Secret:    googlesecretsmanager.SecretSentryAuthToken,
+			ProjectID: googlesecretsmanager.SharedSecretsProjectID,
 		}))
 	if err != nil {
 		return nil, err
@@ -226,6 +236,28 @@ func NewStack(stacks *stack.Set, vars Variables) (*CrossStackOutput, error) {
 				// In case it already exists
 				AdoptExistingChannel: pointers.Ptr(true),
 			})
+
+			// Sentry Slack integration
+			dataSentryOrganizationIntegration := datasentryorganizationintegration.NewDataSentryOrganizationIntegration(stack, id.TerraformID("sentry_integration"), &datasentryorganizationintegration.DataSentryOrganizationIntegrationConfig{
+				Organization: vars.SentryProject.Organization(),
+				ProviderKey:  pointers.Ptr("slack"),
+				Name:         pointers.Ptr("Sourcegraph"),
+			})
+
+			// Provision Sentry Slack notification
+			_ = notificationaction.NewNotificationAction(stack, id.TerraformID("sentry_notification_channel"), &notificationaction.NotificationActionConfig{
+				Organization:     vars.SentryProject.Organization(),
+				Projects:         &[]*string{vars.SentryProject.Slug()},
+				ServiceType:      pointers.Ptr("slack"),
+				IntegrationId:    dataSentryOrganizationIntegration.Id(),
+				TargetDisplay:    slackChannel.Name(),
+				TargetIdentifier: slackChannel.Id(),
+				TriggerType:      pointers.Ptr("spike-protection"),
+			})
+
+			if err = createSentryAlerts(stack, id.Group("sentry_alerts"), vars, slackChannel, dataSentryOrganizationIntegration); err != nil {
+				return nil, errors.Wrap(err, "failed to create Sentry alerts")
+			}
 		}
 
 		channels = append(channels,
@@ -640,3 +672,47 @@ func createRedisAlerts(
 
 	return nil
 }
+
+func createSentryAlerts(
+	stack cdktf.TerraformStack,
+	id resourceid.ID,
+	vars Variables,
+	channel slackconversation.Conversation,
+	slackIntegration datasentryorganizationintegration.DataSentryOrganizationIntegration,
+) error {
+	for _, config := range []sentryalert.Config{
+		{
+			ID:            "all-issues",
+			SentryProject: vars.SentryProject,
+			AlertConfig: sentryalert.AlertConfig{
+				Name:      "Notify in Slack",
+				Frequency: 15, // Notify for an issue at most once every 15 minutes
+				Conditions: []sentryalert.Condition{
+					{
+						ID:       sentryalert.EventFrequencyCondition,
+						Value:    pointers.Ptr(0), // Always (seen more than 0 times) during interval
+						Interval: pointers.Ptr("15m"),
+					},
+				},
+				ActionMatch: sentryalert.ActionMatchAny,
+				Actions: []sentryalert.Action{
+					{
+						ID: sentryalert.SlackNotifyServiceAction,
+						ActionParameters: map[string]any{
+							"workspace":  slackIntegration.Id(),
+							"channel":    channel.Name(),
+							"channel_id": channel.Id(),
+							"tags": pointers.Stringf("msp-%s-%s",
+								vars.Service.ID, vars.EnvironmentID),
+						},
+					},
+				},
+			},
+		},
+	} {
+		if _, err := sentryalert.New(stack, id.Group(config.ID), config); err != nil {
+			return err
+		}
+	}
+	return nil
+}

go.mod

@@ -277,6 +277,7 @@ require (
 	github.com/sourcegraph/managed-services-platform-cdktf/gen/opsgenie v0.0.0-20231230001101-a13188f9c749
 	github.com/sourcegraph/managed-services-platform-cdktf/gen/postgresql v0.0.0-20231220215815-b87ebb3e8c47
 	github.com/sourcegraph/managed-services-platform-cdktf/gen/random v0.0.0-20230822024612-edb48c530722
+	github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry v0.0.0-20240109225336-01188650a68d
 	github.com/sourcegraph/managed-services-platform-cdktf/gen/slack v0.0.0-20240103014439-025608ddf849
 	github.com/sourcegraph/managed-services-platform-cdktf/gen/tfe v0.0.0-20231218231056-4749baca142f
 	github.com/sourcegraph/sourcegraph/lib/managedservicesplatform v0.0.0-00010101000000-000000000000

go.sum

@@ -1656,6 +1656,8 @@ github.com/sourcegraph/managed-services-platform-cdktf/gen/postgresql v0.0.0-202
 github.com/sourcegraph/managed-services-platform-cdktf/gen/postgresql v0.0.0-20231220215815-b87ebb3e8c47/go.mod h1:lQ1E8rSHgTmL8GmtcQFXS75rqQrCmuQRXZWh7A+Fp6s=
 github.com/sourcegraph/managed-services-platform-cdktf/gen/random v0.0.0-20230822024612-edb48c530722 h1:N0OxHqeujHxvVU666KQY9whauLyw4s3BJGBLxx6gKR0=
 github.com/sourcegraph/managed-services-platform-cdktf/gen/random v0.0.0-20230822024612-edb48c530722/go.mod h1:TiUqRvYs/Gah8bGw/toyVWCaP3dnCB4tBh3jf5HGdo0=
+github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry v0.0.0-20240109225336-01188650a68d h1:wHEDTUur95a2HtUUodA8OXQi0Uefwa60CvhN4fLIuhU=
+github.com/sourcegraph/managed-services-platform-cdktf/gen/sentry v0.0.0-20240109225336-01188650a68d/go.mod h1:RyE/dPHsCoF0m2QevmerpS/JPm7ixgwmkUiqYKaxxNw=
 github.com/sourcegraph/managed-services-platform-cdktf/gen/slack v0.0.0-20240103014439-025608ddf849 h1:Iip3TVeiTF8HyeRsK0MKMZ1vQ87fqFqwz0cVq1dib6E=
 github.com/sourcegraph/managed-services-platform-cdktf/gen/slack v0.0.0-20240103014439-025608ddf849/go.mod h1:TYxLrVLdpyViPv8f6le0oSmILOLIQVMq3WvQFIPS964=
 github.com/sourcegraph/managed-services-platform-cdktf/gen/tfe v0.0.0-20231218231056-4749baca142f h1:sCs3+EDrM4pp7PI0/efZPZZK+u955+gwr0byWGg+AVg=

lib/managedservicesplatform/runtime/contract.go

@@ -50,6 +50,7 @@ type internalContract struct {
 	diagnosticsSecret *string
 	opentelemetry     opentelemetry.Config
 	sentryDSN         *string
+	environmentID     *string
 }
 
 func newContract(logger log.Logger, env *Env, service ServiceMetadata) Contract {
@@ -73,7 +74,8 @@ func newContract(logger log.Logger, env *Env, service ServiceMetadata) Contract
 					env.GetOptional("OTEL_GCP_PROJECT_ID", "GCP project ID for OpenTelemetry export"),
 					defaultGCPProjectID),
 			},
-			sentryDSN: env.GetOptional("SENTRY_DSN", "Sentry error reporting DSN"),
+			sentryDSN:     env.GetOptional("SENTRY_DSN", "Sentry error reporting DSN"),
+			environmentID: env.GetOptional("ENVIRONMENT_ID", "MSP Service Environment ID"),
 		},
 	}
 }

lib/managedservicesplatform/runtime/service.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/sourcegraph/sourcegraph/lib/background"
 	"github.com/sourcegraph/sourcegraph/lib/managedservicesplatform/runtime/internal/opentelemetry"
+	"github.com/sourcegraph/sourcegraph/lib/pointers"
 )
 
 type ServiceMetadata interface {
@@ -77,7 +78,8 @@ func Start[
 			return log.SinksConfig{
 				Sentry: &log.SentrySink{
 					ClientOptions: sentry.ClientOptions{
-						Dsn: *contract.internal.sentryDSN,
+						Dsn:         *contract.internal.sentryDSN,
+						Environment: pointers.Deref(contract.internal.environmentID, "unspecified"),
 					},
 				},
 			}