dev: Easier development in sourcegraph.com mode (#18172)

When running with SOURCEGRAPHDOTCOM_MODE=true set we require that at
least one GitHub and GitLab external service have their cloud_default
flags set.

This change allows us to enforce this by setting the CloudDefault field
on config in dev-private.

cmd/frontend/internal/cli/config.go

@@ -25,8 +25,10 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/database"
 	"github.com/sourcegraph/sourcegraph/internal/database/confdb"
 	"github.com/sourcegraph/sourcegraph/internal/database/dbutil"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc"
 	"github.com/sourcegraph/sourcegraph/internal/jsonc"
 	"github.com/sourcegraph/sourcegraph/internal/types"
+	"github.com/sourcegraph/sourcegraph/schema"
 )
 
 func printConfigValidation() {
@@ -125,12 +127,12 @@ func handleConfigOverrides() error {
 		}
 		confGet := func() *conf.Unified { return parsed }
 
-		extsvc, err := ioutil.ReadFile(overrideExtSvcConfig)
+		extsvcConfig, err := ioutil.ReadFile(overrideExtSvcConfig)
 		if err != nil {
 			return errors.Wrap(err, "reading EXTSVC_CONFIG_FILE")
 		}
 		var rawConfigs map[string][]*json.RawMessage
-		if err := jsonc.Unmarshal(string(extsvc), &rawConfigs); err != nil {
+		if err := jsonc.Unmarshal(string(extsvcConfig), &rawConfigs); err != nil {
 			return errors.Wrap(err, "parsing EXTSVC_CONFIG_FILE")
 		}
 		if len(rawConfigs) == 0 {
@@ -167,10 +169,32 @@ func handleConfigOverrides() error {
 				if err != nil {
 					return errors.Wrap(err, fmt.Sprintf("marshaling extsvc config ([%v][%v])", key, i))
 				}
+
+				// In development we can set the value of the cloud_default column by setting the
+				// CloudDefault value in config.
+				var cloudDefault bool
+				switch key {
+				case extsvc.KindGitHub:
+					var c schema.GitHubConnection
+					if err = json.Unmarshal(marshaledCfg, &c); err != nil {
+						return err
+					}
+					cloudDefault = c.CloudDefault
+
+				case extsvc.KindGitLab:
+					var c schema.GitLabConnection
+					if err = json.Unmarshal(marshaledCfg, &c); err != nil {
+						return err
+					}
+					cloudDefault = c.CloudDefault
+
+				}
+
 				toAdd[&types.ExternalService{
-					Kind:        key,
-					DisplayName: fmt.Sprintf("%s #%d", key, i+1),
-					Config:      string(marshaledCfg),
+					Kind:         key,
+					DisplayName:  fmt.Sprintf("%s #%d", key, i+1),
+					Config:       string(marshaledCfg),
+					CloudDefault: cloudDefault,
 				}] = true
 			}
 		}
@@ -214,7 +238,7 @@ func handleConfigOverrides() error {
 		for id, extSvc := range toUpdate {
 			log.Debug("Updating external service", "id", id, "displayName", extSvc.DisplayName)
 
-			update := &database.ExternalServiceUpdate{DisplayName: &extSvc.DisplayName, Config: &extSvc.Config}
+			update := &database.ExternalServiceUpdate{DisplayName: &extSvc.DisplayName, Config: &extSvc.Config, CloudDefault: &extSvc.CloudDefault}
 			if err := database.GlobalExternalServices.Update(ctx, ps, id, update); err != nil {
 				return errors.Wrap(err, "ExternalServices.Update")
 			}

enterprise/dev/start.sh

@@ -10,7 +10,7 @@ if [ ! -d "$DEV_PRIVATE_PATH" ]; then
 fi
 
 # Warn if dev-private needs to be updated.
-required_commit="a661975799edd759b3d6e4c4027e69a9727bdffb"
+required_commit="eff6cce71f6b147f61567d19f16cbf03da0d7696"
 if ! git -C "$DEV_PRIVATE_PATH" merge-base --is-ancestor $required_commit HEAD; then
   echo "Error: You need to update dev-private to a commit that incorporates https://github.com/sourcegraph/dev-private/commit/$required_commit."
   echo

internal/database/external_services.go

@@ -650,8 +650,9 @@ RETURNING *
 
 // ExternalServiceUpdate contains optional fields to update.
 type ExternalServiceUpdate struct {
-	DisplayName *string
-	Config      *string
+	DisplayName  *string
+	Config       *string
+	CloudDefault *bool
 }
 
 // Update updates an external service.
@@ -727,6 +728,12 @@ func (e *ExternalServiceStore) Update(ctx context.Context, ps []schema.AuthProvi
 			return err
 		}
 	}
+
+	if update.CloudDefault != nil {
+		if err := execUpdate(ctx, tx.DB(), sqlf.Sprintf("cloud_default=%s", update.CloudDefault)); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 

internal/database/external_services_test.go

@@ -416,6 +416,7 @@ func TestExternalServicesStore_Update(t *testing.T) {
 		name             string
 		update           *ExternalServiceUpdate
 		wantUnrestricted bool
+		wantCloudDefault bool
 	}{
 		{
 			name: "update with authorization",
@@ -424,6 +425,7 @@ func TestExternalServicesStore_Update(t *testing.T) {
 				Config:      strptr(`{"url": "https://github.com", "repositoryQuery": ["none"], "token": "def", "authorization": {}}`),
 			},
 			wantUnrestricted: false,
+			wantCloudDefault: false,
 		},
 		{
 			name: "update without authorization",
@@ -432,6 +434,7 @@ func TestExternalServicesStore_Update(t *testing.T) {
 				Config:      strptr(`{"url": "https://github.com", "repositoryQuery": ["none"], "token": "def"}`),
 			},
 			wantUnrestricted: true,
+			wantCloudDefault: false,
 		},
 		{
 			name: "update with authorization in comments",
@@ -446,6 +449,23 @@ func TestExternalServicesStore_Update(t *testing.T) {
 }`),
 			},
 			wantUnrestricted: true,
+			wantCloudDefault: false,
+		},
+		{
+			name: "set cloud_default true",
+			update: &ExternalServiceUpdate{
+				DisplayName:  strptr("GITHUB (updated) #3"),
+				CloudDefault: boolptr(true),
+				Config: strptr(`
+{
+	"url": "https://github.com",
+	"repositoryQuery": ["none"],
+	"token": "def",
+	// "authorization": {}
+}`),
+			},
+			wantUnrestricted: true,
+			wantCloudDefault: true,
 		},
 	}
 	for _, test := range tests {
@@ -472,6 +492,10 @@ func TestExternalServicesStore_Update(t *testing.T) {
 			if test.wantUnrestricted != got.Unrestricted {
 				t.Fatalf("Want unrestricted = %v, but got %v", test.wantUnrestricted, got.Unrestricted)
 			}
+
+			if test.wantCloudDefault != got.CloudDefault {
+				t.Fatalf("Want cloud_default = %v, but got %v", test.wantCloudDefault, got.CloudDefault)
+			}
 		})
 	}
 }

internal/database/test_util.go

@@ -7,3 +7,11 @@ func MockEmailExistsErr() error {
 func MockUsernameExistsErr() error {
 	return errCannotCreateUser{errorCodeEmailExists}
 }
+
+func strptr(s string) *string {
+	return &s
+}
+
+func boolptr(b bool) *bool {
+	return &b
+}

internal/database/user_emails_test.go

@@ -525,10 +525,6 @@ func TestUserEmails_GetLatestVerificationSentEmail(t *testing.T) {
 	}
 }
 
-func strptr(s string) *string {
-	return &s
-}
-
 func TestUserEmails_GetVerifiedEmails(t *testing.T) {
 	if testing.Short() {
 		t.Skip()

schema/github.schema.json

@@ -171,6 +171,12 @@
       "type": "boolean",
       "default": false,
       "deprecationMessage": "DEPRECATED: The cloud_default flag should be set in the database instead"
+    },
+    "cloudDefault": {
+      "title": "CloudDefault",
+      "description": "Only to be used in development to override the cloud_default column",
+      "type": "boolean",
+      "default": false
     }
   }
 }

schema/github_stringdata.go

@@ -176,6 +176,12 @@ const GitHubSchemaJSON = `{
       "type": "boolean",
       "default": false,
       "deprecationMessage": "DEPRECATED: The cloud_default flag should be set in the database instead"
+    },
+    "cloudDefault": {
+      "title": "CloudDefault",
+      "description": "Only to be used in development to override the cloud_default column",
+      "type": "boolean",
+      "default": false
     }
   }
 }

schema/gitlab.schema.json

@@ -200,6 +200,12 @@
       "type": "boolean",
       "default": false,
       "deprecationMessage": "DEPRECATED: The cloud_default flag should be set in the database instead"
+    },
+    "cloudDefault": {
+      "title": "CloudDefault",
+      "description": "Only to be used in development to override the cloud_default column",
+      "type": "boolean",
+      "default": false
     }
   },
   "definitions": {

schema/gitlab_stringdata.go

@@ -205,6 +205,12 @@ const GitLabSchemaJSON = `{
       "type": "boolean",
       "default": false,
       "deprecationMessage": "DEPRECATED: The cloud_default flag should be set in the database instead"
+    },
+    "cloudDefault": {
+      "title": "CloudDefault",
+      "description": "Only to be used in development to override the cloud_default column",
+      "type": "boolean",
+      "default": false
     }
   },
   "definitions": {

schema/schema.go

@@ -522,6 +522,8 @@ type GitHubConnection struct {
 	Authorization *GitHubAuthorization `json:"authorization,omitempty"`
 	// Certificate description: TLS certificate of the GitHub Enterprise instance. This is only necessary if the certificate is self-signed or signed by an internal CA. To get the certificate run `openssl s_client -connect HOST:443 -showcerts < /dev/null 2> /dev/null | openssl x509 -outform PEM`. To escape the value into a JSON string, you may want to use a tool like https://json-escape-text.now.sh.
 	Certificate string `json:"certificate,omitempty"`
+	// CloudDefault description: Only to be used in development to override the cloud_default column
+	CloudDefault bool `json:"cloudDefault,omitempty"`
 	// CloudGlobal description: When set to true, this external service will be chosen as our 'Global' GitHub service. Only valid on Sourcegraph.com. Only one service can have this flag set.
 	CloudGlobal bool `json:"cloudGlobal,omitempty"`
 	// Exclude description: A list of repositories to never mirror from this GitHub instance. Takes precedence over "orgs", "repos", and "repositoryQuery" configuration.
@@ -613,6 +615,8 @@ type GitLabConnection struct {
 	Authorization *GitLabAuthorization `json:"authorization,omitempty"`
 	// Certificate description: TLS certificate of the GitLab instance. This is only necessary if the certificate is self-signed or signed by an internal CA. To get the certificate run `openssl s_client -connect HOST:443 -showcerts < /dev/null 2> /dev/null | openssl x509 -outform PEM`. To escape the value into a JSON string, you may want to use a tool like https://json-escape-text.now.sh.
 	Certificate string `json:"certificate,omitempty"`
+	// CloudDefault description: Only to be used in development to override the cloud_default column
+	CloudDefault bool `json:"cloudDefault,omitempty"`
 	// CloudGlobal description: When set to true, this external service will be chosen as our 'Global' GitLab service. Only valid on Sourcegraph.com. Only one service can have this flag set.
 	CloudGlobal bool `json:"cloudGlobal,omitempty"`
 	// Exclude description: A list of projects to never mirror from this GitLab instance. Takes precedence over "projects" and "projectQuery" configuration. Supports excluding by name ({"name": "group/name"}) or by ID ({"id": 42}).