webhooks: Support bitbucket server push events (#45909)

Handle Bitbucket Server webhook push events.

Co-authored-by: Indradhanush Gupta <indradhanush.gupta@gmail.com>

CHANGELOG.md

@@ -22,6 +22,7 @@ All notable changes to Sourcegraph are documented in this file.
 - Templates for certain emails sent by Sourcegraph are now configurable via `email.templates` in site configuration. [#45671](https://github.com/sourcegraph/sourcegraph/pull/45671)
 - Keyboard navigation for search results is now enabled by default. Use Arrow Up/Down keys to navigate between search results, Arrow Left/Right to collapse and expand file matches, Enter to open the search result in the current tab, Ctrl/Cmd+Enter to open the result in a separate tab, / to refocus the search input, and Ctrl/Cmd+Arrow Down to jump from the search input to the first result. Arrow Left/Down/Up/Right in previous examples can be substituted with h/j/k/l for Vim-style bindings. Keyboard navigation can be disabled by creating the `search-results-keyboard-navigation` feature flag and setting it to false. [#45890](https://github.com/sourcegraph/sourcegraph/pull/45890)
 - Added support for receiving GitLab webhook `push` events. [#45856](https://github.com/sourcegraph/sourcegraph/pull/45856)
+- Added support for receiving Bitbucket Server / Datacenter webhook `push` events. [#45909](https://github.com/sourcegraph/sourcegraph/pull/45909)
 
 ### Changed
 

cmd/frontend/enterprise/enterprise.go

@@ -28,8 +28,9 @@ type Services struct {
 	BatchesChangesFileUploadHandler http.Handler
 
 	// Handle `push` events
-	GitHubSyncWebhook webhooks.Registerer
-	GitLabSyncWebhook webhooks.Registerer
+	GitHubSyncWebhook          webhooks.Registerer
+	GitLabSyncWebhook          webhooks.Registerer
+	BitbucketServerSyncWebhook webhooks.Registerer
 
 	PermissionsGitHubWebhook    webhooks.Registerer
 	NewCodeIntelUploadHandler   NewCodeIntelUploadHandler
@@ -79,6 +80,7 @@ func DefaultServices() Services {
 	return Services{
 		GitHubSyncWebhook:               &emptyWebhookHandler{name: "github sync webhook"},
 		GitLabSyncWebhook:               &emptyWebhookHandler{name: "gitlab sync webhook"},
+		BitbucketServerSyncWebhook:      &emptyWebhookHandler{name: "bitbucket server sync webhook"},
 		PermissionsGitHubWebhook:        &emptyWebhookHandler{name: "permissions github webhook"},
 		BatchesGitHubWebhook:            &emptyWebhookHandler{name: "batches github webhook"},
 		BatchesGitLabWebhook:            &emptyWebhookHandler{name: "batches gitlab webhook"},

cmd/frontend/internal/cli/serve_cmd.go

@@ -302,6 +302,7 @@ func makeExternalAPI(db database.DB, logger sglog.Logger, schema *graphql.Schema
 		&httpapi.Handlers{
 			GitHubSyncWebhook:               enterprise.GitHubSyncWebhook,
 			GitLabSyncWebhook:               enterprise.GitLabSyncWebhook,
+			BitbucketServerSyncWebhook:      enterprise.BitbucketServerSyncWebhook,
 			PermissionsGitHubWebhook:        enterprise.PermissionsGitHubWebhook,
 			BatchesGitHubWebhook:            enterprise.BatchesGitHubWebhook,
 			BatchesGitLabWebhook:            enterprise.BatchesGitLabWebhook,

cmd/frontend/internal/httpapi/api_test.go

@@ -37,6 +37,7 @@ func newTest(t *testing.T) *httptestutil.Client {
 			BatchesGitLabWebhook:          enterpriseServices.BatchesGitLabWebhook,
 			GitHubSyncWebhook:             enterpriseServices.GitHubSyncWebhook,
 			GitLabSyncWebhook:             enterpriseServices.GitLabSyncWebhook,
+			BitbucketServerSyncWebhook:    enterpriseServices.BitbucketServerSyncWebhook,
 			BatchesBitbucketServerWebhook: enterpriseServices.BatchesBitbucketServerWebhook,
 			BatchesBitbucketCloudWebhook:  enterpriseServices.BatchesBitbucketCloudWebhook,
 			NewCodeIntelUploadHandler:     enterpriseServices.NewCodeIntelUploadHandler,

cmd/frontend/internal/httpapi/httpapi.go

@@ -39,9 +39,15 @@ import (
 )
 
 type Handlers struct {
-	GitHubSyncWebhook               webhooks.Registerer
-	GitLabSyncWebhook               webhooks.Registerer
-	PermissionsGitHubWebhook        webhooks.Registerer
+	// Repo sync
+	GitHubSyncWebhook          webhooks.Registerer
+	GitLabSyncWebhook          webhooks.Registerer
+	BitbucketServerSyncWebhook webhooks.Registerer
+
+	// Permissions
+	PermissionsGitHubWebhook webhooks.Registerer
+
+	// Batch changes
 	BatchesGitHubWebhook            webhooks.Registerer
 	BatchesGitLabWebhook            webhooks.RegistererHandler
 	BatchesBitbucketServerWebhook   webhooks.RegistererHandler
@@ -49,8 +55,12 @@ type Handlers struct {
 	BatchesChangesFileGetHandler    http.Handler
 	BatchesChangesFileExistsHandler http.Handler
 	BatchesChangesFileUploadHandler http.Handler
-	NewCodeIntelUploadHandler       enterprise.NewCodeIntelUploadHandler
-	NewComputeStreamHandler         enterprise.NewComputeStreamHandler
+
+	// Code intel
+	NewCodeIntelUploadHandler enterprise.NewCodeIntelUploadHandler
+
+	// Compute
+	NewComputeStreamHandler enterprise.NewComputeStreamHandler
 }
 
 // NewHandler returns a new API handler that uses the provided API
@@ -90,12 +100,13 @@ func NewHandler(
 	)
 
 	wh := webhooks.Router{
-		Logger: logger.Scoped("Router", "handling webhook requests and dispatching them to handlers"),
+		Logger: logger.Scoped("webhooks.Router", "handling webhook requests and dispatching them to handlers"),
 		DB:     db,
 	}
 	webhookhandlers.Init(&wh)
 	handlers.BatchesGitHubWebhook.Register(&wh)
 	handlers.BatchesGitLabWebhook.Register(&wh)
+	handlers.BitbucketServerSyncWebhook.Register(&wh)
 	handlers.BatchesBitbucketServerWebhook.Register(&wh)
 	handlers.BatchesBitbucketCloudWebhook.Register(&wh)
 	handlers.GitHubSyncWebhook.Register(&wh)

cmd/frontend/webhooks/bitbucketserver_webhooks.go

@@ -52,9 +52,22 @@ func (wr *Router) handleBitbucketServerWebhook(logger log.Logger, w http.Respons
 		http.Error(w, "Error while reading request body.", http.StatusInternalServerError)
 		return
 	}
-	defer r.Body.Close()
+	if err := r.Body.Close(); err != nil {
+		http.Error(w, "Closing body", http.StatusInternalServerError)
+		return
+	}
+
+	sig := r.Header.Get("X-Hub-Signature")
+	eventKey := r.Header.Get("X-Event-Key")
+
+	// Special case: Even if a secret is configured, Bitbucket server test events are
+	// not signed, so we allow them through without verification.
+	if sig == "" && eventKey == "diagnostics:ping" {
+		wr.HandleBitBucketServerWebhook(logger, w, r, urn, payload)
+		return
+	}
+
 	if secret != "" {
-		sig := r.Header.Get("X-Hub-Signature")
 		if err := gh.ValidateSignature(sig, payload, []byte(secret)); err != nil {
 			http.Error(w, "Could not validate payload with secret.", http.StatusBadRequest)
 			return

doc/admin/config/webhooks.md

@@ -13,7 +13,7 @@ Code host | [Batch changes](../../batch_changes/index.md) | Code push | User per
 --------- | :-: | :-: | :-:
 GitHub | 🟢 | 🟢 | 🟢
 GitLab | 🟢 | 🟢 | 🔴
-Bitbucket Server / Datacenter | 🟢 | 🔴 | 🔴
+Bitbucket Server / Datacenter | 🟢 | 🟢 | 🔴
 Bitbucket Cloud | 🟢 | 🔴 | 🔴
 
 To receive webhooks both Sourcegraph and the code host need to be configured. To configure Sourcegraph, [add an incoming webhook](#adding-an-incoming-webhook). Then [configure webhooks on your code host](#configuring-webhooks-on-the-code-host)
@@ -126,6 +126,10 @@ The [Sourcegraph Bitbucket Server plugin](../../integration/bitbucket_server.md#
 
 Done! Sourcegraph will now receive webhook events from Bitbucket Server / Bitbucket Data Center and use them to sync pull request events, used by [batch changes](../../batch_changes/index.md), faster and more efficiently.
 
+#### Code push
+
+Follow the same steps as above, but ensure you tick the `Push` option. If asked for a specific event, use `repo:refs_changed`.
+
 ### Bitbucket cloud
 
 #### Batch changes

doc/admin/repo/webhooks.md

@@ -22,4 +22,4 @@ For repositories that Sourcegraph is already aware of, it will periodically perf
 
 ## Code host webhooks
 
-We support receiving webhooks directly from your code host for [GitHub](../config/webhooks.md#github) and [GitLab](../config/webhooks.md#gitlab).
+We support receiving webhooks directly from your code host for [GitHub](../config/webhooks.md#github), [GitLab](../config/webhooks.md#gitlab) and [Bitbucket Server](../config/webhooks.md#bitbucket-server).

enterprise/cmd/frontend/internal/repos/webhooks/bitbucket_server_handler.go

@@ -0,0 +1,77 @@
+package webhooks
+
+import (
+	"context"
+	"net/url"
+	"strings"
+
+	"github.com/sourcegraph/log"
+
+	"github.com/sourcegraph/sourcegraph/cmd/frontend/webhooks"
+	"github.com/sourcegraph/sourcegraph/internal/api"
+	"github.com/sourcegraph/sourcegraph/internal/database"
+	"github.com/sourcegraph/sourcegraph/internal/errcode"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc/bitbucketserver"
+	"github.com/sourcegraph/sourcegraph/internal/repoupdater"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+type BitbucketServerHandler struct {
+	logger log.Logger
+}
+
+func (g *BitbucketServerHandler) Register(router *webhooks.Router) {
+	router.Register(func(ctx context.Context, _ database.DB, _ extsvc.CodeHostBaseURL, payload any) error {
+		return g.handlePushEvent(ctx, payload)
+	}, extsvc.KindBitbucketServer, "repo:refs_changed")
+}
+
+func NewBitbucketServerHandler() *BitbucketServerHandler {
+	return &BitbucketServerHandler{
+		logger: log.Scoped("webhooks.BitbucketServerHandler", "bitbucket server webhook handler"),
+	}
+}
+
+func (g *BitbucketServerHandler) handlePushEvent(ctx context.Context, payload any) error {
+	event, ok := payload.(*bitbucketserver.PushEvent)
+	if !ok {
+		return errors.Newf("expected BitbucketServer.PushEvent, got %T", payload)
+	}
+
+	repoName, err := bitbucketServerNameFromEvent(event)
+	if err != nil {
+		return errors.Wrap(err, "handlePushEvent: get name failed")
+	}
+
+	resp, err := repoupdater.DefaultClient.EnqueueRepoUpdate(ctx, repoName)
+	if err != nil {
+		// Repo not existing on Sourcegraph is fine
+		if errcode.IsNotFound(err) {
+			g.logger.Warn("BitbucketServer push event received for unknown repo", log.String("repo", string(repoName)))
+			return nil
+		}
+		return errors.Wrap(err, "handlePushEvent: EnqueueRepoUpdate failed")
+	}
+
+	g.logger.Info("successfully updated", log.String("name", resp.Name))
+	return nil
+}
+
+func bitbucketServerNameFromEvent(event *bitbucketserver.PushEvent) (api.RepoName, error) {
+	if event == nil {
+		return "", errors.New("nil PushEvent received")
+	}
+	for _, link := range event.Repository.Links.Clone {
+		// The ssh link is the closest to our repo name
+		if link.Name != "ssh" {
+			continue
+		}
+		parsed, err := url.Parse(link.Href)
+		if err != nil {
+			return "", errors.Wrap(err, "unable to parse repository URL")
+		}
+		return api.RepoName(parsed.Hostname() + strings.TrimSuffix(parsed.Path, ".git")), nil
+	}
+	return "", errors.New("no ssh URLs found")
+}

enterprise/cmd/frontend/internal/repos/webhooks/bitbucket_server_handler_test.go

@@ -0,0 +1,95 @@
+package webhooks
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/sourcegraph/sourcegraph/internal/api"
+	"github.com/sourcegraph/sourcegraph/internal/extsvc/bitbucketserver"
+	"github.com/sourcegraph/sourcegraph/internal/repoupdater"
+	"github.com/sourcegraph/sourcegraph/internal/repoupdater/protocol"
+	"github.com/sourcegraph/sourcegraph/lib/errors"
+)
+
+func TestBitbucketServerHandler(t *testing.T) {
+	repoName := "bitbucket.sgdev.org/private/test-2020-06-01"
+	handler := NewBitbucketServerHandler()
+	data, err := os.ReadFile("testdata/bitbucket-server-push.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var payload bitbucketserver.PushEvent
+	if err := json.Unmarshal(data, &payload); err != nil {
+		t.Fatal(err)
+	}
+
+	var updateQueued string
+	repoupdater.MockEnqueueRepoUpdate = func(ctx context.Context, repo api.RepoName) (*protocol.RepoUpdateResponse, error) {
+		updateQueued = string(repo)
+		return &protocol.RepoUpdateResponse{
+			ID:   1,
+			Name: string(repo),
+		}, nil
+	}
+	t.Cleanup(func() { repoupdater.MockEnqueueRepoUpdate = nil })
+
+	if err := handler.handlePushEvent(context.Background(), &payload); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, repoName, updateQueued)
+}
+
+func TestBitbucketServerNameFromEvent(t *testing.T) {
+	makeEvent := func(name string, href string) *bitbucketserver.PushEvent {
+		return &bitbucketserver.PushEvent{
+			Repository: bitbucketserver.Repo{
+				Links: bitbucketserver.RepoLinks{
+					Clone: []bitbucketserver.Link{
+						{
+							Href: href,
+							Name: name,
+						},
+					},
+				},
+			},
+		}
+	}
+
+	tests := []struct {
+		name    string
+		event   *bitbucketserver.PushEvent
+		want    api.RepoName
+		wantErr error
+	}{
+		{
+			name:  "valid event",
+			event: makeEvent("ssh", "ssh://git@bitbucket.sgdev.org/private/test-2020-06-01"),
+			want:  api.RepoName("bitbucket.sgdev.org/private/test-2020-06-01"),
+		},
+		{
+			name:  "valid event with port",
+			event: makeEvent("ssh", "ssh://git@bitbucket.sgdev.org:7999/private/test-2020-06-01.git"),
+			want:  api.RepoName("bitbucket.sgdev.org/private/test-2020-06-01"),
+		},
+		{
+			name:    "nil event",
+			event:   nil,
+			want:    api.RepoName(""),
+			wantErr: errors.New("nil PushEvent received"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := bitbucketServerNameFromEvent(tt.event)
+			if tt.wantErr != nil {
+				assert.EqualError(t, tt.wantErr, err.Error())
+				return
+			}
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

enterprise/cmd/frontend/internal/repos/webhooks/github_handler.go

@@ -66,5 +66,5 @@ func githubNameFromEvent(event *gh.PushEvent) (api.RepoName, error) {
 	if err != nil {
 		return "", errors.Wrap(err, "unable to parse repository URL")
 	}
-	return api.RepoName(parsed.Host + parsed.Path), nil
+	return api.RepoName(parsed.Hostname() + parsed.Path), nil
 }

enterprise/cmd/frontend/internal/repos/webhooks/gitlab_handler.go

@@ -40,7 +40,7 @@ func (g *GitLabHandler) handlePushEvent(ctx context.Context, payload any) error
 
 	repoName, err := gitlabNameFromEvent(event)
 	if err != nil {
-		return errors.Wrap(err, "handleGitLabWebhook: get name failed")
+		return errors.Wrap(err, "handlePushEvent: get name failed")
 	}
 
 	resp, err := repoupdater.DefaultClient.EnqueueRepoUpdate(ctx, repoName)
@@ -59,11 +59,11 @@ func (g *GitLabHandler) handlePushEvent(ctx context.Context, payload any) error
 
 func gitlabNameFromEvent(event *gitlabwebhooks.PushEvent) (api.RepoName, error) {
 	if event == nil {
-		return api.RepoName(""), errors.New("nil PushEvent received")
+		return "", errors.New("nil PushEvent received")
 	}
 	parsed, err := url.Parse(event.Project.WebURL)
 	if err != nil {
 		return "", errors.Wrap(err, "parsing project URL")
 	}
-	return api.RepoName(parsed.Host + parsed.Path), nil
+	return api.RepoName(parsed.Hostname() + parsed.Path), nil
 }

enterprise/cmd/frontend/internal/repos/webhooks/init.go

@@ -22,6 +22,8 @@ func Init(
 ) error {
 	enterpriseServices.GitHubSyncWebhook = NewGitHubHandler()
 	enterpriseServices.GitLabSyncWebhook = NewGitLabHandler()
+	enterpriseServices.BitbucketServerSyncWebhook = NewBitbucketServerHandler()
+
 	enterpriseServices.WebhooksResolver = resolvers.NewWebhooksResolver(db)
 	return nil
 }

enterprise/cmd/frontend/internal/repos/webhooks/testdata/bitbucket-server-push.json

@@ -0,0 +1,75 @@
+{
+  "eventKey": "repo:refs_changed",
+  "date": "2022-12-21T14:49:22+0000",
+  "actor": {
+    "name": "milton",
+    "emailAddress": "dev@sourcegraph.com",
+    "id": 1,
+    "displayName": "milton woof",
+    "active": true,
+    "slug": "milton",
+    "type": "NORMAL",
+    "links": {
+      "self": [
+        {
+          "href": "https://bitbucket.sgdev.org/users/milton"
+        }
+      ]
+    }
+  },
+  "repository": {
+    "slug": "test-2020-06-01",
+    "id": 10092,
+    "name": "test-2020-06-01",
+    "hierarchyId": "f6bb9c710c413bb2c5f0",
+    "scmId": "git",
+    "state": "AVAILABLE",
+    "statusMessage": "Available",
+    "forkable": true,
+    "project": {
+      "key": "PRIVATE",
+      "id": 3,
+      "name": "Private",
+      "public": false,
+      "type": "NORMAL",
+      "links": {
+        "self": [
+          {
+            "href": "https://bitbucket.sgdev.org/projects/PRIVATE"
+          }
+        ]
+      }
+    },
+    "public": false,
+    "links": {
+      "clone": [
+        {
+          "href": "ssh://git@bitbucket.sgdev.org:7999/private/test-2020-06-01.git",
+          "name": "ssh"
+        },
+        {
+          "href": "https://bitbucket.sgdev.org/scm/private/test-2020-06-01.git",
+          "name": "http"
+        }
+      ],
+      "self": [
+        {
+          "href": "https://bitbucket.sgdev.org/projects/PRIVATE/repos/test-2020-06-01/browse"
+        }
+      ]
+    }
+  },
+  "changes": [
+    {
+      "ref": {
+        "id": "refs/heads/rs/test-branch",
+        "displayId": "rs/test-branch",
+        "type": "BRANCH"
+      },
+      "refId": "refs/heads/rs/test-branch",
+      "fromHash": "e6dcba6b52fd349279a38d9fa25db894467de609",
+      "toHash": "4cd5ee68a039032fe8a613879fd73b242592ea6a",
+      "type": "UPDATE"
+    }
+  ]
+}

internal/extsvc/bitbucketserver/events.go

@@ -19,8 +19,11 @@ func WebhookEventType(r *http.Request) string {
 
 func ParseWebhookEvent(eventType string, payload []byte) (e any, err error) {
 	switch eventType {
-	case "ping":
+	case "ping", "diagnostics:ping":
 		return PingEvent{}, nil
+	case "repo:refs_changed":
+		e = &PushEvent{}
+		return e, json.Unmarshal(payload, e)
 	case "repo:build_status":
 		e = &BuildStatusEvent{}
 		return e, json.Unmarshal(payload, e)
@@ -37,6 +40,10 @@ func ParseWebhookEvent(eventType string, payload []byte) (e any, err error) {
 
 type PingEvent struct{}
 
+type PushEvent struct {
+	Repository Repo `json:"repository"`
+}
+
 type PullRequestActivityEvent struct {
 	Date        time.Time      `json:"date"`
 	Actor       User           `json:"actor"`