artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 15:31:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add GitHub Auth config options to manage internal repos (#56677)

Browse Source
This commit is contained in:
Petri-Johan Last 2023-10-04 16:43:40 +02:00 committed by GitHub
parent 223839f3f3
commit d271395d10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 2879 additions and 3530 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -17,7 +17,7 @@ All notable changes to Sourcegraph are documented in this file.
### Added
-
- Added two new authorization configuration options to GitHub code host connections: "markInternalReposAsPublic" and "syncInternalRepoPermissions". Setting "markInternalReposAsPublic" to true is useful for organizations that have a large amount of internal repositories that everyone on the instance should be able to access, removing the need to have permissions to access these repositories. Setting "syncInternalRepoPermissions" to true adds an additional step to user permission syncs that explicitly checks for internal repositories. However, this could lead to longer user permission sync times. [#56677](https://github.com/sourcegraph/sourcegraph/pull/56677)
### Changed

2
cmd/frontend/internal/auth/githuboauth/session.go
View File

@ -232,7 +232,7 @@ func (s *sessionIssuerHelper) verifyUserOrgs(ctx context.Context, ghClient *gith
var err error
page := 1
for hasNextPage {
userOrgs, hasNextPage, _, err = ghClient.GetAuthenticatedUserOrgsForPage(ctx, page)
userOrgs, hasNextPage, _, err = ghClient.GetAuthenticatedUserOrgs(ctx, page)
if err != nil {
log15.Warn("Could not get GitHub authenticated user organizations", "error", err)

1
cmd/repo-updater/internal/authz/BUILD.bazel
View File

@ -69,6 +69,7 @@ go_test(
"//internal/httptestutil",
"//internal/observation",
"//internal/ratelimit",
"//internal/rcache",
"//internal/repos",
"//internal/timeutil",
"//internal/types",

579
cmd/repo-updater/internal/authz/integration_test.go
View File

@ -29,6 +29,7 @@ import (
extsvcGitHub "github.com/sourcegraph/sourcegraph/internal/extsvc/github"
"github.com/sourcegraph/sourcegraph/internal/httptestutil"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
"github.com/sourcegraph/sourcegraph/internal/rcache"
"github.com/sourcegraph/sourcegraph/internal/repos"
"github.com/sourcegraph/sourcegraph/internal/timeutil"
"github.com/sourcegraph/sourcegraph/internal/types"
@ -43,6 +44,62 @@ func update(name string) bool {
return regexp.MustCompile(*updateRegex).MatchString(name)
}
func assertGitHubUserPermissions(t *testing.T, ctx context.Context, userID int32, ghURL string, syncer *PermsSyncer, permsStore database.PermsStore, wantIDs []int32) {
t.Helper()
_, providerStates, err := syncer.syncUserPerms(ctx, userID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: ghURL,
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchUserPerms",
}}, providerStates)
p, err := permsStore.LoadUserPermissions(ctx, userID)
if err != nil {
t.Fatal(err)
}
gotIDs := make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
}
func assertGitHubRepoPermissions(t *testing.T, ctx context.Context, repoID api.RepoID, userID int32, ghURL string, syncer *PermsSyncer, permsStore database.PermsStore, wantIDs []int32) {
t.Helper()
_, providerStates, err := syncer.syncRepoPerms(ctx, repoID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: ghURL,
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchRepoPerms",
}}, providerStates)
p, err := permsStore.LoadUserPermissions(ctx, userID)
if err != nil {
t.Fatal(err)
}
gotIDs := make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
}
// NOTE: To update VCR for these tests, please use the token of "sourcegraph-vcr"
// for GITHUB_TOKEN, which can be found in 1Password.
//
@ -70,41 +127,65 @@ func TestIntegration_GitHubPermissions(t *testing.T) {
CreatedAt: timeutil.Now(),
Config: extsvc.NewUnencryptedConfig(`{"url": "https://github.com", "authorization": {}, "token": "abc", "repos": ["owner/name"]}`),
}
uri, err := url.Parse("https://github.com")
uri, err := url.Parse("https://github.com/")
if err != nil {
t.Fatal(err)
}
newUser := database.NewUser{
Email: "sourcegraph-vcr-bob@sourcegraph.com",
Username: "sourcegraph-vcr-bob",
EmailIsVerified: true,
}
testDB := database.NewDB(logger, dbtest.NewDB(logger, t))
ctx := actor.WithInternalActor(context.Background())
reposStore := repos.NewStore(logtest.Scoped(t), testDB)
err = reposStore.ExternalServiceStore().Upsert(ctx, &svc)
if err != nil {
t.Fatal(err)
}
repo := types.Repo{
Name: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
Private: true,
URI: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
ExternalRepo: api.ExternalRepoSpec{
ID: "MDEwOlJlcG9zaXRvcnkzOTk4OTQyODY=",
ServiceType: extsvc.TypeGitHub,
ServiceID: "https://github.com/",
},
Sources: map[string]*types.SourceInfo{
svc.URN(): {
ID: svc.URN(),
},
},
}
err = reposStore.RepoStore().Create(ctx, &repo)
if err != nil {
t.Fatal(err)
}
authData := json.RawMessage(fmt.Sprintf(`{"access_token": "%s"}`, token))
user, err := testDB.UserExternalAccounts().CreateUserAndSave(ctx, newUser, spec, extsvc.AccountData{
AuthData: extsvc.NewUnencryptedData(authData),
})
if err != nil {
t.Fatal(err)
}
permsStore := database.Perms(logger, testDB, timeutil.Now)
syncer := NewPermsSyncer(logger, testDB, reposStore, permsStore, timeutil.Now)
// This integration tests performs a repository-centric permissions syncing against
// https://github.com, then check if permissions are correctly granted for the test
// user "sourcegraph-vcr-bob", who is a outside collaborator of the repository
// "sourcegraph-vcr-repos/private-org-repo-1".
t.Run("repo-centric", func(t *testing.T) {
newUser := database.NewUser{
Email: "sourcegraph-vcr-bob@sourcegraph.com",
Username: "sourcegraph-vcr-bob",
EmailIsVerified: true,
}
t.Run("no-groups", func(t *testing.T) {
name := t.Name()
cf, save := httptestutil.NewGitHubRecorderFactory(t, update(name), name)
defer save()
doer, err := cf.Doer()
if err != nil {
t.Fatal(err)
}
cli := extsvcGitHub.NewV3Client(logtest.Scoped(t), svc.URN(), uri, &auth.OAuthBearerToken{Token: token}, doer)
testDB := database.NewDB(logger, dbtest.NewDB(logger, t))
ctx := actor.WithInternalActor(context.Background())
reposStore := repos.NewStore(logtest.Scoped(t), testDB)
err = reposStore.ExternalServiceStore().Upsert(ctx, &svc)
if err != nil {
t.Fatal(err)
}
cli := newTestRecorderClient(t, svc.URN(), uri, token)
provider := authzGitHub.NewProvider(svc.URN(), authzGitHub.ProviderOptions{
GitHubClient: cli,
@ -117,80 +198,11 @@ func TestIntegration_GitHubPermissions(t *testing.T) {
authz.SetProviders(false, []authz.Provider{provider})
defer authz.SetProviders(true, nil)
repo := types.Repo{
Name: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
Private: true,
URI: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
ExternalRepo: api.ExternalRepoSpec{
ID: "MDEwOlJlcG9zaXRvcnkzOTk4OTQyODY=",
ServiceType: extsvc.TypeGitHub,
ServiceID: "https://github.com/",
},
Sources: map[string]*types.SourceInfo{
svc.URN(): {
ID: svc.URN(),
},
},
}
err = reposStore.RepoStore().Create(ctx, &repo)
if err != nil {
t.Fatal(err)
}
user, err := testDB.UserExternalAccounts().CreateUserAndSave(ctx, newUser, spec, extsvc.AccountData{})
if err != nil {
t.Fatal(err)
}
permsStore := database.Perms(logger, testDB, timeutil.Now)
syncer := NewPermsSyncer(logger, testDB, reposStore, permsStore, timeutil.Now)
_, providerStates, err := syncer.syncRepoPerms(ctx, repo.ID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: "https://github.com/",
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchRepoPerms",
}}, providerStates)
p, err := permsStore.LoadUserPermissions(ctx, user.ID)
if err != nil {
t.Fatal(err)
}
gotIDs := make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
wantIDs := []int32{1}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
assertGitHubRepoPermissions(t, ctx, repo.ID, user.ID, uri.String(), syncer, permsStore, []int32{1})
})
t.Run("groups-enabled", func(t *testing.T) {
name := t.Name()
cf, save := httptestutil.NewGitHubRecorderFactory(t, update(name), name)
defer save()
doer, err := cf.Doer()
if err != nil {
t.Fatal(err)
}
cli := extsvcGitHub.NewV3Client(logtest.Scoped(t), svc.URN(), uri, &auth.OAuthBearerToken{Token: token}, doer)
testDB := database.NewDB(logger, dbtest.NewDB(logger, t))
ctx := actor.WithInternalActor(context.Background())
reposStore := repos.NewStore(logtest.Scoped(t), testDB)
err = reposStore.ExternalServiceStore().Upsert(ctx, &svc)
if err != nil {
t.Fatal(err)
}
cli := newTestRecorderClient(t, svc.URN(), uri, token)
provider := authzGitHub.NewProvider(svc.URN(), authzGitHub.ProviderOptions{
GitHubClient: cli,
@ -203,115 +215,16 @@ func TestIntegration_GitHubPermissions(t *testing.T) {
authz.SetProviders(false, []authz.Provider{provider})
defer authz.SetProviders(true, nil)
repo := types.Repo{
Name: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
Private: true,
URI: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
ExternalRepo: api.ExternalRepoSpec{
ID: "MDEwOlJlcG9zaXRvcnkzOTk4OTQyODY=",
ServiceType: extsvc.TypeGitHub,
ServiceID: "https://github.com/",
},
Sources: map[string]*types.SourceInfo{
svc.URN(): {
ID: svc.URN(),
},
},
}
err = reposStore.RepoStore().Create(ctx, &repo)
if err != nil {
t.Fatal(err)
}
user, err := testDB.UserExternalAccounts().CreateUserAndSave(ctx, newUser, spec, extsvc.AccountData{})
if err != nil {
t.Fatal(err)
}
permsStore := database.Perms(logger, testDB, timeutil.Now)
syncer := NewPermsSyncer(logger, testDB, reposStore, permsStore, timeutil.Now)
_, providerStates, err := syncer.syncRepoPerms(ctx, repo.ID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: "https://github.com/",
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchRepoPerms",
}}, providerStates)
p, err := permsStore.LoadUserPermissions(ctx, user.ID)
if err != nil {
t.Fatal(err)
}
gotIDs := make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
wantIDs := []int32{1}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
// sync again and check
_, providerStates, err = syncer.syncRepoPerms(ctx, repo.ID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: "https://github.com/",
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchRepoPerms",
}}, providerStates)
p, err = permsStore.LoadUserPermissions(ctx, user.ID)
if err != nil {
t.Fatal(err)
}
gotIDs = make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
assertGitHubRepoPermissions(t, ctx, repo.ID, user.ID, uri.String(), syncer, permsStore, []int32{1})
})
})
// This integration tests performs a repository-centric permissions syncing against
// This integration tests performs a user-centric permissions syncing against
// https://github.com, then check if permissions are correctly granted for the test
// user "sourcegraph-vcr", who is a collaborator of "sourcegraph-vcr-repos/private-org-repo-1".
// user "sourcegraph-vcr-bob", who is a collaborator of "sourcegraph-vcr-repos/private-org-repo-1".
t.Run("user-centric", func(t *testing.T) {
newUser := database.NewUser{
Email: "sourcegraph-vcr@sourcegraph.com",
Username: "sourcegraph-vcr",
EmailIsVerified: true,
}
t.Run("no-groups", func(t *testing.T) {
name := t.Name()
cf, save := httptestutil.NewGitHubRecorderFactory(t, update(name), name)
defer save()
doer, err := cf.Doer()
if err != nil {
t.Fatal(err)
}
cli := extsvcGitHub.NewV3Client(logtest.Scoped(t), svc.URN(), uri, &auth.OAuthBearerToken{Token: token}, doer)
testDB := database.NewDB(logger, dbtest.NewDB(logger, t))
ctx := actor.WithInternalActor(context.Background())
reposStore := repos.NewStore(logtest.Scoped(t), testDB)
err = reposStore.ExternalServiceStore().Upsert(ctx, &svc)
if err != nil {
t.Fatal(err)
}
cli := newTestRecorderClient(t, svc.URN(), uri, token)
provider := authzGitHub.NewProvider(svc.URN(), authzGitHub.ProviderOptions{
GitHubClient: cli,
@ -324,83 +237,11 @@ func TestIntegration_GitHubPermissions(t *testing.T) {
authz.SetProviders(false, []authz.Provider{provider})
defer authz.SetProviders(true, nil)
repo := types.Repo{
Name: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
Private: true,
URI: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
ExternalRepo: api.ExternalRepoSpec{
ID: "MDEwOlJlcG9zaXRvcnkzOTk4OTQyODY=",
ServiceType: extsvc.TypeGitHub,
ServiceID: "https://github.com/",
},
Sources: map[string]*types.SourceInfo{
svc.URN(): {
ID: svc.URN(),
},
},
}
err = reposStore.RepoStore().Create(ctx, &repo)
if err != nil {
t.Fatal(err)
}
authData := json.RawMessage(fmt.Sprintf(`{"access_token": "%s"}`, token))
user, err := testDB.UserExternalAccounts().CreateUserAndSave(ctx, newUser, spec, extsvc.AccountData{
AuthData: extsvc.NewUnencryptedData(authData),
})
if err != nil {
t.Fatal(err)
}
permsStore := database.Perms(logger, testDB, timeutil.Now)
syncer := NewPermsSyncer(logger, testDB, reposStore, permsStore, timeutil.Now)
_, providerStates, err := syncer.syncUserPerms(ctx, user.ID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: "https://github.com/",
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchUserPerms",
}}, providerStates)
p, err := permsStore.LoadUserPermissions(ctx, user.ID)
if err != nil {
t.Fatal(err)
}
gotIDs := make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
wantIDs := []int32{1}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
assertGitHubUserPermissions(t, ctx, user.ID, uri.String(), syncer, permsStore, []int32{1})
})
t.Run("groups-enabled", func(t *testing.T) {
name := t.Name()
cf, save := httptestutil.NewGitHubRecorderFactory(t, update(name), name)
defer save()
doer, err := cf.Doer()
if err != nil {
t.Fatal(err)
}
cli := extsvcGitHub.NewV3Client(logtest.Scoped(t), svc.URN(), uri, &auth.OAuthBearerToken{Token: token}, doer)
testDB := database.NewDB(logger, dbtest.NewDB(logger, t))
ctx := actor.WithInternalActor(context.Background())
reposStore := repos.NewStore(logtest.Scoped(t), testDB)
err = reposStore.ExternalServiceStore().Upsert(ctx, &svc)
if err != nil {
t.Fatal(err)
}
cli := newTestRecorderClient(t, svc.URN(), uri, token)
provider := authzGitHub.NewProvider(svc.URN(), authzGitHub.ProviderOptions{
GitHubClient: cli,
@ -413,90 +254,122 @@ func TestIntegration_GitHubPermissions(t *testing.T) {
authz.SetProviders(false, []authz.Provider{provider})
defer authz.SetProviders(true, nil)
repo := types.Repo{
Name: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
Private: true,
URI: "github.com/sourcegraph-vcr-repos/private-org-repo-1",
ExternalRepo: api.ExternalRepoSpec{
ID: "MDEwOlJlcG9zaXRvcnkzOTk4OTQyODY=",
ServiceType: extsvc.TypeGitHub,
ServiceID: "https://github.com/",
},
Sources: map[string]*types.SourceInfo{
svc.URN(): {
ID: svc.URN(),
},
},
}
err = reposStore.RepoStore().Create(ctx, &repo)
if err != nil {
t.Fatal(err)
}
authData := json.RawMessage(fmt.Sprintf(`{"access_token": "%s"}`, token))
user, err := testDB.UserExternalAccounts().CreateUserAndSave(ctx, newUser, spec, extsvc.AccountData{
AuthData: extsvc.NewUnencryptedData(authData),
})
if err != nil {
t.Fatal(err)
}
permsStore := database.Perms(logger, testDB, timeutil.Now)
syncer := NewPermsSyncer(logger, testDB, reposStore, permsStore, timeutil.Now)
_, providerStates, err := syncer.syncUserPerms(ctx, user.ID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: "https://github.com/",
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchUserPerms",
}}, providerStates)
p, err := permsStore.LoadUserPermissions(ctx, user.ID)
if err != nil {
t.Fatal(err)
}
gotIDs := make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
wantIDs := []int32{1}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
// sync again and check
_, providerStates, err = syncer.syncUserPerms(ctx, user.ID, false, authz.FetchPermsOptions{})
if err != nil {
t.Fatal(err)
}
assert.Equal(t, database.CodeHostStatusesSet{{
ProviderID: "https://github.com/",
ProviderType: "github",
Status: database.CodeHostStatusSuccess,
Message: "FetchUserPerms",
}}, providerStates)
p, err = permsStore.LoadUserPermissions(ctx, user.ID)
if err != nil {
t.Fatal(err)
}
gotIDs = make([]int32, len(p))
for i, perm := range p {
gotIDs[i] = perm.RepoID
}
if diff := cmp.Diff(wantIDs, gotIDs); diff != "" {
t.Fatalf("IDs mismatch (-want +got):\n%s", diff)
}
assertGitHubUserPermissions(t, ctx, user.ID, uri.String(), syncer, permsStore, []int32{1})
})
})
}
func newTestRecorderClient(t *testing.T, urn string, apiURL *url.URL, token string) *extsvcGitHub.V3Client {
name := t.Name()
cf, save := httptestutil.NewGitHubRecorderFactory(t, update(name), name)
t.Cleanup(save)
doer, err := cf.Doer()
if err != nil {
t.Fatal(err)
}
cli := extsvcGitHub.NewV3Client(logtest.Scoped(t), urn, apiURL, &auth.OAuthBearerToken{Token: token}, doer)
return cli
}
// TestIntegration_GitHubInternalRepositories asserts that internal repositories of
// organizations that a user belongs to are fetched alongside user permission
// syncs.
//
// The test setup requires a user that belongs to an organization with internal repos.
// It is kept separate from the other integration test since it connects to
// ghe.sgdev.org instead of github.com
func TestIntegration_GitHubInternalRepositories(t *testing.T) {
if testing.Short() {
t.Skip()
}
github.SetupForTest(t)
ratelimit.SetupForTest(t)
rcache.SetupForTest(t)
logger := logtest.Scoped(t)
token := os.Getenv("GITHUB_TOKEN")
spec := extsvc.AccountSpec{
ServiceType: extsvc.TypeGitHub,
ServiceID: "https://ghe.sgdev.org/",
AccountID: "3",
}
svc := types.ExternalService{
Kind: extsvc.KindGitHub,
CreatedAt: timeutil.Now(),
Config: extsvc.NewUnencryptedConfig(`{"url": "https://ghe.sgdev.org/", "authorization": {}, "token": "abc", "repos": ["owner/name"]}`),
}
uri, err := url.Parse("https://ghe.sgdev.org/")
if err != nil {
t.Fatal(err)
}
apiURI, _ := github.APIRoot(uri)
cli := newTestRecorderClient(t, uri.String(), apiURI, token)
testDB := database.NewDB(logger, dbtest.NewDB(logger, t))
ctx := actor.WithInternalActor(context.Background())
reposStore := repos.NewStore(logtest.Scoped(t), testDB)
err = reposStore.ExternalServiceStore().Upsert(ctx, &svc)
if err != nil {
t.Fatal(err)
}
provider := authzGitHub.NewProvider(svc.URN(), authzGitHub.ProviderOptions{
GitHubClient: cli,
GitHubURL: uri,
BaseAuther: &auth.OAuthBearerToken{Token: token},
GroupsCacheTTL: -1, // disable groups caching
DB: testDB,
})
authz.SetProviders(false, []authz.Provider{provider})
defer authz.SetProviders(true, nil)
repo := types.Repo{
Name: "ghe.sgdev.org/sourcegraph/sourcegraph_internal_repo",
Private: true,
URI: "ghe.sgdev.org/sourcegraph/sourcegraph_internal_repo",
ExternalRepo: api.ExternalRepoSpec{
ID: "MDEwOlJlcG9zaXRvcnkxMDU3MDMy",
ServiceType: extsvc.TypeGitHub,
ServiceID: "https://ghe.sgdev.org/",
},
Sources: map[string]*types.SourceInfo{
svc.URN(): {
ID: svc.URN(),
},
},
}
err = reposStore.RepoStore().Create(ctx, &repo)
if err != nil {
t.Fatal(err)
}
newUser := database.NewUser{
Email: "sourcegraph-vcr@sourcegraph.com",
Username: "sourcegraph-vcr",
EmailIsVerified: true,
}
authData := json.RawMessage(fmt.Sprintf(`{"access_token": "%s"}`, token))
user, err := testDB.UserExternalAccounts().CreateUserAndSave(ctx, newUser, spec, extsvc.AccountData{
AuthData: extsvc.NewUnencryptedData(authData),
})
if err != nil {
t.Fatal(err)
}
permsStore := database.Perms(logger, testDB, timeutil.Now)
syncer := NewPermsSyncer(logger, testDB, reposStore, permsStore, timeutil.Now)
assertGitHubUserPermissions(t, ctx, user.ID, uri.String(), syncer, permsStore, []int32{1})
}
func TestIntegration_GitLabPermissions(t *testing.T) {
if testing.Short() {
t.Skip()

189
cmd/repo-updater/internal/authz/testdata/vcr/TestIntegration_GitHubInternalRepositories.yaml vendored Normal file
View File

File diff suppressed because one or more lines are too long

1400
cmd/repo-updater/internal/authz/testdata/vcr/TestIntegration_GitHubPermissions/repo-centric/groups-enabled.yaml vendored
View File

File diff suppressed because it is too large Load Diff

109
cmd/repo-updater/internal/authz/testdata/vcr/TestIntegration_GitHubPermissions/repo-centric/no-groups.yaml vendored
View File

@ -8,20 +8,22 @@ interactions:
Accept:
- application/vnd.github.jean-grey-preview+json,application/vnd.github.mercy-preview+json
- application/vnd.github.machine-man-preview+json
Cache-Control:
- max-age=0
Content-Type:
- application/json; charset=utf-8
url: https://api.github.com/repos/sourcegraph-vcr-repos/private-org-repo-1/collaborators?page=1&per_page=100
method: GET
response:
body: '[{"login":"sourcegraph-vcr","id":63290851,"node_id":"MDQ6VXNlcjYzMjkwODUx","avatar_url":"https://avatars.githubusercontent.com/u/63290851?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr","html_url":"https://github.com/sourcegraph-vcr","followers_url":"https://api.github.com/users/sourcegraph-vcr/followers","following_url":"https://api.github.com/users/sourcegraph-vcr/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr/repos","events_url":"https://api.github.com/users/sourcegraph-vcr/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr/received_events","type":"User","site_admin":false,"permissions":{"admin":true,"maintain":true,"push":true,"triage":true,"pull":true}},{"login":"sourcegraph-vcr-amy","id":66464773,"node_id":"MDQ6VXNlcjY2NDY0Nzcz","avatar_url":"https://avatars.githubusercontent.com/u/66464773?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr-amy","html_url":"https://github.com/sourcegraph-vcr-amy","followers_url":"https://api.github.com/users/sourcegraph-vcr-amy/followers","following_url":"https://api.github.com/users/sourcegraph-vcr-amy/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr-amy/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr-amy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr-amy/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr-amy/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr-amy/repos","events_url":"https://api.github.com/users/sourcegraph-vcr-amy/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr-amy/received_events","type":"User","site_admin":false,"permissions":{"admin":false,"maintain":false,"push":false,"triage":false,"pull":true}},{"login":"sourcegraph-vcr-bob","id":66464926,"node_id":"MDQ6VXNlcjY2NDY0OTI2","avatar_url":"https://avatars.githubusercontent.com/u/66464926?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr-bob","html_url":"https://github.com/sourcegraph-vcr-bob","followers_url":"https://api.github.com/users/sourcegraph-vcr-bob/followers","following_url":"https://api.github.com/users/sourcegraph-vcr-bob/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr-bob/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr-bob/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr-bob/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr-bob/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr-bob/repos","events_url":"https://api.github.com/users/sourcegraph-vcr-bob/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr-bob/received_events","type":"User","site_admin":false,"permissions":{"admin":false,"maintain":false,"push":false,"triage":false,"pull":true}},{"login":"sourcegraph-vcr-dave","id":89494884,"node_id":"MDQ6VXNlcjg5NDk0ODg0","avatar_url":"https://avatars.githubusercontent.com/u/89494884?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr-dave","html_url":"https://github.com/sourcegraph-vcr-dave","followers_url":"https://api.github.com/users/sourcegraph-vcr-dave/followers","following_url":"https://api.github.com/users/sourcegraph-vcr-dave/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr-dave/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr-dave/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr-dave/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr-dave/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr-dave/repos","events_url":"https://api.github.com/users/sourcegraph-vcr-dave/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr-dave/received_events","type":"User","site_admin":false,"permissions":{"admin":false,"maintain":false,"push":false,"triage":false,"pull":true}}]'
body: '[{"login":"sourcegraph-vcr","id":63290851,"node_id":"MDQ6VXNlcjYzMjkwODUx","avatar_url":"https://avatars.githubusercontent.com/u/63290851?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr","html_url":"https://github.com/sourcegraph-vcr","followers_url":"https://api.github.com/users/sourcegraph-vcr/followers","following_url":"https://api.github.com/users/sourcegraph-vcr/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr/repos","events_url":"https://api.github.com/users/sourcegraph-vcr/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr/received_events","type":"User","site_admin":false,"permissions":{"admin":true,"maintain":true,"push":true,"triage":true,"pull":true},"role_name":"admin"},{"login":"sourcegraph-vcr-amy","id":66464773,"node_id":"MDQ6VXNlcjY2NDY0Nzcz","avatar_url":"https://avatars.githubusercontent.com/u/66464773?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr-amy","html_url":"https://github.com/sourcegraph-vcr-amy","followers_url":"https://api.github.com/users/sourcegraph-vcr-amy/followers","following_url":"https://api.github.com/users/sourcegraph-vcr-amy/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr-amy/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr-amy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr-amy/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr-amy/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr-amy/repos","events_url":"https://api.github.com/users/sourcegraph-vcr-amy/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr-amy/received_events","type":"User","site_admin":false,"permissions":{"admin":false,"maintain":false,"push":false,"triage":false,"pull":true},"role_name":"read"},{"login":"sourcegraph-vcr-bob","id":66464926,"node_id":"MDQ6VXNlcjY2NDY0OTI2","avatar_url":"https://avatars.githubusercontent.com/u/66464926?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr-bob","html_url":"https://github.com/sourcegraph-vcr-bob","followers_url":"https://api.github.com/users/sourcegraph-vcr-bob/followers","following_url":"https://api.github.com/users/sourcegraph-vcr-bob/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr-bob/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr-bob/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr-bob/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr-bob/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr-bob/repos","events_url":"https://api.github.com/users/sourcegraph-vcr-bob/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr-bob/received_events","type":"User","site_admin":false,"permissions":{"admin":false,"maintain":false,"push":false,"triage":false,"pull":true},"role_name":"read"},{"login":"sourcegraph-vcr-dave","id":89494884,"node_id":"MDQ6VXNlcjg5NDk0ODg0","avatar_url":"https://avatars.githubusercontent.com/u/89494884?v=4","gravatar_id":"","url":"https://api.github.com/users/sourcegraph-vcr-dave","html_url":"https://github.com/sourcegraph-vcr-dave","followers_url":"https://api.github.com/users/sourcegraph-vcr-dave/followers","following_url":"https://api.github.com/users/sourcegraph-vcr-dave/following{/other_user}","gists_url":"https://api.github.com/users/sourcegraph-vcr-dave/gists{/gist_id}","starred_url":"https://api.github.com/users/sourcegraph-vcr-dave/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sourcegraph-vcr-dave/subscriptions","organizations_url":"https://api.github.com/users/sourcegraph-vcr-dave/orgs","repos_url":"https://api.github.com/users/sourcegraph-vcr-dave/repos","events_url":"https://api.github.com/users/sourcegraph-vcr-dave/events{/privacy}","received_events_url":"https://api.github.com/users/sourcegraph-vcr-dave/received_events","type":"User","site_admin":false,"permissions":{"admin":false,"maintain":false,"push":false,"triage":false,"pull":true},"role_name":"read"}]'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, Deprecation,
Sunset
X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
X-GitHub-Request-Id, Deprecation, Sunset
Cache-Control:
- private, max-age=60, s-maxage=60
Content-Security-Policy:
@ -29,11 +31,9 @@ interactions:
Content-Type:
- application/json; charset=utf-8
Date:
- Tue, 31 Aug 2021 18:24:28 GMT
- Mon, 18 Sep 2023 10:44:01 GMT
Etag:
- W/"6b5bb1cf038a223423216ecdf2d1802d5767b3d072b5b00d04b25a90f730e897"
Github-Authentication-Token-Expiration:
- 2021-09-25 19:00:13 UTC
- W/"a55be06f6ea3ec639475186c03eebc3cb10d340ee0e52c6fcd080136cd15131e"
Referrer-Policy:
- origin-when-cross-origin, strict-origin-when-cross-origin
Server:
@ -43,110 +43,29 @@ interactions:
Vary:
- Accept, Authorization, Cookie, X-GitHub-OTP
- Accept-Encoding, Accept, X-Requested-With
X-Accepted-Oauth-Scopes:
- ""
X-Content-Type-Options:
- nosniff
X-Frame-Options:
- deny
X-Github-Api-Version-Selected:
- "2022-11-28"
X-Github-Media-Type:
- github.v3; param=jean-grey-preview; format=json, github.mercy-preview; param=machine-man-preview;
format=json
X-Github-Request-Id:
- D32C:7D36:E39C2:2835AF:612E73DB
X-Oauth-Scopes:
- admin:enterprise, admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
admin:repo_hook, delete:packages, delete_repo, gist, notifications, repo,
user, workflow, write:discussion, write:packages
- E8BF:613F:7D5F9:98FD2:650829F1
X-Ratelimit-Limit:
- "5000"
X-Ratelimit-Remaining:
- "4978"
- "4999"
X-Ratelimit-Reset:
- "1630436467"
- "1695037441"
X-Ratelimit-Resource:
- core
X-Ratelimit-Used:
- "22"
X-Xss-Protection:
- "0"
status: 200 OK
code: 200
duration: ""
- request:
body: ""
form: {}
headers:
Accept:
- "1"
X-Varied-Accept:
- application/vnd.github.jean-grey-preview+json,application/vnd.github.mercy-preview+json
- application/vnd.github.machine-man-preview+json
Content-Type:
- application/json; charset=utf-8
url: https://api.github.com/repos/sourcegraph-vcr-repos/private-org-repo-1/collaborators?page=2&per_page=100
method: GET
response:
body: '[]'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, Deprecation,
Sunset
Cache-Control:
- private, max-age=60, s-maxage=60
Content-Length:
- "2"
Content-Security-Policy:
- default-src 'none'
Content-Type:
- application/json; charset=utf-8
Date:
- Tue, 31 Aug 2021 18:24:28 GMT
Etag:
- '"e83491e8e7daca72e1fb091193dc660500fdf27f893e8e8385f0857f97a96c21"'
Github-Authentication-Token-Expiration:
- 2021-09-25 19:00:13 UTC
Link:
- <https://api.github.com/repositories/263034151/collaborators?page=1&per_page=100>;
rel="prev", <https://api.github.com/repositories/263034151/collaborators?page=1&per_page=100>;
rel="last", <https://api.github.com/repositories/263034151/collaborators?page=1&per_page=100>;
rel="first"
Referrer-Policy:
- origin-when-cross-origin, strict-origin-when-cross-origin
Server:
- GitHub.com
Strict-Transport-Security:
- max-age=31536000; includeSubdomains; preload
Vary:
- Accept, Authorization, Cookie, X-GitHub-OTP
- Accept-Encoding, Accept, X-Requested-With
X-Accepted-Oauth-Scopes:
- ""
X-Content-Type-Options:
- nosniff
X-Frame-Options:
- deny
X-Github-Media-Type:
- github.v3; param=jean-grey-preview; format=json, github.mercy-preview; param=machine-man-preview;
format=json
X-Github-Request-Id:
- D32C:7D36:E39D0:2835DB:612E73DC
X-Oauth-Scopes:
- admin:enterprise, admin:gpg_key, admin:org, admin:org_hook, admin:public_key,
admin:repo_hook, delete:packages, delete_repo, gist, notifications, repo,
user, workflow, write:discussion, write:packages
X-Ratelimit-Limit:
- "5000"
X-Ratelimit-Remaining:
- "4977"
X-Ratelimit-Reset:
- "1630436467"
X-Ratelimit-Resource:
- core
X-Ratelimit-Used:
- "23"
X-Xss-Protection:
- "0"
status: 200 OK

2894
cmd/repo-updater/internal/authz/testdata/vcr/TestIntegration_GitHubPermissions/user-centric/groups-enabled.yaml vendored
View File

File diff suppressed because one or more lines are too long

598
cmd/repo-updater/internal/authz/testdata/vcr/TestIntegration_GitHubPermissions/user-centric/no-groups.yaml vendored
View File

File diff suppressed because one or more lines are too long

24
doc/admin/external_service/github.md
View File

@ -275,6 +275,30 @@ Repo-centric permission syncing is done by calling the [list repository collabor
> NOTE: It can take some time to complete full cycle of repository permissions sync if you have a large number of users or repositories. [See sync duration time](../permissions/syncing.md#sync-duration) for more information.
### Internal repositories
GitHub Enterprise has internal repositories in addition to the usual public and private repositories. Depending on how your organization structure is configured, you may want to make these internal repositories available to everyone on your Sourcegraph instance without relying on permission syncs. To mark all internal repositories as public, add the following field to the `authorization` field:
```json
{
// ...
"authorization": {
"markInternalReposAsPublic": true
}
}
```
If you would like internal repositories to remain private, but you're experiencing issues where user permission syncs aren't granting access to internal repositories, you can add the following field instead:
```json
{
// ...
"authorization": {
"syncInternalRepoPermissions": true
}
}
```
### Trigger permissions sync from GitHub webhooks
Follow the link to [configure webhooks for permissions for Github](../config/webhooks/incoming.md#user-permissions)

2
internal/authz/providers/github/BUILD.bazel
View File

@ -14,6 +14,7 @@ go_library(
deps = [
"//internal/authz",
"//internal/authz/types",
"//internal/collections",
"//internal/database",
"//internal/encryption/keyring",
"//internal/extsvc",
@ -58,5 +59,6 @@ go_test(
"@com_github_gregjones_httpcache//:httpcache",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@org_golang_x_exp//slices",
],
)

9
internal/authz/providers/github/authz.go
View File

@ -141,10 +141,11 @@ func newAuthzProvider(
ttl := time.Duration(c.Authorization.GroupsCacheTTL) * time.Hour
return NewProvider(c.GitHubConnection.URN, ProviderOptions{
GitHubURL: baseURL,
BaseAuther: auther,
GroupsCacheTTL: ttl,
DB: db,
GitHubURL: baseURL,
BaseAuther: auther,
GroupsCacheTTL: ttl,
DB: db,
SyncInternalRepoPermissions: (c.Authorization != nil) && c.Authorization.SyncInternalRepoPermissions,
}), nil
}

1
internal/authz/providers/github/client.go
View File

@ -37,6 +37,7 @@ type client interface {
GetAuthenticatedUserOrgsDetailsAndMembership(ctx context.Context, page int) (orgs []github.OrgDetailsAndMembership, hasNextPage bool, rateLimitCost int, err error)
GetAuthenticatedUserTeams(ctx context.Context, page int) (teams []*github.Team, hasNextPage bool, rateLimitCost int, err error)
GetAuthenticatedUserOrgs(ctx context.Context, page int) (orgs []*github.Org, hasNextPage bool, rateLimitCost int, err error)
GetOrganization(ctx context.Context, login string) (org *github.OrgDetails, err error)
GetRepository(ctx context.Context, owner, name string) (*github.Repository, error)

257
internal/authz/providers/github/github.go
View File

@ -12,6 +12,7 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/collections"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
"github.com/sourcegraph/sourcegraph/internal/extsvc/auth"
@ -30,6 +31,8 @@ type Provider struct {
// groupsCache may be nil if group caching is disabled (negative TTL)
groupsCache *cachedGroups
syncInternalRepoPermissions bool
// enableGithubInternalRepoVisibility is a feature flag to optionally enable a fix for
// internal repos on GithHub Enterprise. At the moment we do not handle internal repos
// explicitly and allow all org members to read it irrespective of repo permissions. We have
@ -50,6 +53,8 @@ type ProviderOptions struct {
GroupsCacheTTL time.Duration
IsApp bool
DB database.DB
SyncInternalRepoPermissions bool
}
func NewProvider(urn string, opts ProviderOptions) *Provider {
@ -77,7 +82,8 @@ func NewProvider(urn string, opts ProviderOptions) *Provider {
client: func() (client, error) {
return &ClientAdapter{V3Client: opts.GitHubClient}, nil
},
db: opts.DB,
db: opts.DB,
syncInternalRepoPermissions: opts.SyncInternalRepoPermissions,
}
}
@ -158,88 +164,115 @@ func (p *Provider) requiredAuthScopes() (requiredAuthScope, bool) {
}, true
}
// fetchUserPermsByToken fetches all the private repo ids that the token can access.
//
// This may return a partial result if an error is encountered, e.g. via rate limits.
func (p *Provider) fetchUserPermsByToken(ctx context.Context, accountID extsvc.AccountID, token *auth.OAuthBearerToken, opts authz.FetchPermsOptions) (*authz.ExternalUserPermissions, error) {
// 🚨 SECURITY: Use user token is required to only list repositories the user has access to.
logger := log.Scoped("fetchUserPermsByToken", "fetches all the private repo ids that the token can access.")
client, err := p.client()
if err != nil {
return nil, errors.Wrap(err, "get client")
}
client = client.WithAuthenticator(token)
// 100 matches the maximum page size, thus a good default to avoid multiple allocations
// when appending the first 100 results to the slice.
const repoSetSize = 100
var (
// perms tracks repos this user has access to
perms = &authz.ExternalUserPermissions{
Exacts: make([]extsvc.RepoID, 0, repoSetSize),
// getAllAuthenticatedUserOrgs gets all the orgs the authenticated user is a member of.
func getAllAuthenticatedUserOrgs(ctx context.Context, cli client) ([]*github.Org, error) {
var orgs []*github.Org
for page := 1; true; page++ {
select {
case <-ctx.Done():
return orgs, ctx.Err()
default:
}
// seenRepos helps prevent duplication if necessary for groupsCache. Left unset
// indicates it is unused.
seenRepos map[extsvc.RepoID]struct{}
// addRepoToUserPerms checks if the given repos are already tracked before adding
// it to perms for groupsCache, otherwise just adds directly
addRepoToUserPerms func(repos ...extsvc.RepoID)
// Repository affiliations to list for - groupsCache only lists for a subset. Left
// unset indicates all affiliations should be sync'd.
affiliations []github.RepositoryAffiliation
)
// If cache is disabled the code path is simpler, avoid allocating memory
if p.groupsCache == nil { // Groups cache is disabled
// addRepoToUserPerms just appends
addRepoToUserPerms = func(repos ...extsvc.RepoID) {
perms.Exacts = append(perms.Exacts, repos...)
}
} else { // Groups cache is enabled
// Instantiate map for deduplicating repos
seenRepos = make(map[extsvc.RepoID]struct{}, repoSetSize)
// addRepoToUserPerms checks for duplicates before appending
addRepoToUserPerms = func(repos ...extsvc.RepoID) {
for _, repo := range repos {
if _, exists := seenRepos[repo]; !exists {
seenRepos[repo] = struct{}{}
perms.Exacts = append(perms.Exacts, repo)
}
}
}
// We sync just a subset of direct affiliations - we let other permissions
// ('organization' affiliation) be sync'd by teams/orgs.
affiliations = []github.RepositoryAffiliation{github.AffiliationOwner, github.AffiliationCollaborator}
}
// Sync direct affiliations
hasNextPage := true
for page := 1; hasNextPage; page++ {
var err error
var repos []*github.Repository
repos, hasNextPage, _, err = client.ListAffiliatedRepositories(ctx, github.VisibilityPrivate, page, 100, affiliations...)
pageOrgs, hasNextPage, _, err := cli.GetAuthenticatedUserOrgs(ctx, page)
if err != nil {
return perms, errors.Wrap(err, "list repos for user")
// We return partial results
return orgs, errors.Wrap(err, "list orgs for authenticated user")
}
orgs = append(orgs, pageOrgs...)
for _, r := range repos {
addRepoToUserPerms(extsvc.RepoID(r.ID))
if !hasNextPage {
break
}
}
// We're done if groups caching is disabled or no accountID is available.
if p.groupsCache == nil || accountID == "" {
return perms, nil
return orgs, nil
}
// getAllInternalRepositoriesForOrg fetches all internal repositories for the given org.
func getAllInternalRepositoriesForOrg(ctx context.Context, cli client, orgLogin string) ([]*github.Repository, error) {
var repos []*github.Repository
for page := 1; true; page++ {
select {
case <-ctx.Done():
return repos, ctx.Err()
default:
}
reposPage, hasNextPage, _, err := cli.ListOrgRepositories(ctx, orgLogin, page, "internal")
if err != nil {
// We return partial results
return repos, errors.Wrap(err, "list internal repos for org")
}
repos = append(repos, reposPage...)
if !hasNextPage {
break
}
}
return repos, nil
}
// getAllAuthenticatedUserInternalRepositories returns all internal repositories that the authenticated
// user has access to across all orgs.
func getAllAuthenticatedUserInternalRepositories(ctx context.Context, cli client) ([]*github.Repository, error) {
var repos []*github.Repository
orgs, err := getAllAuthenticatedUserOrgs(ctx, cli)
if err != nil {
return nil, err
}
for _, org := range orgs {
select {
case <-ctx.Done():
return repos, ctx.Err()
default:
}
orgRepos, err := getAllInternalRepositoriesForOrg(ctx, cli, org.Login)
if err != nil {
return repos, err
}
repos = append(repos, orgRepos...)
}
return repos, nil
}
func getAllAuthenticatedUserAffiliatedRepositories(ctx context.Context, cli client, affiliations ...github.RepositoryAffiliation) ([]*github.Repository, error) {
var repos []*github.Repository
// Sync direct affiliations
for page := 1; true; page++ {
select {
case <-ctx.Done():
return repos, ctx.Err()
default:
}
reposPage, hasNextPage, _, err := cli.ListAffiliatedRepositories(ctx, github.VisibilityPrivate, page, 100, affiliations...)
if err != nil {
return repos, errors.Wrap(err, "list affiliated repos for user")
}
repos = append(repos, reposPage...)
if !hasNextPage {
break
}
}
return repos, nil
}
// fetchCachedAuthenticatedUserPerms fetches permissions for the authenticated user using cached group membership data.
func (p *Provider) fetchCachedAuthenticatedUserPerms(ctx context.Context, logger log.Logger, cli client, accountID extsvc.AccountID, opts authz.FetchPermsOptions) (collections.Set[extsvc.RepoID], error) {
userRepos := collections.NewSet[extsvc.RepoID]()
// If groupsCache is disabled, return early
if p.groupsCache == nil {
return userRepos, nil
}
// Now, we look for groups this user belongs to that give access to additional
// repositories.
groups, err := p.getUserAffiliatedGroups(ctx, client, opts)
groups, err := p.getUserAffiliatedGroups(ctx, cli, opts)
if err != nil {
return perms, errors.Wrap(err, "get groups affiliated with user")
return userRepos, errors.Wrap(err, "get groups affiliated with user")
}
// Get repos from groups, cached if possible.
@ -265,20 +298,20 @@ func (p *Provider) fetchUserPermsByToken(ctx context.Context, accountID extsvc.A
// because it is possible this cached group does not have any repositories, in
// which case it should have a non-nil length 0 slice of repositories.
if group.Repositories != nil {
addRepoToUserPerms(group.Repositories...)
userRepos.Add(group.Repositories...)
continue
}
// Perform full sync. Start with instantiating the repos slice.
group.Repositories = make([]extsvc.RepoID, 0, repoSetSize)
group.Repositories = make([]extsvc.RepoID, 0, 100)
isOrg := group.Team == ""
hasNextPage = true
hasNextPage := true
for page := 1; hasNextPage; page++ {
var repos []*github.Repository
if isOrg {
repos, hasNextPage, _, err = client.ListOrgRepositories(ctx, group.Org, page, "")
repos, hasNextPage, _, err = cli.ListOrgRepositories(ctx, group.Org, page, "")
} else {
repos, hasNextPage, _, err = client.ListTeamRepositories(ctx, group.Org, group.Team, page)
repos, hasNextPage, _, err = cli.ListTeamRepositories(ctx, group.Org, group.Team, page)
}
if github.IsNotFound(err) || github.HTTPErrorCode(err) == http.StatusForbidden {
// If we get a 403/404 here, something funky is going on and this is very
@ -295,15 +328,15 @@ func (p *Provider) fetchUserPermsByToken(ctx context.Context, accountID extsvc.A
// Add and return what we've found on this page but don't persist group
// to cache
for _, r := range repos {
addRepoToUserPerms(extsvc.RepoID(r.ID))
userRepos.Add(extsvc.RepoID(r.ID))
}
return perms, errors.Wrap(err, "list repos for group")
return userRepos, errors.Wrap(err, "list repos for group")
}
// Add results to both group (for persistence) and permissions for user
for _, r := range repos {
repoID := extsvc.RepoID(r.ID)
group.Repositories = append(group.Repositories, repoID)
addRepoToUserPerms(repoID)
userRepos.Add(repoID)
}
}
@ -313,7 +346,62 @@ func (p *Provider) fetchUserPermsByToken(ctx context.Context, accountID extsvc.A
}
}
return perms, nil
return userRepos, nil
}
// fetchAuthenticatedUserPerms fetches all the private repo ids that the authenticated client can access.
//
// This may return a partial result if an error is encountered, e.g. via rate limits.
func (p *Provider) fetchAuthenticatedUserPerms(ctx context.Context, cli client, accountID extsvc.AccountID, opts authz.FetchPermsOptions) (*authz.ExternalUserPermissions, error) {
// 🚨 SECURITY: Use user token is required to only list repositories the user has access to.
logger := log.Scoped("fetchUserPermsByToken", "fetches all the private repo ids that the token can access.")
// Repository affiliations to list for - groupsCache only lists for a subset. Left
// unset indicates all affiliations should be sync'd.
affiliations := []github.RepositoryAffiliation{}
if p.groupsCache != nil {
// We sync just a subset of direct affiliations - we let other permissions
// ('organization' affiliation) be sync'd by teams/orgs.
affiliations = []github.RepositoryAffiliation{github.AffiliationOwner, github.AffiliationCollaborator}
}
userRepos := collections.NewSet[extsvc.RepoID]()
// Sync direct affiliations
repos, err := getAllAuthenticatedUserAffiliatedRepositories(ctx, cli, affiliations...)
// May return partial results, so we add possible repos first
for _, r := range repos {
userRepos.Add(extsvc.RepoID(r.ID))
}
if err != nil {
return &authz.ExternalUserPermissions{Exacts: userRepos.Values()}, errors.Wrap(err, "list repos for user")
}
// If cache is disabled and syncInternalRepoPermissions is true, we also need to fetch a list of
// internal repositories for each org the user belongs to.
// If caching is enabled, then the internal repositories will be fetched as part of the group sync.
if p.groupsCache == nil && p.syncInternalRepoPermissions {
internalRepos, err := getAllAuthenticatedUserInternalRepositories(ctx, cli)
for _, r := range internalRepos {
userRepos.Add(extsvc.RepoID(r.ID))
}
if err != nil {
return &authz.ExternalUserPermissions{Exacts: userRepos.Values()}, err
}
}
if p.groupsCache != nil {
// If groups caching is enabled, we need to fetch cached repositories as well
groupsPerms, err := p.fetchCachedAuthenticatedUserPerms(ctx, logger, cli, accountID, opts)
userRepos = userRepos.Union(groupsPerms)
if err != nil {
return &authz.ExternalUserPermissions{Exacts: userRepos.Values()}, err
}
}
return &authz.ExternalUserPermissions{
Exacts: userRepos.Values(),
}, nil
}
// FetchUserPerms returns a list of repository IDs (on code host) that the given account
@ -347,7 +435,14 @@ func (p *Provider) FetchUserPerms(ctx context.Context, account *extsvc.Account,
NeedsRefreshBuffer: 5,
}
return p.fetchUserPermsByToken(ctx, extsvc.AccountID(account.AccountID), oauthToken, opts)
client, err := p.client()
if err != nil {
return nil, errors.Wrap(err, "get client")
}
client = client.WithAuthenticator(oauthToken)
return p.fetchAuthenticatedUserPerms(ctx, client, extsvc.AccountID(account.AccountID), opts)
}
// FetchRepoPerms returns a list of user IDs (on code host) who have read access to

13
internal/authz/providers/github/github_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/gregjones/httpcache"
"golang.org/x/exp/slices"
"github.com/sourcegraph/sourcegraph/internal/api"
"github.com/sourcegraph/sourcegraph/internal/authz"
@ -252,6 +253,8 @@ func TestProvider_FetchUserPerms(t *testing.T) {
"MDEwOlJlcG9zaXRvcnkyNDQ1MTc1MzY=",
"MDEwOlJlcG9zaXRvcnkyNDI2NTEwMDA=",
}
slices.Sort(wantRepoIDs)
slices.Sort(repoIDs.Exacts)
if diff := cmp.Diff(wantRepoIDs, repoIDs.Exacts); diff != "" {
t.Fatalf("RepoIDs mismatch (-want +got):\n%s", diff)
}
@ -286,6 +289,8 @@ func TestProvider_FetchUserPerms(t *testing.T) {
"MDEwOlJlcG9zaXRvcnkyNDQ1MTc1234=",
"MDEwOlJlcG9zaXRvcnkyNDI2NTE5678=",
}
slices.Sort(wantRepoIDs)
slices.Sort(repoIDs.Exacts)
if diff := cmp.Diff(wantRepoIDs, repoIDs.Exacts); diff != "" {
t.Fatalf("RepoIDs mismatch (-want +got):\n%s", diff)
}
@ -357,6 +362,8 @@ func TestProvider_FetchUserPerms(t *testing.T) {
"MDEwOlJlcG9zaXRvcnkyNDQ1nsteam1=",
"MDEwOlJlcG9zaXRvcnkyNDI2nsteam2=",
}
slices.Sort(wantRepoIDs)
slices.Sort(repoIDs.Exacts)
if diff := cmp.Diff(wantRepoIDs, repoIDs.Exacts); diff != "" {
t.Fatalf("RepoIDs mismatch (-want +got):\n%s", diff)
}
@ -409,6 +416,8 @@ func TestProvider_FetchUserPerms(t *testing.T) {
"MDEwOlJlcG9zaXRvcnkyNDQ1MTc1234=", // from ListOrgRepositories
"MDEwOlJlcG9zaXRvcnkyNDI2NTE5678=", // from ListOrgRepositories
}
slices.Sort(wantRepoIDs)
slices.Sort(repoIDs.Exacts)
if diff := cmp.Diff(wantRepoIDs, repoIDs.Exacts); diff != "" {
t.Fatalf("RepoIDs mismatch (-want +got):\n%s", diff)
}
@ -461,6 +470,7 @@ func TestProvider_FetchUserPerms(t *testing.T) {
"MDEwOlJlcG9zaXRvcnkyNDI2NTE5678=",
"MDEwOlJlcG9zaXRvcnkyNDI2nsteam1=",
}
slices.Sort(wantRepoIDs)
// first call
t.Run("first call", func(t *testing.T) {
@ -475,6 +485,7 @@ func TestProvider_FetchUserPerms(t *testing.T) {
t.Fatalf("expected repos to be listed: callsToListOrgRepos=%d, callsToListTeamRepos=%d",
callsToListOrgRepos, callsToListTeamRepos)
}
slices.Sort(repoIDs.Exacts)
if diff := cmp.Diff(wantRepoIDs, repoIDs.Exacts); diff != "" {
t.Fatalf("RepoIDs mismatch (-want +got):\n%s", diff)
}
@ -495,6 +506,7 @@ func TestProvider_FetchUserPerms(t *testing.T) {
t.Fatalf("expected repos not to be listed: callsToListOrgRepos=%d, callsToListTeamRepos=%d",
callsToListOrgRepos, callsToListTeamRepos)
}
slices.Sort(repoIDs.Exacts)
if diff := cmp.Diff(wantRepoIDs, repoIDs.Exacts); diff != "" {
t.Fatalf("RepoIDs mismatch (-want +got):\n%s", diff)
}
@ -515,6 +527,7 @@ func TestProvider_FetchUserPerms(t *testing.T) {
t.Fatalf("expected repos to be listed: callsToListOrgRepos=%d, callsToListTeamRepos=%d",
callsToListOrgRepos, callsToListTeamRepos)
}
slices.Sort(repoIDs.Exacts)
if diff := cmp.Diff(wantRepoIDs, repoIDs.Exacts); diff != "" {
t.Fatalf("RepoIDs mismatch (-want +got):\n%s", diff)
}

133
internal/authz/providers/github/mocks_test.go generated
View File

@ -23,6 +23,9 @@ type MockClient struct {
// object controlling the behavior of the method
// GetAuthenticatedOAuthScopes.
GetAuthenticatedOAuthScopesFunc *ClientGetAuthenticatedOAuthScopesFunc
// GetAuthenticatedUserOrgsFunc is an instance of a mock function object
// controlling the behavior of the method GetAuthenticatedUserOrgs.
GetAuthenticatedUserOrgsFunc *ClientGetAuthenticatedUserOrgsFunc
// GetAuthenticatedUserOrgsDetailsAndMembershipFunc is an instance of a
// mock function object controlling the behavior of the method
// GetAuthenticatedUserOrgsDetailsAndMembership.
@ -77,6 +80,11 @@ func NewMockClient() *MockClient {
return
},
},
GetAuthenticatedUserOrgsFunc: &ClientGetAuthenticatedUserOrgsFunc{
defaultHook: func(context.Context, int) (r0 []*github.Org, r1 bool, r2 int, r3 error) {
return
},
},
GetAuthenticatedUserOrgsDetailsAndMembershipFunc: &ClientGetAuthenticatedUserOrgsDetailsAndMembershipFunc{
defaultHook: func(context.Context, int) (r0 []github.OrgDetailsAndMembership, r1 bool, r2 int, r3 error) {
return
@ -154,6 +162,11 @@ func NewStrictMockClient() *MockClient {
panic("unexpected invocation of MockClient.GetAuthenticatedOAuthScopes")
},
},
GetAuthenticatedUserOrgsFunc: &ClientGetAuthenticatedUserOrgsFunc{
defaultHook: func(context.Context, int) ([]*github.Org, bool, int, error) {
panic("unexpected invocation of MockClient.GetAuthenticatedUserOrgs")
},
},
GetAuthenticatedUserOrgsDetailsAndMembershipFunc: &ClientGetAuthenticatedUserOrgsDetailsAndMembershipFunc{
defaultHook: func(context.Context, int) ([]github.OrgDetailsAndMembership, bool, int, error) {
panic("unexpected invocation of MockClient.GetAuthenticatedUserOrgsDetailsAndMembership")
@ -227,6 +240,7 @@ func NewStrictMockClient() *MockClient {
// is redefined here as it is unexported in the source package.
type surrogateMockClient interface {
GetAuthenticatedOAuthScopes(context.Context) ([]string, error)
GetAuthenticatedUserOrgs(context.Context, int) ([]*github.Org, bool, int, error)
GetAuthenticatedUserOrgsDetailsAndMembership(context.Context, int) ([]github.OrgDetailsAndMembership, bool, int, error)
GetAuthenticatedUserTeams(context.Context, int) ([]*github.Team, bool, int, error)
GetOrganization(context.Context, string) (*github.OrgDetails, error)
@ -249,6 +263,9 @@ func NewMockClientFrom(i surrogateMockClient) *MockClient {
GetAuthenticatedOAuthScopesFunc: &ClientGetAuthenticatedOAuthScopesFunc{
defaultHook: i.GetAuthenticatedOAuthScopes,
},
GetAuthenticatedUserOrgsFunc: &ClientGetAuthenticatedUserOrgsFunc{
defaultHook: i.GetAuthenticatedUserOrgs,
},
GetAuthenticatedUserOrgsDetailsAndMembershipFunc: &ClientGetAuthenticatedUserOrgsDetailsAndMembershipFunc{
defaultHook: i.GetAuthenticatedUserOrgsDetailsAndMembership,
},
@ -399,6 +416,122 @@ func (c ClientGetAuthenticatedOAuthScopesFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1}
}
// ClientGetAuthenticatedUserOrgsFunc describes the behavior when the
// GetAuthenticatedUserOrgs method of the parent MockClient instance is
// invoked.
type ClientGetAuthenticatedUserOrgsFunc struct {
defaultHook func(context.Context, int) ([]*github.Org, bool, int, error)
hooks []func(context.Context, int) ([]*github.Org, bool, int, error)
history []ClientGetAuthenticatedUserOrgsFuncCall
mutex sync.Mutex
}
// GetAuthenticatedUserOrgs delegates to the next hook function in the queue
// and stores the parameter and result values of this invocation.
func (m *MockClient) GetAuthenticatedUserOrgs(v0 context.Context, v1 int) ([]*github.Org, bool, int, error) {
r0, r1, r2, r3 := m.GetAuthenticatedUserOrgsFunc.nextHook()(v0, v1)
m.GetAuthenticatedUserOrgsFunc.appendCall(ClientGetAuthenticatedUserOrgsFuncCall{v0, v1, r0, r1, r2, r3})
return r0, r1, r2, r3
}
// SetDefaultHook sets function that is called when the
// GetAuthenticatedUserOrgs method of the parent MockClient instance is
// invoked and the hook queue is empty.
func (f *ClientGetAuthenticatedUserOrgsFunc) SetDefaultHook(hook func(context.Context, int) ([]*github.Org, bool, int, error)) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// GetAuthenticatedUserOrgs method of the parent MockClient instance invokes
// the hook at the front of the queue and discards it. After the queue is
// empty, the default hook function is invoked for any future action.
func (f *ClientGetAuthenticatedUserOrgsFunc) PushHook(hook func(context.Context, int) ([]*github.Org, bool, int, error)) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *ClientGetAuthenticatedUserOrgsFunc) SetDefaultReturn(r0 []*github.Org, r1 bool, r2 int, r3 error) {
f.SetDefaultHook(func(context.Context, int) ([]*github.Org, bool, int, error) {
return r0, r1, r2, r3
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *ClientGetAuthenticatedUserOrgsFunc) PushReturn(r0 []*github.Org, r1 bool, r2 int, r3 error) {
f.PushHook(func(context.Context, int) ([]*github.Org, bool, int, error) {
return r0, r1, r2, r3
})
}
func (f *ClientGetAuthenticatedUserOrgsFunc) nextHook() func(context.Context, int) ([]*github.Org, bool, int, error) {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *ClientGetAuthenticatedUserOrgsFunc) appendCall(r0 ClientGetAuthenticatedUserOrgsFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of ClientGetAuthenticatedUserOrgsFuncCall
// objects describing the invocations of this function.
func (f *ClientGetAuthenticatedUserOrgsFunc) History() []ClientGetAuthenticatedUserOrgsFuncCall {
f.mutex.Lock()
history := make([]ClientGetAuthenticatedUserOrgsFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// ClientGetAuthenticatedUserOrgsFuncCall is an object that describes an
// invocation of method GetAuthenticatedUserOrgs on an instance of
// MockClient.
type ClientGetAuthenticatedUserOrgsFuncCall struct {
// Arg0 is the value of the 1st argument passed to this method
// invocation.
Arg0 context.Context
// Arg1 is the value of the 2nd argument passed to this method
// invocation.
Arg1 int
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 []*github.Org
// Result1 is the value of the 2nd result returned from this method
// invocation.
Result1 bool
// Result2 is the value of the 3rd result returned from this method
// invocation.
Result2 int
// Result3 is the value of the 4th result returned from this method
// invocation.
Result3 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c ClientGetAuthenticatedUserOrgsFuncCall) Args() []interface{} {
return []interface{}{c.Arg0, c.Arg1}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c ClientGetAuthenticatedUserOrgsFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1, c.Result2, c.Result3}
}
// ClientGetAuthenticatedUserOrgsDetailsAndMembershipFunc describes the
// behavior when the GetAuthenticatedUserOrgsDetailsAndMembership method of
// the parent MockClient instance is invoked.

3
internal/batches/sources/testdata/golden/GithubSource_GetFork_success_with_existing_changeset_and_existing_fork vendored
View File

@ -32,6 +32,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "ADMIN",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
},
@ -40,4 +41,4 @@
"IsFork": false
}
}
}
}

3
internal/batches/sources/testdata/golden/GithubSource_GetFork_success_with_existing_changeset_and_new_fork vendored
View File

@ -32,6 +32,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "ADMIN",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
},
@ -40,4 +41,4 @@
"IsFork": false
}
}
}
}

3
internal/batches/sources/testdata/golden/GithubSource_GetFork_success_with_new_changeset_and_existing_fork vendored
View File

@ -32,6 +32,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "ADMIN",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
},
@ -40,4 +41,4 @@
"IsFork": false
}
}
}
}

3
internal/batches/sources/testdata/golden/GithubSource_GetFork_success_with_new_changeset_and_new_fork vendored
View File

@ -32,6 +32,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "ADMIN",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
},
@ -40,4 +41,4 @@
"IsFork": false
}
}
}
}

7
internal/extsvc/github/common.go
View File

@ -1810,7 +1810,7 @@ type Repository struct {
// This is available for GitHub Enterprise Cloud and GitHub Enterprise Server 3.3.0+ and is used
// to identify if a repository is public or private or internal.
// https://developer.github.com/changes/2019-12-03-internal-visibility-changes/#repository-visibility-fields
Visibility Visibility `json:",omitempty"`
Visibility Visibility `json:"visibility,omitempty"`
// Parent is non-nil for forks and contains details of the parent repository.
Parent *ParentRepository `json:",omitempty"`
@ -1903,6 +1903,7 @@ func convertRestRepo(restRepo restRepository) *Repository {
StargazerCount: restRepo.Stars,
ForkCount: restRepo.Forks,
RepositoryTopics: RepositoryTopics{topics},
Visibility: Visibility(restRepo.Visibility),
}
if restRepo.Parent != nil {
@ -1912,10 +1913,6 @@ func convertRestRepo(restRepo restRepository) *Repository {
}
}
if conf.ExperimentalFeatures().EnableGithubInternalRepoVisibility {
repo.Visibility = Visibility(restRepo.Visibility)
}
return &repo
}

3
internal/extsvc/github/testdata/golden/ListRepositoriesForSearch vendored
View File

@ -11,8 +11,9 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "READ",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
}
}
]
]

4
internal/extsvc/github/testdata/golden/SearchRepos-Enterprise-narrow-query-enterprise vendored
View File

@ -21,9 +21,9 @@
}
]
},
"Visibility": "INTERNAL"
"visibility": "INTERNAL"
}
],
"TotalCount": 1,
"EndCursor": ""
}
}

3
internal/extsvc/github/testdata/golden/TestGetRepository/cached-response/first_run vendored
View File

@ -10,6 +10,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "WRITE",
"visibility": "public",
"RepositoryTopics": {
"Nodes": [
{
@ -46,4 +47,4 @@
},
"StargazerCount": 8662,
"ForkCount": 1116
}
}

3
internal/extsvc/github/testdata/golden/TestGetRepository/cached-response/second_run vendored
View File

@ -10,6 +10,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "WRITE",
"visibility": "public",
"RepositoryTopics": {
"Nodes": [
{
@ -46,4 +47,4 @@
},
"StargazerCount": 8662,
"ForkCount": 1116
}
}

3
internal/extsvc/github/testdata/golden/TestGetRepository/forked_repo vendored
View File

@ -10,6 +10,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "READ",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
},
@ -17,4 +18,4 @@
"NameWithOwner": "sourcegraph/sourcegraph",
"IsFork": false
}
}
}

3
internal/extsvc/github/testdata/golden/TestV3Client_Fork_success_sourcegraph-testing vendored
View File

@ -10,6 +10,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "ADMIN",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
},
@ -17,4 +18,4 @@
"NameWithOwner": "sourcegraph/automation-testing",
"IsFork": false
}
}
}

3
internal/extsvc/github/testdata/golden/TestV3Client_Fork_success_user vendored
View File

@ -10,6 +10,7 @@
"IsLocked": false,
"IsDisabled": false,
"ViewerPermission": "ADMIN",
"visibility": "public",
"RepositoryTopics": {
"Nodes": []
},
@ -17,4 +18,4 @@
"NameWithOwner": "sourcegraph/automation-testing",
"IsFork": false
}
}
}

6
internal/extsvc/github/v3.go
View File

@ -385,9 +385,9 @@ var MockGetAuthenticatedUserOrgs struct {
PagesMock map[int][]*Org
}
// GetAuthenticatedUserOrgsForPage returns given page of 100 organizations associated with the currently
// GetAuthenticatedUserOrgs returns given page of 100 organizations associated with the currently
// authenticated user.
func (c *V3Client) GetAuthenticatedUserOrgsForPage(ctx context.Context, page int) (
func (c *V3Client) GetAuthenticatedUserOrgs(ctx context.Context, page int) (
orgs []*Org,
hasNextPage bool,
rateLimitCost int,
@ -431,7 +431,7 @@ func (c *V3Client) GetAuthenticatedUserOrgsDetailsAndMembership(ctx context.Cont
rateLimitCost int,
err error,
) {
orgNames, hasNextPage, cost, err := c.GetAuthenticatedUserOrgsForPage(ctx, page)
orgNames, hasNextPage, cost, err := c.GetAuthenticatedUserOrgs(ctx, page)
if err != nil {
return
}

2
internal/extsvc/github/v3_test.go
View File

@ -297,7 +297,7 @@ func TestGetAuthenticatedUserOrgs(t *testing.T) {
defer save()
ctx := context.Background()
orgs, _, _, err := cli.GetAuthenticatedUserOrgsForPage(ctx, 1)
orgs, _, _, err := cli.GetAuthenticatedUserOrgs(ctx, 1)
if err != nil {
t.Fatal(err)
}

3
internal/extsvc/github/v4.go
View File

@ -19,7 +19,6 @@ import (
"github.com/sourcegraph/log"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/extsvc/auth"
"github.com/sourcegraph/sourcegraph/internal/httpcli"
"github.com/sourcegraph/sourcegraph/internal/ratelimit"
@ -617,7 +616,7 @@ fragment RepositoryFields on Repository {
conditionalGHEFields = append(conditionalGHEFields, "stargazerCount")
}
if conf.ExperimentalFeatures().EnableGithubInternalRepoVisibility && ghe330PlusOrDotComSemver.Check(version) {
if ghe330PlusOrDotComSemver.Check(version) {
conditionalGHEFields = append(conditionalGHEFields, "visibility")
}

28
internal/repos/github.go
View File

@ -50,6 +50,8 @@ type GitHubSource struct {
originalHostname string
logger log.Logger
markInternalReposAsPublic bool
}
var (
@ -188,15 +190,16 @@ func newGitHubSource(
}
return &GitHubSource{
svc: svc,
config: c,
exclude: exclude,
baseURL: baseURL,
githubDotCom: githubDotCom,
v3Client: v3Client,
v4Client: v4Client,
searchClient: searchClient,
originalHostname: originalHostname,
svc: svc,
config: c,
exclude: exclude,
baseURL: baseURL,
githubDotCom: githubDotCom,
v3Client: v3Client,
v4Client: v4Client,
searchClient: searchClient,
originalHostname: originalHostname,
markInternalReposAsPublic: (c.Authorization != nil) && c.Authorization.MarkInternalReposAsPublic,
logger: logger.With(
log.Object("GitHubSource",
log.Bool("excludeForks", excludeForks),
@ -363,7 +366,7 @@ func (s *GitHubSource) ListNamespaces(ctx context.Context, results chan SourceNa
return
}
var pageOrgs []*github.Org
pageOrgs, hasNextPage, _, err = s.v3Client.GetAuthenticatedUserOrgsForPage(ctx, page)
pageOrgs, hasNextPage, _, err = s.v3Client.GetAuthenticatedUserOrgs(ctx, page)
if err != nil {
results <- SourceNamespaceResult{Source: s, Err: err}
continue
@ -396,6 +399,11 @@ func (s *GitHubSource) makeRepo(r *github.Repository) *types.Repo {
// so we don't want to store it.
metadata.ViewerPermission = ""
metadata.Description = sanitizeToUTF8(metadata.Description)
if github.Visibility(strings.ToLower(string(r.Visibility))) == github.VisibilityInternal && s.markInternalReposAsPublic {
r.IsPrivate = false
}
return &types.Repo{
Name: reposource.GitHubRepoName(
s.config.RepositoryPathPattern,

39
internal/repos/github_test.go
View File

@ -299,13 +299,13 @@ func TestGithubSource_GetRepo_Enterprise(t *testing.T) {
testCases := []struct {
name string
nameWithOwner string
assert func(*testing.T, *types.Repo)
assert func(*testing.T, *types.Repo, bool)
err string
}{
{
name: "internal repo in github enterprise",
nameWithOwner: "admiring-austin-120/fluffy-enigma",
assert: func(t *testing.T, have *types.Repo) {
assert: func(t *testing.T, have *types.Repo, private bool) {
t.Helper()
want := &types.Repo{
@ -313,7 +313,7 @@ func TestGithubSource_GetRepo_Enterprise(t *testing.T) {
Description: "Internal repo used in tests in sourcegraph code.",
URI: "ghe.sgdev.org/admiring-austin-120/fluffy-enigma",
Stars: 0,
Private: true,
Private: private,
ExternalRepo: api.ExternalRepoSpec{
ID: "MDEwOlJlcG9zaXRvcnk0NDIyODU=",
ServiceType: "github",
@ -400,7 +400,38 @@ func TestGithubSource_GetRepo_Enterprise(t *testing.T) {
}
if tc.assert != nil {
tc.assert(t, repo)
tc.assert(t, repo, true)
}
// Configure external service to mark internal repositories as public
// and sync again
svc = &types.ExternalService{
Kind: extsvc.KindGitHub,
Config: extsvc.NewUnencryptedConfig(MarshalJSON(t, &schema.GitHubConnection{
Url: "https://ghe.sgdev.org",
Token: gheToken,
Authorization: &schema.GitHubAuthorization{
MarkInternalReposAsPublic: true,
},
})),
}
githubSrc, err = NewGitHubSource(ctx, logtest.Scoped(t), dbmocks.NewMockDB(), svc, cf)
if err != nil {
t.Fatal(err)
}
repo, err = githubSrc.GetRepo(context.Background(), tc.nameWithOwner)
if err != nil {
t.Fatalf("GetRepo failed: %v", err)
}
if have, want := fmt.Sprint(err), tc.err; have != want {
t.Errorf("error:\nhave: %q\nwant: %q", have, want)
}
if tc.assert != nil {
tc.assert(t, repo, false)
}
})
}

63
internal/repos/testdata/sources/GITHUB-ENTERPRISE/internal-repo-in-github-enterprise.yaml vendored
View File

File diff suppressed because one or more lines are too long

10
schema/github.schema.json
View File

@ -169,6 +169,16 @@
"description": "Experimental: If set, configures hours cached permissions from teams and organizations should be kept for. Setting a negative value disables syncing from teams and organizations, and falls back to the default behaviour of syncing all permisisons directly from user-repository affiliations instead. [Learn more](https://docs.sourcegraph.com/admin/external_service/github#teams-and-organizations-permissions-caching).",
"type": "number",
"default": 72
},
"syncInternalRepoPermissions": {
"description": "If true, access to internal repositories will be synced as part of user permission syncs. This can lead to slower user permission syncs for organizations with many internal repositories. Defaults to false.",
"type": "boolean",
"default": false
},
"markInternalReposAsPublic": {
"description": "If true, internal repositories will be accessible to all users on Sourcegraph as if they were public. This overrides repository permissions but allows easier discovery and access to internal repositories, and may be desirable if all users on the Sourcegraph instance should have access to all internal repositories anyways. Defaults to false.",
"type": "boolean",
"default": false
}
}
},

4
schema/schema.go
View File

@ -1130,6 +1130,10 @@ type GitHubAuthProvider struct {
type GitHubAuthorization struct {
// GroupsCacheTTL description: Experimental: If set, configures hours cached permissions from teams and organizations should be kept for. Setting a negative value disables syncing from teams and organizations, and falls back to the default behaviour of syncing all permisisons directly from user-repository affiliations instead. [Learn more](https://docs.sourcegraph.com/admin/external_service/github#teams-and-organizations-permissions-caching).
GroupsCacheTTL float64 `json:"groupsCacheTTL,omitempty"`
// MarkInternalReposAsPublic description: If true, internal repositories will be accessible to all users on Sourcegraph as if they were public. This overrides repository permissions but allows easier discovery and access to internal repositories, and may be desirable if all users on the Sourcegraph instance should have access to all internal repositories anyways. Defaults to false.
MarkInternalReposAsPublic bool `json:"markInternalReposAsPublic,omitempty"`
// SyncInternalRepoPermissions description: If true, access to internal repositories will be synced as part of user permission syncs. This can lead to slower user permission syncs for organizations with many internal repositories. Defaults to false.
SyncInternalRepoPermissions bool `json:"syncInternalRepoPermissions,omitempty"`
}
// GitHubConnection description: Configuration for a connection to GitHub or GitHub Enterprise.