artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 15:51:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Categorize enterprise frontend startup behaviors (#12539)

Browse Source
This commit is contained in:
Eric Fritz 2020-07-29 10:40:52 -05:00 committed by GitHub
parent b1e5893c7e
commit 407a025658
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 241 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/CODEOWNERS vendored
View File

@ -143,6 +143,7 @@
/cmd/frontend/graphqlbackend/campaigns.go @sourcegraph/campaigns
/doc/user/campaigns @sourcegraph/campaigns
/enterprise/internal/campaigns @sourcegraph/campaigns
/enterprise/cmd/frontend/internal/campaigns @sourcegraph/campaigns
/internal/campaigns @sourcegraph/campaigns
/web/**/campaigns/** @sourcegraph/campaigns
@ -230,6 +231,7 @@ Dockerfile @sourcegraph/distribution
/enterprise/cmd/precise-code-intel-worker/ @sourcegraph/code-intel
/enterprise/cmd/precise-code-intel-indexer/ @sourcegraph/code-intel
/enterprise/internal/codeintel @sourcegraph/code-intel
/enterprise/cmd/frontend/internal/codeintel @sourcegraph/code-intel
/cmd/frontend/graphqlbackend/codeintel.go @sourcegraph/code-intel
/internal/cmd/precise-code-intel-tester @sourcegraph/code-intel

36
enterprise/cmd/frontend/internal/authz/init.go Normal file
View File

@ -0,0 +1,36 @@
package authz
import (
"context"
"time"
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
eauthz "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/authz"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/authz/resolvers"
eiauthz "github.com/sourcegraph/sourcegraph/enterprise/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/db"
"github.com/sourcegraph/sourcegraph/internal/db/dbconn"
)
func Init(ctx context.Context, enterpriseServices *enterprise.Services) error {
eauthz.Init(dbconn.Global, msResolutionClock)
go func() {
t := time.NewTicker(5 * time.Second)
for range t.C {
allowAccessByDefault, authzProviders, _, _ :=
eiauthz.ProvidersFromConfig(ctx, conf.Get(), db.ExternalServices)
authz.SetProviders(allowAccessByDefault, authzProviders)
}
}()
enterpriseServices.AuthzResolver = resolvers.NewResolver(dbconn.Global, func() time.Time {
return time.Now().UTC().Truncate(time.Microsecond)
})
return nil
}
var msResolutionClock = func() time.Time { return time.Now().UTC().Truncate(time.Microsecond) }

37
enterprise/cmd/frontend/internal/campaigns/init.go Normal file
View File

@ -0,0 +1,37 @@
package campaigns
import (
"context"
"database/sql"
"time"
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
"github.com/sourcegraph/sourcegraph/cmd/repo-updater/repos"
"github.com/sourcegraph/sourcegraph/enterprise/internal/campaigns"
"github.com/sourcegraph/sourcegraph/enterprise/internal/campaigns/resolvers"
"github.com/sourcegraph/sourcegraph/internal/db/dbconn"
"github.com/sourcegraph/sourcegraph/internal/db/globalstatedb"
)
func Init(ctx context.Context, enterpriseServices *enterprise.Services) error {
globalState, err := globalstatedb.Get(ctx)
if err != nil {
return err
}
campaignsStore := campaigns.NewStoreWithClock(dbconn.Global, msResolutionClock)
repositories := repos.NewDBStore(dbconn.Global, sql.TxOptions{})
enterpriseServices.CampaignsResolver = resolvers.NewResolver(dbconn.Global)
enterpriseServices.GithubWebhook = campaigns.NewGitHubWebhook(campaignsStore, repositories, msResolutionClock)
enterpriseServices.BitbucketServerWebhook = campaigns.NewBitbucketServerWebhook(
campaignsStore,
repositories,
msResolutionClock,
"sourcegraph-"+globalState.SiteID,
)
return nil
}
var msResolutionClock = func() time.Time { return time.Now().UTC().Truncate(time.Microsecond) }

67
enterprise/cmd/frontend/internal/codeintel/init.go Normal file
View File

@ -0,0 +1,67 @@
package codeintel
import (
"context"
"fmt"
"net/http"
"strconv"
"github.com/inconshreveable/log15"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
codeintelapi "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/api"
bundles "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/bundles/client"
codeintelgitserver "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/gitserver"
codeintelhttpapi "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/httpapi"
codeintelresolvers "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/resolvers"
codeintelgqlresolvers "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/resolvers/graphql"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/store"
"github.com/sourcegraph/sourcegraph/internal/db/basestore"
"github.com/sourcegraph/sourcegraph/internal/db/dbconn"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/trace"
)
var bundleManagerURL = env.Get("PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL", "", "HTTP address for internal LSIF bundle manager server.")
var rawHunkCacheSize = env.Get("PRECISE_CODE_INTEL_HUNK_CACHE_CAPACITY", "1000", "Maximum number of git diff hunk objects that can be loaded into the hunk cache at once.")
func Init(ctx context.Context, enterpriseServices *enterprise.Services) error {
if bundleManagerURL == "" {
return fmt.Errorf("invalid value for PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL: no value supplied")
}
hunkCacheSize, err := strconv.ParseInt(rawHunkCacheSize, 10, 64)
if err != nil {
return fmt.Errorf("invalid int %q for PRECISE_CODE_INTEL_HUNK_CACHE_CAPACITY: %s", rawHunkCacheSize, err)
}
observationContext := &observation.Context{
Logger: log15.Root(),
Tracer: &trace.Tracer{Tracer: opentracing.GlobalTracer()},
Registerer: prometheus.DefaultRegisterer,
}
store := store.NewObserved(store.NewWithHandle(basestore.NewHandleWithDB(dbconn.Global)), observationContext)
bundleManagerClient := bundles.New(bundleManagerURL)
api := codeintelapi.NewObserved(codeintelapi.New(store, bundleManagerClient, codeintelgitserver.DefaultClient), observationContext)
hunkCache, err := codeintelresolvers.NewHunkCache(int(hunkCacheSize))
if err != nil {
return fmt.Errorf("failed to initialize hunk cache: %s", err)
}
enterpriseServices.CodeIntelResolver = codeintelgqlresolvers.NewResolver(codeintelresolvers.NewResolver(
store,
bundleManagerClient,
api,
hunkCache,
))
enterpriseServices.NewCodeIntelUploadHandler = func(internal bool) http.Handler {
return codeintelhttpapi.NewUploadHandler(store, bundleManagerClient, internal)
}
return nil
}

69
enterprise/cmd/frontend/internal/licensing/init/init.go Normal file
View File

@ -0,0 +1,69 @@
package init
import (
"context"
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
"github.com/sourcegraph/sourcegraph/cmd/frontend/envvar"
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/dotcom/productsubscription"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/licensing"
"github.com/sourcegraph/sourcegraph/internal/db"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/graphqlbackend"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/registry"
)
// TODO(efritz) - de-globalize assignments in this function
// TODO(efritz) - refactor licensing packages - this is a huge mess!
func Init(ctx context.Context, enterpriseServices *enterprise.Services) error {
// Enforce the license's max user count by preventing the creation of new users when the max is
// reached.
db.Users.PreCreateUser = licensing.NewPreCreateUserHook(&usersStore{})
// Make the Site.productSubscription.productNameWithBrand GraphQL field (and other places) use the
// proper product name.
graphqlbackend.GetProductNameWithBrand = licensing.ProductNameWithBrand
// Make the Site.productSubscription.actualUserCount and Site.productSubscription.actualUserCountDate
// GraphQL fields return the proper max user count and timestamp on the current license.
graphqlbackend.ActualUserCount = licensing.ActualUserCount
graphqlbackend.ActualUserCountDate = licensing.ActualUserCountDate
noLicenseMaximumAllowedUserCount := licensing.NoLicenseMaximumAllowedUserCount
graphqlbackend.NoLicenseMaximumAllowedUserCount = &noLicenseMaximumAllowedUserCount
noLicenseWarningUserCount := licensing.NoLicenseWarningUserCount
graphqlbackend.NoLicenseWarningUserCount = &noLicenseWarningUserCount
// Make the Site.productSubscription GraphQL field return the actual info about the product license,
// if any.
graphqlbackend.GetConfiguredProductLicenseInfo = func() (*graphqlbackend.ProductLicenseInfo, error) {
info, err := licensing.GetConfiguredProductLicenseInfo()
if info == nil || err != nil {
return nil, err
}
return &graphqlbackend.ProductLicenseInfo{
TagsValue: info.Tags,
UserCountValue: info.UserCount,
ExpiresAtValue: info.ExpiresAt,
}, nil
}
goroutine.Go(func() {
licensing.StartMaxUserCount(&usersStore{})
})
if envvar.SourcegraphDotComMode() {
goroutine.Go(productsubscription.StartCheckForUpcomingLicenseExpirations)
}
return nil
}
type usersStore struct{}
func (usersStore) Count(ctx context.Context) (int, error) {
return db.Users.Count(ctx, nil)
}

207
enterprise/cmd/frontend/main.go
View File

@ -7,195 +7,48 @@ package main
import (
"context"
"database/sql"
"fmt"
"log"
"net/http"
"os"
"strconv"
"time"
"github.com/inconshreveable/log15"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sourcegraph/sourcegraph/cmd/frontend/enterprise"
"github.com/sourcegraph/sourcegraph/cmd/frontend/envvar"
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend"
"github.com/sourcegraph/sourcegraph/cmd/frontend/shared"
"github.com/sourcegraph/sourcegraph/cmd/repo-updater/repos"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/authz"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/campaigns"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/codeintel"
licensing "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/licensing/init"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth"
eauthz "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/authz"
authzResolvers "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/authz/resolvers"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/dotcom/productsubscription"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/graphqlbackend"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/licensing"
_ "github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/internal/registry"
eiauthz "github.com/sourcegraph/sourcegraph/enterprise/internal/authz"
"github.com/sourcegraph/sourcegraph/enterprise/internal/campaigns"
campaignsResolvers "github.com/sourcegraph/sourcegraph/enterprise/internal/campaigns/resolvers"
codeintelapi "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/api"
bundles "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/bundles/client"
codeintelgitserver "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/gitserver"
codeintelhttpapi "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/httpapi"
codeintelresolvers "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/resolvers"
codeintelgqlresolvers "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/resolvers/graphql"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/store"
"github.com/sourcegraph/sourcegraph/internal/authz"
"github.com/sourcegraph/sourcegraph/internal/conf"
"github.com/sourcegraph/sourcegraph/internal/db"
"github.com/sourcegraph/sourcegraph/internal/db/basestore"
"github.com/sourcegraph/sourcegraph/internal/db/dbconn"
"github.com/sourcegraph/sourcegraph/internal/db/globalstatedb"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/trace"
)
func main() {
shared.Main(func() enterprise.Services {
debug, _ := strconv.ParseBool(os.Getenv("DEBUG"))
if debug {
log.Println("enterprise edition")
shared.Main(enterpriseSetupHook)
}
var initFunctions = map[string]func(ctx context.Context, enterpriseServices *enterprise.Services) error{
"authz": authz.Init,
"campaigns": campaigns.Init,
"codeintel": codeintel.Init,
"licensing": licensing.Init,
}
func enterpriseSetupHook() enterprise.Services {
debug, _ := strconv.ParseBool(os.Getenv("DEBUG"))
if debug {
log.Println("enterprise edition")
}
ctx := context.Background()
enterpriseServices := enterprise.DefaultServices()
for name, fn := range initFunctions {
if err := fn(ctx, &enterpriseServices); err != nil {
log.Fatal(fmt.Sprintf("failed to initialize %s: %s", name, err))
}
}
ctx := context.Background()
enterpriseServices := enterprise.DefaultServices()
initLicensing()
initAuthz(ctx, &enterpriseServices)
initCampaigns(ctx, &enterpriseServices)
initCodeIntel(&enterpriseServices)
return enterpriseServices
})
}
var msResolutionClock = func() time.Time {
return time.Now().UTC().Truncate(time.Microsecond)
}
func initLicensing() {
// TODO(efritz) - de-globalize assignments in this function
// Enforce the license's max user count by preventing the creation of new users when the max is
// reached.
db.Users.PreCreateUser = licensing.NewPreCreateUserHook(&usersStore{})
// Make the Site.productSubscription.productNameWithBrand GraphQL field (and other places) use the
// proper product name.
graphqlbackend.GetProductNameWithBrand = licensing.ProductNameWithBrand
// Make the Site.productSubscription.actualUserCount and Site.productSubscription.actualUserCountDate
// GraphQL fields return the proper max user count and timestamp on the current license.
graphqlbackend.ActualUserCount = licensing.ActualUserCount
graphqlbackend.ActualUserCountDate = licensing.ActualUserCountDate
noLicenseMaximumAllowedUserCount := licensing.NoLicenseMaximumAllowedUserCount
graphqlbackend.NoLicenseMaximumAllowedUserCount = &noLicenseMaximumAllowedUserCount
noLicenseWarningUserCount := licensing.NoLicenseWarningUserCount
graphqlbackend.NoLicenseWarningUserCount = &noLicenseWarningUserCount
// Make the Site.productSubscription GraphQL field return the actual info about the product license,
// if any.
graphqlbackend.GetConfiguredProductLicenseInfo = func() (*graphqlbackend.ProductLicenseInfo, error) {
info, err := licensing.GetConfiguredProductLicenseInfo()
if info == nil || err != nil {
return nil, err
}
return &graphqlbackend.ProductLicenseInfo{
TagsValue: info.Tags,
UserCountValue: info.UserCount,
ExpiresAtValue: info.ExpiresAt,
}, nil
}
goroutine.Go(func() {
licensing.StartMaxUserCount(&usersStore{})
})
if envvar.SourcegraphDotComMode() {
goroutine.Go(productsubscription.StartCheckForUpcomingLicenseExpirations)
}
}
func initAuthz(ctx context.Context, enterpriseServices *enterprise.Services) {
eauthz.Init(dbconn.Global, msResolutionClock)
go func() {
t := time.NewTicker(5 * time.Second)
for range t.C {
allowAccessByDefault, authzProviders, _, _ :=
eiauthz.ProvidersFromConfig(ctx, conf.Get(), db.ExternalServices)
authz.SetProviders(allowAccessByDefault, authzProviders)
}
}()
enterpriseServices.AuthzResolver = authzResolvers.NewResolver(dbconn.Global, func() time.Time {
return time.Now().UTC().Truncate(time.Microsecond)
})
}
func initCampaigns(ctx context.Context, enterpriseServices *enterprise.Services) {
globalState, err := globalstatedb.Get(ctx)
if err != nil {
log.Fatalf("FATAL: %v", err)
}
campaignsStore := campaigns.NewStoreWithClock(dbconn.Global, msResolutionClock)
repositories := repos.NewDBStore(dbconn.Global, sql.TxOptions{})
enterpriseServices.CampaignsResolver = campaignsResolvers.NewResolver(dbconn.Global)
enterpriseServices.GithubWebhook = campaigns.NewGitHubWebhook(campaignsStore, repositories, msResolutionClock)
enterpriseServices.BitbucketServerWebhook = campaigns.NewBitbucketServerWebhook(
campaignsStore,
repositories,
msResolutionClock,
"sourcegraph-"+globalState.SiteID,
)
}
var bundleManagerURL = env.Get("PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL", "", "HTTP address for internal LSIF bundle manager server.")
var rawHunkCacheSize = env.Get("PRECISE_CODE_INTEL_HUNK_CACHE_CAPACITY", "1000", "Maximum number of git diff hunk objects that can be loaded into the hunk cache at once.")
func initCodeIntel(enterpriseServices *enterprise.Services) {
if bundleManagerURL == "" {
log.Fatalf("invalid value for PRECISE_CODE_INTEL_BUNDLE_MANAGER_URL: no value supplied")
}
hunkCacheSize, err := strconv.ParseInt(rawHunkCacheSize, 10, 64)
if err != nil {
log.Fatalf("invalid int %q for PRECISE_CODE_INTEL_HUNK_CACHE_CAPACITY: %s", rawHunkCacheSize, err)
}
observationContext := &observation.Context{
Logger: log15.Root(),
Tracer: &trace.Tracer{Tracer: opentracing.GlobalTracer()},
Registerer: prometheus.DefaultRegisterer,
}
store := store.NewObserved(store.NewWithHandle(basestore.NewHandleWithDB(dbconn.Global)), observationContext)
bundleManagerClient := bundles.New(bundleManagerURL)
api := codeintelapi.NewObserved(codeintelapi.New(store, bundleManagerClient, codeintelgitserver.DefaultClient), observationContext)
hunkCache, err := codeintelresolvers.NewHunkCache(int(hunkCacheSize))
if err != nil {
log.Fatalf("failed to initialize hunk cache: %s", err)
}
enterpriseServices.CodeIntelResolver = codeintelgqlresolvers.NewResolver(codeintelresolvers.NewResolver(
store,
bundleManagerClient,
api,
hunkCache,
))
enterpriseServices.NewCodeIntelUploadHandler = func(internal bool) http.Handler {
return codeintelhttpapi.NewUploadHandler(store, bundleManagerClient, internal)
}
}
type usersStore struct{}
func (usersStore) Count(ctx context.Context) (int, error) {
return db.Users.Count(ctx, nil)
return enterpriseServices
}