sourcegraph/internal/conf/computed.go
Erik Seliger d04e821ceb
Experiment: Natively run SSBC in docker (#44034)
This adds an experimental code path that I will use to test a docker-only execution mode for server-side batch changes. This code path is never executed for customers until we make the switch when we deem it ready. This will allow me to dogfood this while it's not available to customer instances yet.

Ultimately, the goal of this is to make executors simply be "the job runner platform through a generic interface". Today, this depends on src-cli to do a good bunch of the work. This is a blocker for going full docker-based with executors, which will ultimately be a requirement on the road to k8s-based executors.

As this removes the dependency on src-cli, nothing but the job interface and API endpoints tie executor and Sourcegraph instance together. Ultimately, this will allow us to support larger version spans between the two (pending executors going GA and being feature-complete).

Known issues/limitations:

Steps skipped in between steps that run don't work yet
Skipping steps dynamically is inefficient as we cannot tell the executor to skip a step IF X, so we replace the script by exit 0
It is unclear if all variants of file mounts still work. Basic cases do work. Files used to be read-only in src-cli, they aren't now, but content is still reset in between steps.
The assumption that everything operates in /work is broken here, because we need to use what executors give us to persist out-of-repo state in between containers (like the step result from the previous step)
It is unclear if workspace mounts work
Cache keys are not correctly computed if using workspace mounts - the metadataretriever is nil
We still use log outputs to transfer the AfterStepResults to the Sourcegraph instance, this should finally become an artifact instead. Then, we don't have to rely on the execution_log_entires anymore and can theoretically prune those after some time. This column is currently growing indefinitely.
It depends on tee being available in the docker images to capture the cmd.stdout/cmd.stderr properly for template variable rendering
Env-vars are not rendered in their evaluated form post-execution
File permissions are unclear and might be similarly broken to how they are now - or even worse
Disclaimer: It's not feature complete today! But it is also not hitting any default code paths either. As development on this goes on, we can eventually remove the feature flag and run the new job format on all instances. This PR handles fallback of rendering old records correctly in the UI already.
2022-11-10 00:20:43 +01:00

470 lines
12 KiB
Go

package conf
import (
"context"
"log"
"strings"
"time"
"github.com/sourcegraph/sourcegraph/internal/api/internalapi"
"github.com/sourcegraph/sourcegraph/internal/conf/confdefaults"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
"github.com/sourcegraph/sourcegraph/internal/conf/deploy"
"github.com/sourcegraph/sourcegraph/internal/extsvc"
srccli "github.com/sourcegraph/sourcegraph/internal/src-cli"
"github.com/sourcegraph/sourcegraph/internal/version"
"github.com/sourcegraph/sourcegraph/schema"
)
func init() {
deployType := deploy.Type()
if !deploy.IsValidDeployType(deployType) {
log.Fatalf("The 'DEPLOY_TYPE' environment variable is invalid. Expected one of: %q, %q, %q, %q, %q, %q. Got: %q", deploy.Kubernetes, deploy.DockerCompose, deploy.PureDocker, deploy.SingleDocker, deploy.Dev, deploy.Helm, deployType)
}
confdefaults.Default = defaultConfigForDeployment()
}
func defaultConfigForDeployment() conftypes.RawUnified {
deployType := deploy.Type()
switch {
case deploy.IsDev(deployType):
return confdefaults.DevAndTesting
case deploy.IsDeployTypeSingleDockerContainer(deployType):
return confdefaults.DockerContainer
case deploy.IsDeployTypeKubernetes(deployType), deploy.IsDeployTypeDockerCompose(deployType), deploy.IsDeployTypePureDocker(deployType):
return confdefaults.KubernetesOrDockerComposeOrPureDocker
default:
panic("deploy type did not register default configuration")
}
}
func BitbucketServerConfigs(ctx context.Context) ([]*schema.BitbucketServerConnection, error) {
var config []*schema.BitbucketServerConnection
if err := internalapi.Client.ExternalServiceConfigs(ctx, extsvc.KindBitbucketServer, &config); err != nil {
return nil, err
}
return config, nil
}
func GitHubConfigs(ctx context.Context) ([]*schema.GitHubConnection, error) {
var config []*schema.GitHubConnection
if err := internalapi.Client.ExternalServiceConfigs(ctx, extsvc.KindGitHub, &config); err != nil {
return nil, err
}
return config, nil
}
func GitLabConfigs(ctx context.Context) ([]*schema.GitLabConnection, error) {
var config []*schema.GitLabConnection
if err := internalapi.Client.ExternalServiceConfigs(ctx, extsvc.KindGitLab, &config); err != nil {
return nil, err
}
return config, nil
}
func GitoliteConfigs(ctx context.Context) ([]*schema.GitoliteConnection, error) {
var config []*schema.GitoliteConnection
if err := internalapi.Client.ExternalServiceConfigs(ctx, extsvc.KindGitolite, &config); err != nil {
return nil, err
}
return config, nil
}
func PhabricatorConfigs(ctx context.Context) ([]*schema.PhabricatorConnection, error) {
var config []*schema.PhabricatorConnection
if err := internalapi.Client.ExternalServiceConfigs(ctx, extsvc.KindPhabricator, &config); err != nil {
return nil, err
}
return config, nil
}
type AccessTokenAllow string
const (
AccessTokensNone AccessTokenAllow = "none"
AccessTokensAll AccessTokenAllow = "all-users-create"
AccessTokensAdmin AccessTokenAllow = "site-admin-create"
)
// AccessTokensAllow returns whether access tokens are enabled, disabled, or
// restricted creation to only site admins.
func AccessTokensAllow() AccessTokenAllow {
cfg := Get().AuthAccessTokens
if cfg == nil || cfg.Allow == "" {
return AccessTokensAll
}
v := AccessTokenAllow(cfg.Allow)
switch v {
case AccessTokensAll, AccessTokensAdmin:
return v
default:
return AccessTokensNone
}
}
// EmailVerificationRequired returns whether users must verify an email address before they
// can perform most actions on this site.
//
// It's false for sites that do not have an email sending API key set up.
func EmailVerificationRequired() bool {
return Get().EmailSmtp != nil
}
// CanSendEmail returns whether the site can send emails (e.g., to reset a password or
// invite a user to an org).
//
// It's false for sites that do not have an email sending API key set up.
func CanSendEmail() bool {
return Get().EmailSmtp != nil
}
// UpdateChannel tells the update channel. Default is "release".
func UpdateChannel() string {
channel := Get().UpdateChannel
if channel == "" {
return "release"
}
return channel
}
// SearchIndexEnabled returns true if sourcegraph should index all
// repositories for text search.
func SearchIndexEnabled() bool {
if v := Get().SearchIndexEnabled; v != nil {
return *v
}
return true // always on by default in all deployment types, see confdefaults.go
}
func BatchChangesEnabled() bool {
if enabled := Get().BatchChangesEnabled; enabled != nil {
return *enabled
}
return true
}
func BatchChangesRestrictedToAdmins() bool {
if restricted := Get().BatchChangesRestrictToAdmins; restricted != nil {
return *restricted
}
return false
}
func ExecutorsEnabled() bool {
return Get().ExecutorsAccessToken != ""
}
func ExecutorsFrontendURL() string {
current := Get()
if current.ExecutorsFrontendURL != "" {
return current.ExecutorsFrontendURL
}
return current.ExternalURL
}
func ExecutorsSrcCLIImage() string {
current := Get()
if current.ExecutorsSrcCLIImage != "" {
return current.ExecutorsSrcCLIImage
}
return "sourcegraph/src-cli"
}
func ExecutorsSrcCLIImageTag() string {
current := Get()
if current.ExecutorsSrcCLIImageTag != "" {
return current.ExecutorsSrcCLIImageTag
}
return srccli.MinimumVersion
}
func ExecutorsBatcheshelperImage() string {
current := Get()
if current.ExecutorsBatcheshelperImage != "" {
return current.ExecutorsBatcheshelperImage
}
return "sourcegraph/batcheshelper"
}
func ExecutorsBatcheshelperImageTag() string {
current := Get()
if current.ExecutorsBatcheshelperImageTag != "" {
return current.ExecutorsBatcheshelperImageTag
}
if version.IsDev(version.Version()) {
return "insiders"
}
return version.Version()
}
func CodeIntelAutoIndexingEnabled() bool {
if enabled := Get().CodeIntelAutoIndexingEnabled; enabled != nil {
return *enabled
}
return false
}
func CodeIntelAutoIndexingAllowGlobalPolicies() bool {
if enabled := Get().CodeIntelAutoIndexingAllowGlobalPolicies; enabled != nil {
return *enabled
}
return false
}
func CodeIntelAutoIndexingPolicyRepositoryMatchLimit() int {
val := Get().CodeIntelAutoIndexingPolicyRepositoryMatchLimit
if val == nil || *val < -1 {
return -1
}
return *val
}
func ProductResearchPageEnabled() bool {
if enabled := Get().ProductResearchPageEnabled; enabled != nil {
return *enabled
}
return true
}
func ExternalURL() string {
return Get().ExternalURL
}
func UsingExternalURL() bool {
url := Get().ExternalURL
return !(url == "" || strings.HasPrefix(url, "http://localhost") || strings.HasPrefix(url, "https://localhost") || strings.HasPrefix(url, "http://127.0.0.1") || strings.HasPrefix(url, "https://127.0.0.1")) // CI:LOCALHOST_OK
}
func IsExternalURLSecure() bool {
return strings.HasPrefix(Get().ExternalURL, "https:")
}
func IsBuiltinSignupAllowed() bool {
provs := Get().AuthProviders
for _, prov := range provs {
if prov.Builtin != nil {
return prov.Builtin.AllowSignup
}
}
return false
}
// SearchSymbolsParallelism returns 20, or the site config
// "debug.search.symbolsParallelism" value if configured.
func SearchSymbolsParallelism() int {
val := Get().DebugSearchSymbolsParallelism
if val == 0 {
return 20
}
return val
}
func BitbucketServerPluginPerm() bool {
val := ExperimentalFeatures().BitbucketServerFastPerm
return val == "enabled"
}
func EventLoggingEnabled() bool {
val := ExperimentalFeatures().EventLogging
if val == "" {
return true
}
return val == "enabled"
}
func StructuralSearchEnabled() bool {
val := ExperimentalFeatures().StructuralSearch
if val == "" {
return true
}
return val == "enabled"
}
func ExperimentalFeatures() schema.ExperimentalFeatures {
val := Get().ExperimentalFeatures
if val == nil {
return schema.ExperimentalFeatures{}
}
return *val
}
func Tracer() string {
ot := Get().ObservabilityTracing
if ot == nil {
return ""
}
return ot.Type
}
// AuthMinPasswordLength returns the value of minimum password length requirement.
// If not set, it returns the default value 12.
func AuthMinPasswordLength() int {
val := Get().AuthMinPasswordLength
if val <= 0 {
return 12
}
return val
}
// GenericPasswordPolicy is a generic password policy that defines password requirements.
type GenericPasswordPolicy struct {
Enabled bool
MinimumLength int
NumberOfSpecialCharacters int
RequireAtLeastOneNumber bool
RequireUpperandLowerCase bool
}
// AuthPasswordPolicy returns a GenericPasswordPolicy for password validation
func AuthPasswordPolicy() GenericPasswordPolicy {
ml := Get().AuthMinPasswordLength
if p := Get().AuthPasswordPolicy; p != nil {
return GenericPasswordPolicy{
Enabled: p.Enabled,
MinimumLength: ml,
NumberOfSpecialCharacters: p.NumberOfSpecialCharacters,
RequireAtLeastOneNumber: p.RequireAtLeastOneNumber,
RequireUpperandLowerCase: p.RequireUpperandLowerCase,
}
}
if ep := ExperimentalFeatures().PasswordPolicy; ep != nil {
return GenericPasswordPolicy{
Enabled: ep.Enabled,
MinimumLength: ml,
NumberOfSpecialCharacters: ep.NumberOfSpecialCharacters,
RequireAtLeastOneNumber: ep.RequireAtLeastOneNumber,
RequireUpperandLowerCase: ep.RequireUpperandLowerCase,
}
}
return GenericPasswordPolicy{
Enabled: false,
MinimumLength: 0,
NumberOfSpecialCharacters: 0,
RequireAtLeastOneNumber: false,
RequireUpperandLowerCase: false,
}
}
func PasswordPolicyEnabled() bool {
pc := AuthPasswordPolicy()
return pc.Enabled
}
// By default, password reset links are valid for 4 hours.
const defaultPasswordLinkExpiry = 14400
// AuthPasswordResetLinkExpiry returns the time (in seconds) indicating how long password
// reset links are considered valid. If not set, it returns the default value.
func AuthPasswordResetLinkExpiry() int {
val := Get().AuthPasswordResetLinkExpiry
if val <= 0 {
return defaultPasswordLinkExpiry
}
return val
}
// AuthLockout populates and returns the *schema.AuthLockout with default values
// for fields that are not initialized.
func AuthLockout() *schema.AuthLockout {
val := Get().AuthLockout
if val == nil {
return &schema.AuthLockout{
ConsecutivePeriod: 3600,
FailedAttemptThreshold: 5,
LockoutPeriod: 1800,
}
}
if val.ConsecutivePeriod <= 0 {
val.ConsecutivePeriod = 3600
}
if val.FailedAttemptThreshold <= 0 {
val.FailedAttemptThreshold = 5
}
if val.LockoutPeriod <= 0 {
val.LockoutPeriod = 1800
}
return val
}
type ExternalServiceMode int
const (
ExternalServiceModeDisabled ExternalServiceMode = 0
ExternalServiceModePublic ExternalServiceMode = 1
ExternalServiceModeAll ExternalServiceMode = 2
)
func (e ExternalServiceMode) String() string {
switch e {
case ExternalServiceModeDisabled:
return "disabled"
case ExternalServiceModePublic:
return "public"
case ExternalServiceModeAll:
return "all"
default:
return "unknown"
}
}
// ExternalServiceUserMode returns the site level mode describing if users are
// allowed to add external services for public and private repositories. It does
// NOT take into account permissions granted to the current user.
func ExternalServiceUserMode() ExternalServiceMode {
switch Get().ExternalServiceUserMode {
case "public":
return ExternalServiceModePublic
case "all":
return ExternalServiceModeAll
default:
return ExternalServiceModeDisabled
}
}
const defaultGitLongCommandTimeout = time.Hour
// GitLongCommandTimeout returns the maximum amount of time in seconds that a
// long Git command (e.g. clone or remote update) is allowed to execute. If not
// set, it returns the default value.
//
// In general, Git commands that are expected to take a long time should be
// executed in the background in a non-blocking fashion.
func GitLongCommandTimeout() time.Duration {
val := Get().GitLongCommandTimeout
if val < 1 {
return defaultGitLongCommandTimeout
}
return time.Duration(val) * time.Second
}
// GitMaxCodehostRequestsPerSecond returns maximum number of remote code host
// git operations to be run per second per gitserver. If not set, it returns the
// default value -1.
func GitMaxCodehostRequestsPerSecond() int {
val := Get().GitMaxCodehostRequestsPerSecond
if val == nil || *val < -1 {
return -1
}
return *val
}
func GitMaxConcurrentClones() int {
v := Get().GitMaxConcurrentClones
if v <= 0 {
return 5
}
return v
}