artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 19:21:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

RFC 619: Move upload commitgraph background job into the new job skeleton (#36112)

Browse Source
This commit is contained in:
Cesar Jimenez 2022-05-27 10:27:11 -04:00 committed by GitHub
parent 1385aaa591
commit dc2068fac6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 319 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
doc/admin/observability/alert_solutions.md generated
View File

@ -5915,3 +5915,28 @@ with your code hosts connections or networking issues affecting communication wi
<br />
## codeintel-uploads: codeintel_commit_graph_queued_max_age
<p class="subtitle">repository queue longest time in queue</p>
**Descriptions**
- <span class="badge badge-critical">critical</span> codeintel-uploads: 3600s+ repository queue longest time in queue
**Possible solutions**
- An alert here is generally indicative of either underprovisioned worker instance(s) and/or
an underprovisioned main postgres instance.
- Learn more about the related dashboard panel in the [dashboards reference](./dashboards.md#codeintel-uploads-codeintel-commit-graph-queued-max-age).
- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert:
```json
"observability.silenceAlerts": [
"critical_codeintel-uploads_codeintel_commit_graph_queued_max_age"
]
```
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>
<br />

73
doc/admin/observability/dashboards.md generated
View File

@ -19705,6 +19705,71 @@ Query: `sum(increase(src_codeintel_uploads_background_cleanup_errors_total{job=~
<br />
### Code Intelligence > Uploads: Codeintel: Repository with stale commit graph
#### codeintel-uploads: codeintel_commit_graph_queue_size
<p class="subtitle">Repository queue size</p>
This panel has no related alerts.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100500` on your Sourcegraph instance.
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>
<details>
<summary>Technical details</summary>
Query: `max(src_codeintel_commit_graph_total{job=~"^.*"})`
</details>
<br />
#### codeintel-uploads: codeintel_commit_graph_queue_growth_rate
<p class="subtitle">Repository queue growth rate over 30m</p>
This value compares the rate of enqueues against the rate of finished jobs.
- A value < than 1 indicates that process rate > enqueue rate
- A value = than 1 indicates that process rate = enqueue rate
- A value > than 1 indicates that process rate < enqueue rate
This panel has no related alerts.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100501` on your Sourcegraph instance.
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>
<details>
<summary>Technical details</summary>
Query: `sum(increase(src_codeintel_commit_graph_total{job=~"^.*"}[30m])) / sum(increase(src_codeintel_commit_graph_processor_total{job=~"^.*"}[30m]))`
</details>
<br />
#### codeintel-uploads: codeintel_commit_graph_queued_max_age
<p class="subtitle">Repository queue longest time in queue</p>
Refer to the [alert solutions reference](./alert_solutions.md#codeintel-uploads-codeintel-commit-graph-queued-max-age) for 1 alert related to this panel.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100502` on your Sourcegraph instance.
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>
<details>
<summary>Technical details</summary>
Query: `max(src_codeintel_commit_graph_queued_duration_seconds_total{job=~"^.*"})`
</details>
<br />
### Code Intelligence > Uploads: Codeintel: Uploads > Expiration task
#### codeintel-uploads: codeintel_background_repositories_scanned_total_total
@ -19715,7 +19780,7 @@ Number of repositories scanned for data retention
This panel has no related alerts.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100500` on your Sourcegraph instance.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100600` on your Sourcegraph instance.
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>
@ -19736,7 +19801,7 @@ Number of codeintel upload records scanned for data retention
This panel has no related alerts.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100501` on your Sourcegraph instance.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100601` on your Sourcegraph instance.
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>
@ -19757,7 +19822,7 @@ Number of commits reachable from a codeintel upload record scanned for data rete
This panel has no related alerts.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100502` on your Sourcegraph instance.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100602` on your Sourcegraph instance.
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>
@ -19778,7 +19843,7 @@ Number of codeintel upload records marked as expired
This panel has no related alerts.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100503` on your Sourcegraph instance.
To see this panel, visit `/-/debug/grafana/d/codeintel-uploads/codeintel-uploads?viewPanel=100603` on your Sourcegraph instance.
<sub>*Managed by the [Sourcegraph Code intelligence team](https://handbook.sourcegraph.com/departments/product-engineering/engineering/code-graph/code-intelligence).*</sub>

3
enterprise/cmd/worker/internal/codeintel/commitgraph/gen.go
View File

@ -1,3 +0,0 @@
package commitgraph
//go:generate ../../../../../../dev/mockgen.sh github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/internal/codeintel/commitgraph -i DBStore -i Locker -i GitserverClient -o mock_iface_test.go

17
enterprise/cmd/worker/internal/codeintel/commitgraph/main_test.go
View File

@ -1,17 +0,0 @@
package commitgraph
import (
"flag"
"os"
"testing"
"github.com/inconshreveable/log15"
)
func TestMain(m *testing.M) {
flag.Parse()
if !testing.Verbose() {
log15.Root().SetHandler(log15.DiscardHandler())
}
os.Exit(m.Run())
}

23
enterprise/cmd/worker/internal/codeintel/commitgraph_config.go
View File

@ -1,23 +0,0 @@
package codeintel
import (
"time"
"github.com/sourcegraph/sourcegraph/internal/env"
)
type commitGraphConfig struct {
env.BaseConfig
MaxAgeForNonStaleBranches time.Duration
MaxAgeForNonStaleTags time.Duration
CommitGraphUpdateTaskInterval time.Duration
}
var commitGraphConfigInst = &commitGraphConfig{}
func (c *commitGraphConfig) Load() {
c.MaxAgeForNonStaleBranches = c.GetInterval("PRECISE_CODE_INTEL_MAX_AGE_FOR_NON_STALE_BRANCHES", "2160h", "The age after which a branch should be considered stale. Code intelligence indexes will be evicted from stale branches.") // about 3 months
c.MaxAgeForNonStaleTags = c.GetInterval("PRECISE_CODE_INTEL_MAX_AGE_FOR_NON_STALE_TAGS", "8760h", "The age after which a tagged commit should be considered stale. Code intelligence indexes will be evicted from stale tagged commits.") // about 1 year
c.CommitGraphUpdateTaskInterval = c.GetInterval("PRECISE_CODE_INTEL_COMMIT_GRAPH_UPDATE_TASK_INTERVAL", "10s", "The frequency with which to run periodic codeintel commit graph update tasks.")
}

87
enterprise/cmd/worker/internal/codeintel/commitgraph_job.go
View File

@ -1,87 +0,0 @@
package codeintel
import (
"context"
"time"
"github.com/inconshreveable/log15"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sourcegraph/sourcegraph/cmd/worker/job"
"github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/codeintel"
workerdb "github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/db"
"github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/internal/codeintel/commitgraph"
"github.com/sourcegraph/sourcegraph/internal/database/locker"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/log"
)
type commitGraphJob struct{}
func NewCommitGraphJob() job.Job {
return &commitGraphJob{}
}
func (j *commitGraphJob) Description() string {
return ""
}
func (j *commitGraphJob) Config() []env.Config {
return []env.Config{commitGraphConfigInst}
}
func (j *commitGraphJob) Routines(ctx context.Context, logger log.Logger) ([]goroutine.BackgroundRoutine, error) {
observationContext := &observation.Context{
Logger: logger.Scoped("routines", "commit graph job routines"),
Tracer: &trace.Tracer{Tracer: opentracing.GlobalTracer()},
Registerer: prometheus.DefaultRegisterer,
}
db, err := workerdb.Init()
if err != nil {
return nil, err
}
dbStore, err := codeintel.InitDBStore()
if err != nil {
return nil, err
}
locker := locker.NewWithDB(db, "codeintel")
gitserverClient, err := codeintel.InitGitserverClient()
if err != nil {
return nil, err
}
routines := []goroutine.BackgroundRoutine{
commitgraph.NewUpdater(
dbStore,
locker,
gitserverClient,
commitGraphConfigInst.MaxAgeForNonStaleBranches,
commitGraphConfigInst.MaxAgeForNonStaleTags,
commitGraphConfigInst.CommitGraphUpdateTaskInterval,
observationContext,
),
}
observationContext.Registerer.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "src_codeintel_commit_graph_queued_duration_seconds_total",
Help: "The maximum amount of time a repository has had a stale commit graph.",
}, func() float64 {
age, err := dbStore.MaxStaleAge(context.Background())
if err != nil {
log15.Error("Failed to determine stale commit graph age", "error", err)
return 0
}
return float64(age) / float64(time.Second)
}))
return routines, nil
}

33
enterprise/cmd/worker/internal/codeintel/fresh/commitgraph_updater_job.go
View File

@ -3,10 +3,18 @@ package codeintel
import (
"context"
"github.com/opentracing/opentracing-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/sourcegraph/sourcegraph/cmd/worker/job"
"github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/codeintel"
workerdb "github.com/sourcegraph/sourcegraph/cmd/worker/shared/init/db"
"github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/commitgraph"
"github.com/sourcegraph/sourcegraph/internal/database/locker"
"github.com/sourcegraph/sourcegraph/internal/env"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/internal/trace"
"github.com/sourcegraph/sourcegraph/lib/log"
)
@ -27,7 +35,30 @@ func (j *commitGraphUpdaterJob) Config() []env.Config {
}
func (j *commitGraphUpdaterJob) Routines(ctx context.Context, logger log.Logger) ([]goroutine.BackgroundRoutine, error) {
observationContext := &observation.Context{
Logger: logger.Scoped("routines", "commit graph job routines"),
Tracer: &trace.Tracer{Tracer: opentracing.GlobalTracer()},
Registerer: prometheus.DefaultRegisterer,
}
dbStore, err := codeintel.InitDBStore()
if err != nil {
return nil, err
}
operations := commitgraph.NewOperations(dbStore, observationContext)
workerDb, err := workerdb.Init()
if err != nil {
return nil, err
}
locker := locker.NewWithDB(workerDb, "codeintel")
gitserverClient, err := codeintel.InitGitserverClient()
if err != nil {
return nil, err
}
return []goroutine.BackgroundRoutine{
commitgraph.NewUpdater(),
commitgraph.NewUpdater(dbStore, locker, gitserverClient, operations),
}, nil
}

1
enterprise/cmd/worker/main.go
View File

@ -39,7 +39,6 @@ func main() {
go setAuthzProviders()
additionalJobs := map[string]job.Job{
"codeintel-commitgraph": codeintel.NewCommitGraphJob(),
"codeintel-janitor": codeintel.NewJanitorJob(),
"codeintel-auto-indexing": codeintel.NewIndexingJob(),
"codehost-version-syncing": versions.NewSyncingJob(),

12
internal/codeintel/uploads/background/commitgraph/config.go
View File

@ -9,11 +9,21 @@ import (
type config struct {
env.BaseConfig
Interval time.Duration
Interval time.Duration
MaxAgeForNonStaleBranches time.Duration
MaxAgeForNonStaleTags time.Duration
CommitGraphUpdateTaskInterval time.Duration
}
var ConfigInst = &config{}
func (c *config) Load() {
maxAgeForNonStaleBranches := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_COMMITGRAPH_MAX_AGE_FOR_NON_STALE_BRANCHES", "PRECISE_CODE_INTEL_MAX_AGE_FOR_NON_STALE_BRANCHES")
maxAgeForNonStaleTags := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_COMMITGRAPH_MAX_AGE_FOR_NON_STALE_TAGS", "PRECISE_CODE_INTEL_MAX_AGE_FOR_NON_STALE_TAGS")
commitGraphUpdateTaskInterval := env.ChooseFallbackVariableName("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATE_TASK_INTERVAL", "PRECISE_CODE_INTEL_COMMIT_GRAPH_UPDATE_TASK_INTERVAL")
c.Interval = c.GetInterval("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATER_INTERVAL", "1s", "How frequently to run the upload commitgraph updater routine.")
c.MaxAgeForNonStaleBranches = c.GetInterval(maxAgeForNonStaleBranches, "2160h", "The age after which a branch should be considered stale. Code intelligence indexes will be evicted from stale branches.") // about 3 months
c.MaxAgeForNonStaleTags = c.GetInterval(maxAgeForNonStaleTags, "8760h", "The age after which a tagged commit should be considered stale. Code intelligence indexes will be evicted from stale tagged commits.") // about 1 year
c.CommitGraphUpdateTaskInterval = c.GetInterval(commitGraphUpdateTaskInterval, "10s", "The frequency with which to run periodic codeintel commit graph update tasks.")
}

3
internal/codeintel/uploads/background/commitgraph/gen.go Normal file
View File

@ -0,0 +1,3 @@
package commitgraph
//go:generate ../../../../../dev/mockgen.sh github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/commitgraph -i DBStore -i Locker -i GitserverClient -o mock_iface_test.go

1
enterprise/cmd/worker/internal/codeintel/commitgraph/iface.go → internal/codeintel/uploads/background/commitgraph/iface.go
View File

@ -20,6 +20,7 @@ type DBStore interface {
dirtyToken int,
) error
GetOldestCommitDate(ctx context.Context, repositoryID int) (time.Time, bool, error)
MaxStaleAge(ctx context.Context) (_ time.Duration, err error)
}
type Locker interface {

9
internal/codeintel/uploads/background/commitgraph/init.go
View File

@ -6,6 +6,11 @@ import (
"github.com/sourcegraph/sourcegraph/internal/goroutine"
)
func NewUpdater() goroutine.BackgroundRoutine {
return goroutine.NewPeriodicGoroutine(context.Background(), ConfigInst.Interval, &updater{})
func NewUpdater(dbStore DBStore, locker Locker, gitserverClient GitserverClient, operation *operations) goroutine.BackgroundRoutine {
return goroutine.NewPeriodicGoroutine(context.Background(), ConfigInst.Interval, &updater{
dbStore: dbStore,
locker: locker,
gitserverClient: gitserverClient,
operations: operation,
})
}

127
enterprise/cmd/worker/internal/codeintel/commitgraph/mock_iface_test.go → internal/codeintel/uploads/background/commitgraph/mock_iface_test.go generated
View File

@ -14,7 +14,7 @@ import (
// MockDBStore is a mock implementation of the DBStore interface (from the
// package
// github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/internal/codeintel/commitgraph)
// github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/commitgraph)
// used for unit testing.
type MockDBStore struct {
// CalculateVisibleUploadsFunc is an instance of a mock function object
@ -26,6 +26,9 @@ type MockDBStore struct {
// GetOldestCommitDateFunc is an instance of a mock function object
// controlling the behavior of the method GetOldestCommitDate.
GetOldestCommitDateFunc *DBStoreGetOldestCommitDateFunc
// MaxStaleAgeFunc is an instance of a mock function object controlling
// the behavior of the method MaxStaleAge.
MaxStaleAgeFunc *DBStoreMaxStaleAgeFunc
}
// NewMockDBStore creates a new mock of the DBStore interface. All methods
@ -47,6 +50,11 @@ func NewMockDBStore() *MockDBStore {
return
},
},
MaxStaleAgeFunc: &DBStoreMaxStaleAgeFunc{
defaultHook: func(context.Context) (r0 time.Duration, r1 error) {
return
},
},
}
}
@ -69,6 +77,11 @@ func NewStrictMockDBStore() *MockDBStore {
panic("unexpected invocation of MockDBStore.GetOldestCommitDate")
},
},
MaxStaleAgeFunc: &DBStoreMaxStaleAgeFunc{
defaultHook: func(context.Context) (time.Duration, error) {
panic("unexpected invocation of MockDBStore.MaxStaleAge")
},
},
}
}
@ -85,6 +98,9 @@ func NewMockDBStoreFrom(i DBStore) *MockDBStore {
GetOldestCommitDateFunc: &DBStoreGetOldestCommitDateFunc{
defaultHook: i.GetOldestCommitDate,
},
MaxStaleAgeFunc: &DBStoreMaxStaleAgeFunc{
defaultHook: i.MaxStaleAge,
},
}
}
@ -426,9 +442,114 @@ func (c DBStoreGetOldestCommitDateFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1, c.Result2}
}
// DBStoreMaxStaleAgeFunc describes the behavior when the MaxStaleAge method
// of the parent MockDBStore instance is invoked.
type DBStoreMaxStaleAgeFunc struct {
defaultHook func(context.Context) (time.Duration, error)
hooks []func(context.Context) (time.Duration, error)
history []DBStoreMaxStaleAgeFuncCall
mutex sync.Mutex
}
// MaxStaleAge delegates to the next hook function in the queue and stores
// the parameter and result values of this invocation.
func (m *MockDBStore) MaxStaleAge(v0 context.Context) (time.Duration, error) {
r0, r1 := m.MaxStaleAgeFunc.nextHook()(v0)
m.MaxStaleAgeFunc.appendCall(DBStoreMaxStaleAgeFuncCall{v0, r0, r1})
return r0, r1
}
// SetDefaultHook sets function that is called when the MaxStaleAge method
// of the parent MockDBStore instance is invoked and the hook queue is
// empty.
func (f *DBStoreMaxStaleAgeFunc) SetDefaultHook(hook func(context.Context) (time.Duration, error)) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// MaxStaleAge method of the parent MockDBStore instance invokes the hook at
// the front of the queue and discards it. After the queue is empty, the
// default hook function is invoked for any future action.
func (f *DBStoreMaxStaleAgeFunc) PushHook(hook func(context.Context) (time.Duration, error)) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *DBStoreMaxStaleAgeFunc) SetDefaultReturn(r0 time.Duration, r1 error) {
f.SetDefaultHook(func(context.Context) (time.Duration, error) {
return r0, r1
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *DBStoreMaxStaleAgeFunc) PushReturn(r0 time.Duration, r1 error) {
f.PushHook(func(context.Context) (time.Duration, error) {
return r0, r1
})
}
func (f *DBStoreMaxStaleAgeFunc) nextHook() func(context.Context) (time.Duration, error) {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *DBStoreMaxStaleAgeFunc) appendCall(r0 DBStoreMaxStaleAgeFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of DBStoreMaxStaleAgeFuncCall objects
// describing the invocations of this function.
func (f *DBStoreMaxStaleAgeFunc) History() []DBStoreMaxStaleAgeFuncCall {
f.mutex.Lock()
history := make([]DBStoreMaxStaleAgeFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// DBStoreMaxStaleAgeFuncCall is an object that describes an invocation of
// method MaxStaleAge on an instance of MockDBStore.
type DBStoreMaxStaleAgeFuncCall struct {
// Arg0 is the value of the 1st argument passed to this method
// invocation.
Arg0 context.Context
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 time.Duration
// Result1 is the value of the 2nd result returned from this method
// invocation.
Result1 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c DBStoreMaxStaleAgeFuncCall) Args() []interface{} {
return []interface{}{c.Arg0}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c DBStoreMaxStaleAgeFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1}
}
// MockGitserverClient is a mock implementation of the GitserverClient
// interface (from the package
// github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/internal/codeintel/commitgraph)
// github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/commitgraph)
// used for unit testing.
type MockGitserverClient struct {
// CommitGraphFunc is an instance of a mock function object controlling
@ -721,7 +842,7 @@ func (c GitserverClientRefDescriptionsFuncCall) Results() []interface{} {
// MockLocker is a mock implementation of the Locker interface (from the
// package
// github.com/sourcegraph/sourcegraph/enterprise/cmd/worker/internal/codeintel/commitgraph)
// github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/commitgraph)
// used for unit testing.
type MockLocker struct {
// LockFunc is an instance of a mock function object controlling the

16
enterprise/cmd/worker/internal/codeintel/commitgraph/observability.go → internal/codeintel/uploads/background/commitgraph/operations.go
View File

@ -2,6 +2,7 @@ package commitgraph
import (
"context"
"time"
"github.com/inconshreveable/log15"
"github.com/prometheus/client_golang/prometheus"
@ -14,7 +15,7 @@ type operations struct {
commitUpdate *observation.Operation
}
func newOperations(dbStore DBStore, observationContext *observation.Context) *operations {
func NewOperations(dbStore DBStore, observationContext *observation.Context) *operations {
commitUpdate := observationContext.Operation(observation.Op{
Name: "codeintel.commitUpdater",
Metrics: metrics.NewREDMetrics(
@ -36,6 +37,19 @@ func newOperations(dbStore DBStore, observationContext *observation.Context) *op
return float64(len(dirtyRepositories))
}))
observationContext.Registerer.MustRegister(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "src_codeintel_commit_graph_queued_duration_seconds_total",
Help: "The maximum amount of time a repository has had a stale commit graph.",
}, func() float64 {
age, err := dbStore.MaxStaleAge(context.Background())
if err != nil {
log15.Error("Failed to determine stale commit graph age", "error", err)
return 0
}
return float64(age) / float64(time.Second)
}))
return &operations{
commitUpdate: commitUpdate,
}

23
internal/codeintel/uploads/background/commitgraph/updater.go
View File

@ -6,15 +6,24 @@ import (
"github.com/sourcegraph/sourcegraph/internal/goroutine"
)
type updater struct{}
type updater struct {
dbStore DBStore
locker Locker
gitserverClient GitserverClient
operations *operations
}
var _ goroutine.Handler = &updater{}
var _ goroutine.ErrorHandler = &updater{}
var (
_ goroutine.Handler = &updater{}
_ goroutine.ErrorHandler = &updater{}
)
func (u *updater) Handle(ctx context.Context) error {
if err := u.HandleUpdater(ctx); err != nil {
return err
}
func (r *updater) Handle(ctx context.Context) error {
// To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375
return nil
}
func (r *updater) HandleError(err error) {
}
func (u *updater) HandleError(err error) {}

53
enterprise/cmd/worker/internal/codeintel/commitgraph/updater.go → internal/codeintel/uploads/background/commitgraph/updater_job.go
View File

@ -9,7 +9,6 @@ import (
"github.com/sourcegraph/sourcegraph/internal/gitserver"
"github.com/sourcegraph/sourcegraph/internal/gitserver/gitdomain"
"github.com/sourcegraph/sourcegraph/internal/goroutine"
"github.com/sourcegraph/sourcegraph/internal/observation"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -19,43 +18,9 @@ import (
// the upload processing as it is likely that we are processing multiple uploads concurrently
// for the same repository and should not repeat the work since the last calculation performed
// will always be the one we want.
type Updater struct {
dbStore DBStore
locker Locker
gitserverClient GitserverClient
maxAgeForNonStaleBranches time.Duration
maxAgeForNonStaleTags time.Duration
operations *operations
}
var (
_ goroutine.Handler = &Updater{}
_ goroutine.ErrorHandler = &Updater{}
)
// NewUpdater returns a background routine that periodically updates the commit graph
// and visible uploads for each repository marked as dirty.
func NewUpdater(
dbStore DBStore,
locker Locker,
gitserverClient GitserverClient,
maxAgeForNonStaleBranches time.Duration,
maxAgeForNonStaleTags time.Duration,
interval time.Duration,
observationContext *observation.Context,
) goroutine.BackgroundRoutine {
return goroutine.NewPeriodicGoroutine(context.Background(), interval, &Updater{
dbStore: dbStore,
locker: locker,
gitserverClient: gitserverClient,
maxAgeForNonStaleBranches: maxAgeForNonStaleBranches,
maxAgeForNonStaleTags: maxAgeForNonStaleTags,
operations: newOperations(dbStore, observationContext),
})
}
// Handle checks for dirty repositories and invokes the underlying updater on each one.
func (u *Updater) Handle(ctx context.Context) error {
// HandleUpdater checks for dirty repositories and invokes the underlying updater on each one.
func (u *updater) HandleUpdater(ctx context.Context) error {
repositoryIDs, err := u.dbStore.DirtyRepositories(ctx)
if err != nil {
return errors.Wrap(err, "dbstore.DirtyRepositories")
@ -75,14 +40,14 @@ func (u *Updater) Handle(ctx context.Context) error {
return updateErr
}
func (u *Updater) HandleError(err error) {
log15.Error("Failed to run update process", "err", err)
}
// func (u *updater) HandleError(err error) {
// log15.Error("Failed to run update process", "err", err)
// }
// tryUpdate will call update while holding an advisory lock to give exclusive access to the
// update procedure for this repository. If the lock is already held, this method will simply
// do nothing.
func (u *Updater) tryUpdate(ctx context.Context, repositoryID, dirtyToken int) (err error) {
func (u *updater) tryUpdate(ctx context.Context, repositoryID, dirtyToken int) (err error) {
ok, unlock, err := u.locker.Lock(ctx, int32(repositoryID), false)
if err != nil || !ok {
return errors.Wrap(err, "locker.Lock")
@ -101,7 +66,7 @@ func (u *Updater) tryUpdate(ctx context.Context, repositoryID, dirtyToken int) (
// The user should supply a dirty token that is associated with the given repository so that
// the repository can be unmarked as long as the repository is not marked as dirty again before
// the update completes.
func (u *Updater) update(ctx context.Context, repositoryID, dirtyToken int) (err error) {
func (u *updater) update(ctx context.Context, repositoryID, dirtyToken int) (err error) {
ctx, trace, endObservation := u.operations.commitUpdate.With(ctx, &err, observation.Args{
LogFields: []log.Field{
log.Int("repositoryID", repositoryID),
@ -126,7 +91,7 @@ func (u *Updater) update(ctx context.Context, repositoryID, dirtyToken int) (err
// Decorate the commit graph with the set of processed uploads are visible from each commit,
// then bulk update the denormalized view in Postgres. We call this with an empty graph as well
// so that we end up clearing the stale data and bulk inserting nothing.
if err := u.dbStore.CalculateVisibleUploads(ctx, repositoryID, commitGraph, refDescriptions, u.maxAgeForNonStaleBranches, u.maxAgeForNonStaleTags, dirtyToken); err != nil {
if err := u.dbStore.CalculateVisibleUploads(ctx, repositoryID, commitGraph, refDescriptions, ConfigInst.MaxAgeForNonStaleBranches, ConfigInst.MaxAgeForNonStaleTags, dirtyToken); err != nil {
return errors.Wrap(err, "dbstore.CalculateVisibleUploads")
}
@ -144,7 +109,7 @@ func (u *Updater) update(ctx context.Context, repositoryID, dirtyToken int) (err
// The number of commits pulled back here should not grow over time unless the repo is growing at an
// accelerating rate, as we routinely expire old information for active repositories in a janitor
// process.
func (u *Updater) getCommitGraph(ctx context.Context, repositoryID int) (*gitdomain.CommitGraph, error) {
func (u *updater) getCommitGraph(ctx context.Context, repositoryID int) (*gitdomain.CommitGraph, error) {
commitDate, ok, err := u.dbStore.GetOldestCommitDate(ctx, repositoryID)
if err != nil {
return nil, err

12
enterprise/cmd/worker/internal/codeintel/commitgraph/updater_test.go → internal/codeintel/uploads/background/commitgraph/updater_job_test.go
View File

@ -29,11 +29,11 @@ func TestUpdater(t *testing.T) {
"b": {{IsDefaultBranch: true}},
}, nil)
updater := &Updater{
updater := &updater{
dbStore: mockDBStore,
locker: mockLocker,
gitserverClient: mockGitserverClient,
operations: newOperations(mockDBStore, &observation.TestContext),
operations: NewOperations(mockDBStore, &observation.TestContext),
}
if err := updater.Handle(context.Background()); err != nil {
@ -75,11 +75,11 @@ func TestUpdaterNoUploads(t *testing.T) {
"b": {{IsDefaultBranch: true}},
}, nil)
updater := &Updater{
updater := &updater{
dbStore: mockDBStore,
locker: mockLocker,
gitserverClient: mockGitserverClient,
operations: newOperations(mockDBStore, &observation.TestContext),
operations: NewOperations(mockDBStore, &observation.TestContext),
}
if err := updater.Handle(context.Background()); err != nil {
@ -108,11 +108,11 @@ func TestUpdaterLocked(t *testing.T) {
"b": {{IsDefaultBranch: true}},
}, nil)
updater := &Updater{
updater := &updater{
dbStore: mockDBStore,
locker: mockLocker,
gitserverClient: mockGitserverClient,
operations: newOperations(mockDBStore, &observation.TestContext),
operations: NewOperations(mockDBStore, &observation.TestContext),
}
if err := updater.Handle(context.Background()); err != nil {

1
monitoring/definitions/codeintel_uploads.go
View File

@ -17,6 +17,7 @@ func CodeIntelUploads() *monitoring.Dashboard {
shared.CodeIntelligence.NewUploadsGraphQLTransportGroup(""),
shared.CodeIntelligence.NewUploadsHTTPTransportGroup(""),
shared.CodeIntelligence.NewUploadsCleanupTaskGroup(""),
shared.CodeIntelligence.NewCommitGraphQueueGroup(""),
shared.CodeIntelligence.NewUploadsExpirationTaskGroup(""),
},
}