artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 19:21:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

ranking: Add cron expression for ranking job (#52738)

Browse Source
This commit is contained in:
Eric Fritz 2023-06-01 15:38:01 -05:00 committed by GitHub
parent d92faf4036
commit 462df98fa7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 136 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
enterprise/internal/codeintel/ranking/internal/background/coordinator/BUILD.bazel generated
View File

@ -15,5 +15,6 @@ go_library(
"//internal/env",
"//internal/goroutine",
"//internal/observation",
"@com_github_sourcegraph_log//:log",
],
)

23
enterprise/internal/codeintel/ranking/internal/background/coordinator/job.go
View File

@ -4,6 +4,8 @@ import (
"context"
"time"
"github.com/sourcegraph/log"
rankingshared "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/shared"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/store"
"github.com/sourcegraph/sourcegraph/internal/conf"
@ -28,12 +30,29 @@ func NewCoordinator(
return nil
}
derivativeGraphKeyPrefix, err := store.DerivativeGraphKey(ctx, s)
if expr, err := conf.CodeIntelRankingDocumentReferenceCountsCronExpression(); err != nil {
observationCtx.Logger.Warn("Illegal ranking cron expression", log.Error(err))
} else {
_, previous, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return err
}
if delta := time.Until(expr.Next(previous)); delta <= 0 {
observationCtx.Logger.Info("Starting a new ranking calculation", log.Int("seconds overdue", -int(delta/time.Second)))
if err := s.BumpDerivativeGraphKey(ctx); err != nil {
return err
}
}
}
derivativeGraphKeyPrefix, _, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return err
}
return s.Coordinate(ctx, rankingshared.DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix, time.Now()))
return s.Coordinate(ctx, rankingshared.DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix))
}),
)
}

13
enterprise/internal/codeintel/ranking/internal/background/janitor/job.go
View File

@ -2,7 +2,6 @@ package janitor
import (
"context"
"time"
rankingshared "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/shared"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/store"
@ -118,12 +117,12 @@ func vacuumDeletedExportedUploads(ctx context.Context, s store.Store) (int, erro
return 0, nil
}
derivativeGraphKeyPrefix, err := store.DerivativeGraphKey(ctx, s)
derivativeGraphKeyPrefix, _, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return 0, err
}
return s.VacuumDeletedExportedUploads(ctx, rankingshared.DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix, time.Now()))
return s.VacuumDeletedExportedUploads(ctx, rankingshared.DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix))
}
const vacuumBatchSize = 100 // TODO - configure via envvar
@ -141,12 +140,12 @@ func vacuumStaleGraphs(ctx context.Context, s store.Store) (int, error) {
return 0, nil
}
derivativeGraphKeyPrefix, err := store.DerivativeGraphKey(ctx, s)
derivativeGraphKeyPrefix, _, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return 0, err
}
return s.VacuumStaleGraphs(ctx, rankingshared.DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix, time.Now()), vacuumBatchSize)
return s.VacuumStaleGraphs(ctx, rankingshared.DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix), vacuumBatchSize)
}
func vacuumStaleRanks(ctx context.Context, s store.Store) (int, int, error) {
@ -154,10 +153,10 @@ func vacuumStaleRanks(ctx context.Context, s store.Store) (int, int, error) {
return 0, 0, nil
}
derivativeGraphKeyPrefix, err := store.DerivativeGraphKey(ctx, s)
derivativeGraphKeyPrefix, _, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return 0, 0, err
}
return s.VacuumStaleRanks(ctx, rankingshared.DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix, time.Now()))
return s.VacuumStaleRanks(ctx, rankingshared.DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix))
}

9
enterprise/internal/codeintel/ranking/internal/background/mapper/job.go
View File

@ -2,7 +2,6 @@ package mapper
import (
"context"
"time"
rankingshared "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/shared"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/store"
@ -71,14 +70,14 @@ func mapInitializerRankingGraph(
return 0, 0, nil
}
derivativeGraphKeyPrefix, err := store.DerivativeGraphKey(ctx, s)
derivativeGraphKeyPrefix, _, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return 0, 0, err
}
return s.InsertInitialPathCounts(
ctx,
rankingshared.DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix, time.Now()),
rankingshared.DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix),
batchSize,
)
}
@ -92,14 +91,14 @@ func mapRankingGraph(
return 0, 0, nil
}
derivativeGraphKeyPrefix, err := store.DerivativeGraphKey(ctx, s)
derivativeGraphKeyPrefix, _, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return 0, 0, err
}
return s.InsertPathCountInputs(
ctx,
rankingshared.DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix, time.Now()),
rankingshared.DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix),
batchSize,
)
}

5
enterprise/internal/codeintel/ranking/internal/background/reducer/job.go
View File

@ -2,7 +2,6 @@ package reducer
import (
"context"
"time"
rankingshared "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/shared"
"github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/ranking/internal/store"
@ -40,14 +39,14 @@ func reduceRankingGraph(
return 0, 0, nil
}
derivativeGraphKeyPrefix, err := store.DerivativeGraphKey(ctx, s)
derivativeGraphKeyPrefix, _, err := store.DerivativeGraphKey(ctx, s)
if err != nil {
return 0, 0, err
}
return s.InsertPathRanks(
ctx,
rankingshared.DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix, time.Now()),
rankingshared.DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix),
batchSize,
)
}

25
enterprise/internal/codeintel/ranking/internal/shared/keys.go
View File

@ -3,7 +3,6 @@ package shared
import (
"fmt"
"strings"
"time"
"github.com/sourcegraph/sourcegraph/internal/conf"
)
@ -16,12 +15,11 @@ func NewGraphKey(graphKey string) string {
// NewDerivativeGraphKey creates a new derivative graph key. This key identifies work related
// to ranking, excluding the SCIP export tasks, which are identified by the same root graph key
// but with different bucket or derivative graph key prefix values.
func NewDerivativeGraphKey(graphKey, derivativeGraphKeyPrefix string, bucket int64) string {
return fmt.Sprintf("%s.%s-%d",
// but with different derivative graph key prefix values.
func NewDerivativeGraphKey(graphKey, derivativeGraphKeyPrefix string) string {
return fmt.Sprintf("%s.%s",
encode(graphKey),
encode(derivativeGraphKeyPrefix),
bucket,
)
}
@ -30,18 +28,15 @@ func GraphKey() string {
return NewGraphKey(conf.CodeIntelRankingDocumentReferenceCountsGraphKey())
}
// DerivativeGraphKeyFromTime returns a derivative key from the configured root used for exports
// as well as the current "bucket" of time containing the current instant. Each bucket of time is
// the same configurable length, packed end-to-end since the Unix epoch.
// DerivativeGraphKeyFromPrefix returns a derivative key from the configured root used for exports
// as well as the current "bucket" of time containing the current instant identified by the given
// prefix.
//
// Constructing a graph key for the mapper and reducer jobs in this way ensures that begin a fresh
// map/reduce job on a periodic cadence (equal to the bucket length). Changing the root graph key
// will also create a new map/reduce job (without switching buckets).
func DerivativeGraphKeyFromTime(derivativeGraphKeyPrefix string, now time.Time) string {
graphKey := conf.CodeIntelRankingDocumentReferenceCountsGraphKey()
bucket := now.UTC().Unix() / int64(conf.CodeIntelRankingStaleResultAge().Seconds())
return NewDerivativeGraphKey(graphKey, derivativeGraphKeyPrefix, bucket)
// map/reduce job on a periodic cadence (determined by a cron-like site config setting). Changing
// the root graph key will also create a new map/reduce job.
func DerivativeGraphKeyFromPrefix(derivativeGraphKeyPrefix string) string {
return NewDerivativeGraphKey(conf.CodeIntelRankingDocumentReferenceCountsGraphKey(), derivativeGraphKeyPrefix)
}
// GraphKeyFromDerivativeGraphKey returns the root of the given derivative graph key.

6
enterprise/internal/codeintel/ranking/internal/store/coordinator_test.go
View File

@ -31,9 +31,9 @@ func TestCoordinateAndSummaries(t *testing.T) {
now3 := now2.Add(time.Hour * 5)
now4 := now2.Add(time.Hour * 7)
key1 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key2 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 234)
key3 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 345)
key1 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
key2 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "234")
key3 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "345")
//
// Insert data

31
enterprise/internal/codeintel/ranking/internal/store/graph_keys.go
View File

@ -2,6 +2,7 @@ package store
import (
"context"
"time"
"github.com/google/uuid"
"github.com/keegancsmith/sqlf"
@ -10,29 +11,43 @@ import (
"github.com/sourcegraph/sourcegraph/internal/observation"
)
func DerivativeGraphKey(ctx context.Context, store Store) (string, error) {
if key, ok, err := store.DerivativeGraphKey(ctx); err != nil {
return "", err
func DerivativeGraphKey(ctx context.Context, store Store) (string, time.Time, error) {
if key, createdAt, ok, err := store.DerivativeGraphKey(ctx); err != nil {
return "", time.Time{}, err
} else if ok {
return key, nil
return key, createdAt, nil
}
if err := store.BumpDerivativeGraphKey(ctx); err != nil {
return "", err
return "", time.Time{}, err
}
return DerivativeGraphKey(ctx, store)
}
func (s *store) DerivativeGraphKey(ctx context.Context) (_ string, _ bool, err error) {
func (s *store) DerivativeGraphKey(ctx context.Context) (graphKey string, createdAt time.Time, _ bool, err error) {
ctx, _, endObservation := s.operations.derivativeGraphKey.With(ctx, &err, observation.Args{})
defer endObservation(1, observation.Args{})
return basestore.ScanFirstString(s.db.Query(ctx, sqlf.Sprintf(derivativeGraphKeyQuery)))
rows, err := s.db.Query(ctx, sqlf.Sprintf(derivativeGraphKeyQuery))
if err != nil {
return "", time.Time{}, false, err
}
defer func() { err = basestore.CloseRows(rows, err) }()
if rows.Next() {
if err := rows.Scan(&graphKey, &createdAt); err != nil {
return "", time.Time{}, false, err
}
return graphKey, createdAt, true, nil
}
return "", time.Time{}, false, nil
}
const derivativeGraphKeyQuery = `
SELECT graph_key
SELECT graph_key, created_at
FROM codeintel_ranking_graph_keys
ORDER BY created_at DESC
LIMIT 1

2
enterprise/internal/codeintel/ranking/internal/store/graph_keys_test.go
View File

@ -17,7 +17,7 @@ func TestDerivativeGraphKey(t *testing.T) {
db := database.NewDB(logger, dbtest.NewDB(logger, t))
store := New(&observation.TestContext, db)
if _, err := DerivativeGraphKey(ctx, store); err != nil {
if _, _, err := DerivativeGraphKey(ctx, store); err != nil {
t.Fatalf("unexpected error: %s", err)
}
}

12
enterprise/internal/codeintel/ranking/internal/store/mapper_test.go
View File

@ -27,7 +27,7 @@ func TestInsertPathCountInputs(t *testing.T) {
db := database.NewDB(logger, dbtest.NewDB(logger, t))
store := New(&observation.TestContext, db)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
// Insert and export uploads
insertUploads(t, db,
@ -200,7 +200,7 @@ func TestInsertInitialPathCounts(t *testing.T) {
db := database.NewDB(logger, dbtest.NewDB(logger, t))
store := New(&observation.TestContext, db)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
// Insert and export upload
// N.B. This creates repository 50 implicitly
@ -258,7 +258,7 @@ func TestVacuumStaleGraphs(t *testing.T) {
db := database.NewDB(logger, dbtest.NewDB(logger, t))
store := New(&observation.TestContext, db)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
// Insert and export uploads
insertUploads(t, db,
@ -303,8 +303,8 @@ func TestVacuumStaleGraphs(t *testing.T) {
for _, graphKey := range []string{
key,
rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 456),
rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 789),
rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "456"),
rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "789"),
} {
if _, err := db.ExecContext(ctx, `
INSERT INTO codeintel_ranking_references_processed (graph_key, codeintel_ranking_reference_id)
@ -336,7 +336,7 @@ func TestVacuumStaleGraphs(t *testing.T) {
assertCounts(3 * 30)
// remove records associated with other ranking keys
if _, err := store.VacuumStaleGraphs(ctx, rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 456), 50); err != nil {
if _, err := store.VacuumStaleGraphs(ctx, rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "456"), 50); err != nil {
t.Fatalf("unexpected error vacuuming stale graphs: %s", err)
}

10
enterprise/internal/codeintel/ranking/internal/store/reducer_test.go
View File

@ -29,7 +29,7 @@ func TestInsertPathRanks(t *testing.T) {
db := database.NewDB(logger, dbtest.NewDB(logger, t))
store := New(&observation.TestContext, db)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
// Insert and export upload
insertUploads(t, db, uploadsshared.Upload{ID: 4})
@ -144,10 +144,10 @@ func TestVacuumStaleRanks(t *testing.T) {
t.Fatalf("failed to insert repos: %s", err)
}
key1 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key2 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 234)
key3 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 345)
key4 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 456)
key1 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
key2 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "234")
key3 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "345")
key4 := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "456")
// Insert metadata to rank progress by completion date
if _, err := db.ExecContext(ctx, `

6
enterprise/internal/codeintel/ranking/internal/store/retrieval_test.go
View File

@ -74,7 +74,7 @@ func TestDocumentRanks(t *testing.T) {
store := New(&observation.TestContext, db)
repoName := api.RepoName("foo")
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
if _, err := db.ExecContext(ctx, `
INSERT INTO codeintel_ranking_progress(graph_key, max_export_id, mappers_started_at, reducer_completed_at)
@ -130,7 +130,7 @@ func TestGetReferenceCountStatistics(t *testing.T) {
db := database.NewDB(logger, dbtest.NewDB(logger, t))
store := New(&observation.TestContext, db)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
if _, err := db.ExecContext(ctx, `
INSERT INTO codeintel_ranking_progress(graph_key, max_export_id, mappers_started_at, reducer_completed_at)
@ -175,7 +175,7 @@ func TestLastUpdatedAt(t *testing.T) {
db := database.NewDB(logger, dbtest.NewDB(logger, t))
store := New(&observation.TestContext, db)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123)
key := rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123")
if _, err := db.ExecContext(ctx, `
INSERT INTO codeintel_ranking_progress(graph_key, max_export_id, mappers_started_at, reducer_completed_at)

2
enterprise/internal/codeintel/ranking/internal/store/store.go
View File

@ -38,7 +38,7 @@ type Store interface {
InsertInitialPathRanks(ctx context.Context, exportedUploadID int, documentPaths []string, batchSize int, graphKey string) error
// Graph keys
DerivativeGraphKey(ctx context.Context) (string, bool, error)
DerivativeGraphKey(ctx context.Context) (string, time.Time, bool, error)
BumpDerivativeGraphKey(ctx context.Context) error
// Coordinates mapper+reducer phases

2
enterprise/internal/codeintel/ranking/internal/store/uploads_test.go
View File

@ -236,7 +236,7 @@ func TestVacuumDeletedExportedUploads(t *testing.T) {
// records only soft-deleted
assertCounts(9 + 9)
_, err = store.VacuumDeletedExportedUploads(ctx, rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "", 123))
_, err = store.VacuumDeletedExportedUploads(ctx, rankingshared.NewDerivativeGraphKey(mockRankingGraphKey, "123"))
if err != nil {
t.Fatalf("unexpected error vacuuming deleted uploads: %s", err)
}

43
enterprise/internal/codeintel/ranking/mocks_test.go generated
View File

@ -110,7 +110,7 @@ func NewMockStore() *MockStore {
},
},
DerivativeGraphKeyFunc: &StoreDerivativeGraphKeyFunc{
defaultHook: func(context.Context) (r0 string, r1 bool, r2 error) {
defaultHook: func(context.Context) (r0 string, r1 time.Time, r2 bool, r3 error) {
return
},
},
@ -222,7 +222,7 @@ func NewStrictMockStore() *MockStore {
},
},
DerivativeGraphKeyFunc: &StoreDerivativeGraphKeyFunc{
defaultHook: func(context.Context) (string, bool, error) {
defaultHook: func(context.Context) (string, time.Time, bool, error) {
panic("unexpected invocation of MockStore.DerivativeGraphKey")
},
},
@ -599,24 +599,24 @@ func (c StoreCoordinateFuncCall) Results() []interface{} {
// StoreDerivativeGraphKeyFunc describes the behavior when the
// DerivativeGraphKey method of the parent MockStore instance is invoked.
type StoreDerivativeGraphKeyFunc struct {
defaultHook func(context.Context) (string, bool, error)
hooks []func(context.Context) (string, bool, error)
defaultHook func(context.Context) (string, time.Time, bool, error)
hooks []func(context.Context) (string, time.Time, bool, error)
history []StoreDerivativeGraphKeyFuncCall
mutex sync.Mutex
}
// DerivativeGraphKey delegates to the next hook function in the queue and
// stores the parameter and result values of this invocation.
func (m *MockStore) DerivativeGraphKey(v0 context.Context) (string, bool, error) {
r0, r1, r2 := m.DerivativeGraphKeyFunc.nextHook()(v0)
m.DerivativeGraphKeyFunc.appendCall(StoreDerivativeGraphKeyFuncCall{v0, r0, r1, r2})
return r0, r1, r2
func (m *MockStore) DerivativeGraphKey(v0 context.Context) (string, time.Time, bool, error) {
r0, r1, r2, r3 := m.DerivativeGraphKeyFunc.nextHook()(v0)
m.DerivativeGraphKeyFunc.appendCall(StoreDerivativeGraphKeyFuncCall{v0, r0, r1, r2, r3})
return r0, r1, r2, r3
}
// SetDefaultHook sets function that is called when the DerivativeGraphKey
// method of the parent MockStore instance is invoked and the hook queue is
// empty.
func (f *StoreDerivativeGraphKeyFunc) SetDefaultHook(hook func(context.Context) (string, bool, error)) {
func (f *StoreDerivativeGraphKeyFunc) SetDefaultHook(hook func(context.Context) (string, time.Time, bool, error)) {
f.defaultHook = hook
}
@ -624,7 +624,7 @@ func (f *StoreDerivativeGraphKeyFunc) SetDefaultHook(hook func(context.Context)
// DerivativeGraphKey method of the parent MockStore instance invokes the
// hook at the front of the queue and discards it. After the queue is empty,
// the default hook function is invoked for any future action.
func (f *StoreDerivativeGraphKeyFunc) PushHook(hook func(context.Context) (string, bool, error)) {
func (f *StoreDerivativeGraphKeyFunc) PushHook(hook func(context.Context) (string, time.Time, bool, error)) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
@ -632,20 +632,20 @@ func (f *StoreDerivativeGraphKeyFunc) PushHook(hook func(context.Context) (strin
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *StoreDerivativeGraphKeyFunc) SetDefaultReturn(r0 string, r1 bool, r2 error) {
f.SetDefaultHook(func(context.Context) (string, bool, error) {
return r0, r1, r2
func (f *StoreDerivativeGraphKeyFunc) SetDefaultReturn(r0 string, r1 time.Time, r2 bool, r3 error) {
f.SetDefaultHook(func(context.Context) (string, time.Time, bool, error) {
return r0, r1, r2, r3
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *StoreDerivativeGraphKeyFunc) PushReturn(r0 string, r1 bool, r2 error) {
f.PushHook(func(context.Context) (string, bool, error) {
return r0, r1, r2
func (f *StoreDerivativeGraphKeyFunc) PushReturn(r0 string, r1 time.Time, r2 bool, r3 error) {
f.PushHook(func(context.Context) (string, time.Time, bool, error) {
return r0, r1, r2, r3
})
}
func (f *StoreDerivativeGraphKeyFunc) nextHook() func(context.Context) (string, bool, error) {
func (f *StoreDerivativeGraphKeyFunc) nextHook() func(context.Context) (string, time.Time, bool, error) {
f.mutex.Lock()
defer f.mutex.Unlock()
@ -686,10 +686,13 @@ type StoreDerivativeGraphKeyFuncCall struct {
Result0 string
// Result1 is the value of the 2nd result returned from this method
// invocation.
Result1 bool
Result1 time.Time
// Result2 is the value of the 3rd result returned from this method
// invocation.
Result2 error
Result2 bool
// Result3 is the value of the 4th result returned from this method
// invocation.
Result3 error
}
// Args returns an interface slice containing the arguments of this
@ -701,7 +704,7 @@ func (c StoreDerivativeGraphKeyFuncCall) Args() []interface{} {
// Results returns an interface slice containing the results of this
// invocation.
func (c StoreDerivativeGraphKeyFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1, c.Result2}
return []interface{}{c.Result0, c.Result1, c.Result2, c.Result3}
}
// StoreGetDocumentRanksFunc describes the behavior when the

1
go.mod
View File

@ -265,6 +265,7 @@ require (
require (
github.com/go-redsync/redsync/v4 v4.8.1
github.com/hashicorp/cronexpr v1.1.1
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
github.com/vektah/gqlparser/v2 v2.4.5
)

2
go.sum
View File

@ -1266,6 +1266,8 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt
github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c=
github.com/hashicorp/cronexpr v1.1.1/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=

1
internal/conf/BUILD.bazel generated
View File

@ -37,6 +37,7 @@ go_library(
"//schema",
"@com_github_getsentry_sentry_go//:sentry-go",
"@com_github_grafana_regexp//:regexp",
"@com_github_hashicorp_cronexpr//:cronexpr",
"@com_github_sourcegraph_jsonx//:jsonx",
"@com_github_sourcegraph_log//:log",
"@com_github_tidwall_gjson//:gjson",

10
internal/conf/computed.go
View File

@ -7,6 +7,8 @@ import (
"strings"
"time"
"github.com/hashicorp/cronexpr"
"github.com/sourcegraph/sourcegraph/internal/api/internalapi"
"github.com/sourcegraph/sourcegraph/internal/conf/confdefaults"
"github.com/sourcegraph/sourcegraph/internal/conf/conftypes"
@ -280,6 +282,14 @@ func CodeIntelRankingDocumentReferenceCountsEnabled() bool {
return false
}
func CodeIntelRankingDocumentReferenceCountsCronExpression() (*cronexpr.Expression, error) {
if cronExpression := Get().CodeIntelRankingDocumentReferenceCountsCronExpression; cronExpression != nil {
return cronexpr.Parse(*cronExpression)
}
return cronexpr.Parse("@weekly")
}
func CodeIntelRankingDocumentReferenceCountsGraphKey() string {
if val := Get().CodeIntelRankingDocumentReferenceCountsGraphKey; val != "" {
return val

3
schema/schema.go
View File

@ -2382,6 +2382,8 @@ type SiteConfiguration struct {
CodeIntelAutoIndexingIndexerMap map[string]string `json:"codeIntelAutoIndexing.indexerMap,omitempty"`
// CodeIntelAutoIndexingPolicyRepositoryMatchLimit description: The maximum number of repositories to which a single auto-indexing policy can apply. Default is -1, which is unlimited.
CodeIntelAutoIndexingPolicyRepositoryMatchLimit *int `json:"codeIntelAutoIndexing.policyRepositoryMatchLimit,omitempty"`
// CodeIntelRankingDocumentReferenceCountsCronExpression description: A cron expression indicating when to run the document reference counts graph reduction job.
CodeIntelRankingDocumentReferenceCountsCronExpression *string `json:"codeIntelRanking.documentReferenceCountsCronExpression,omitempty"`
// CodeIntelRankingDocumentReferenceCountsDerivativeGraphKeyPrefix description: An arbitrary identifier used to group calculated rankings from SCIP data (excluding the SCIP export).
CodeIntelRankingDocumentReferenceCountsDerivativeGraphKeyPrefix string `json:"codeIntelRanking.documentReferenceCountsDerivativeGraphKeyPrefix,omitempty"`
// CodeIntelRankingDocumentReferenceCountsEnabled description: Enables/disables the document reference counts feature. Currently experimental.
@ -2635,6 +2637,7 @@ func (v *SiteConfiguration) UnmarshalJSON(data []byte) error {
delete(m, "codeIntelAutoIndexing.enabled")
delete(m, "codeIntelAutoIndexing.indexerMap")
delete(m, "codeIntelAutoIndexing.policyRepositoryMatchLimit")
delete(m, "codeIntelRanking.documentReferenceCountsCronExpression")
delete(m, "codeIntelRanking.documentReferenceCountsDerivativeGraphKeyPrefix")
delete(m, "codeIntelRanking.documentReferenceCountsEnabled")
delete(m, "codeIntelRanking.documentReferenceCountsGraphKey")

9
schema/site.schema.json
View File

@ -697,6 +697,15 @@
"group": "Code intelligence",
"default": false
},
"codeIntelRanking.documentReferenceCountsCronExpression": {
"description": "A cron expression indicating when to run the document reference counts graph reduction job.",
"type": "string",
"!go": {
"pointer": true
},
"group": "Code intelligence",
"default": "@weekly"
},
"codeIntelRanking.documentReferenceCountsGraphKey": {
"description": "An arbitrary identifier used to group calculated rankings from SCIP data (including the SCIP export).",
"type": "string",