From 254aea69eee7ac64d624fd2046b033818cb4e192 Mon Sep 17 00:00:00 2001 From: Eric Fritz Date: Thu, 21 Apr 2022 19:03:35 -0500 Subject: [PATCH] RFC 619: (M1) Generate uploads service skeleton (#33613) --- .../codeintel/commitgraph_updater_job.go | 28 +++ .../internal/codeintel/upload_expirer.go | 28 +++ .../internal/codeintel/upload_janitor.go | 28 +++ cmd/worker/shared/main.go | 3 + doc/admin/workers.md | 12 ++ .../frontend/internal/codeintel/services.go | 11 ++ .../uploads/background/cleanup/config.go | 19 +++ .../uploads/background/cleanup/init.go | 11 ++ .../uploads/background/cleanup/janitor.go | 20 +++ .../uploads/background/commitgraph/config.go | 19 +++ .../uploads/background/commitgraph/init.go | 11 ++ .../uploads/background/commitgraph/updater.go | 20 +++ .../uploads/background/expiration/config.go | 19 +++ .../uploads/background/expiration/expirer.go | 20 +++ .../uploads/background/expiration/init.go | 11 ++ internal/codeintel/uploads/gen.go | 3 + internal/codeintel/uploads/iface.go | 11 ++ internal/codeintel/uploads/init.go | 46 +++++ .../codeintel/uploads/internal/store/init.go | 36 ++++ .../uploads/internal/store/observability.go | 33 ++++ .../codeintel/uploads/internal/store/scan.go | 29 ++++ .../codeintel/uploads/internal/store/store.go | 70 ++++++++ internal/codeintel/uploads/mock_iface_test.go | 161 ++++++++++++++++++ internal/codeintel/uploads/observability.go | 45 +++++ internal/codeintel/uploads/service.go | 93 ++++++++++ internal/codeintel/uploads/shared/types.go | 5 + .../codeintel/uploads/transport/http/init.go | 32 ++++ .../uploads/transport/http/observability.go | 33 ++++ .../uploads/transport/http/resolver.go | 69 ++++++++ .../codeintel/uploads/transport/http/util.go | 33 ++++ 30 files changed, 959 insertions(+) create mode 100644 cmd/worker/internal/codeintel/commitgraph_updater_job.go create mode 100644 cmd/worker/internal/codeintel/upload_expirer.go create mode 100644 cmd/worker/internal/codeintel/upload_janitor.go create mode 100644 internal/codeintel/uploads/background/cleanup/config.go create mode 100644 internal/codeintel/uploads/background/cleanup/init.go create mode 100644 internal/codeintel/uploads/background/cleanup/janitor.go create mode 100644 internal/codeintel/uploads/background/commitgraph/config.go create mode 100644 internal/codeintel/uploads/background/commitgraph/init.go create mode 100644 internal/codeintel/uploads/background/commitgraph/updater.go create mode 100644 internal/codeintel/uploads/background/expiration/config.go create mode 100644 internal/codeintel/uploads/background/expiration/expirer.go create mode 100644 internal/codeintel/uploads/background/expiration/init.go create mode 100644 internal/codeintel/uploads/gen.go create mode 100644 internal/codeintel/uploads/iface.go create mode 100644 internal/codeintel/uploads/init.go create mode 100644 internal/codeintel/uploads/internal/store/init.go create mode 100644 internal/codeintel/uploads/internal/store/observability.go create mode 100644 internal/codeintel/uploads/internal/store/scan.go create mode 100644 internal/codeintel/uploads/internal/store/store.go create mode 100644 internal/codeintel/uploads/mock_iface_test.go create mode 100644 internal/codeintel/uploads/observability.go create mode 100644 internal/codeintel/uploads/service.go create mode 100644 internal/codeintel/uploads/shared/types.go create mode 100644 internal/codeintel/uploads/transport/http/init.go create mode 100644 internal/codeintel/uploads/transport/http/observability.go create mode 100644 internal/codeintel/uploads/transport/http/resolver.go create mode 100644 internal/codeintel/uploads/transport/http/util.go diff --git a/cmd/worker/internal/codeintel/commitgraph_updater_job.go b/cmd/worker/internal/codeintel/commitgraph_updater_job.go new file mode 100644 index 00000000000..2843360f890 --- /dev/null +++ b/cmd/worker/internal/codeintel/commitgraph_updater_job.go @@ -0,0 +1,28 @@ +package codeintel + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/cmd/worker/job" + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/commitgraph" + "github.com/sourcegraph/sourcegraph/internal/env" + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +type commitGraphUpdaterJob struct{} + +func NewCommitGraphUpdaterJob() job.Job { + return &commitGraphUpdaterJob{} +} + +func (j *commitGraphUpdaterJob) Config() []env.Config { + return []env.Config{ + commitgraph.ConfigInst, + } +} + +func (j *commitGraphUpdaterJob) Routines(ctx context.Context) ([]goroutine.BackgroundRoutine, error) { + return []goroutine.BackgroundRoutine{ + commitgraph.NewUpdater(), + }, nil +} diff --git a/cmd/worker/internal/codeintel/upload_expirer.go b/cmd/worker/internal/codeintel/upload_expirer.go new file mode 100644 index 00000000000..4b61f41e53e --- /dev/null +++ b/cmd/worker/internal/codeintel/upload_expirer.go @@ -0,0 +1,28 @@ +package codeintel + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/cmd/worker/job" + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/expiration" + "github.com/sourcegraph/sourcegraph/internal/env" + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +type uploadExpirerJob struct{} + +func NewUploadExpirerJob() job.Job { + return &uploadExpirerJob{} +} + +func (j *uploadExpirerJob) Config() []env.Config { + return []env.Config{ + expiration.ConfigInst, + } +} + +func (j *uploadExpirerJob) Routines(ctx context.Context) ([]goroutine.BackgroundRoutine, error) { + return []goroutine.BackgroundRoutine{ + expiration.NewExpirer(), + }, nil +} diff --git a/cmd/worker/internal/codeintel/upload_janitor.go b/cmd/worker/internal/codeintel/upload_janitor.go new file mode 100644 index 00000000000..285277a3e92 --- /dev/null +++ b/cmd/worker/internal/codeintel/upload_janitor.go @@ -0,0 +1,28 @@ +package codeintel + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/cmd/worker/job" + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/background/cleanup" + "github.com/sourcegraph/sourcegraph/internal/env" + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +type uploadJanitorJob struct{} + +func NewUploadJanitorJob() job.Job { + return &uploadJanitorJob{} +} + +func (j *uploadJanitorJob) Config() []env.Config { + return []env.Config{ + cleanup.ConfigInst, + } +} + +func (j *uploadJanitorJob) Routines(ctx context.Context) ([]goroutine.BackgroundRoutine, error) { + return []goroutine.BackgroundRoutine{ + cleanup.NewJanitor(), + }, nil +} diff --git a/cmd/worker/shared/main.go b/cmd/worker/shared/main.go index 1d102661b75..f2eb2a2abbb 100644 --- a/cmd/worker/shared/main.go +++ b/cmd/worker/shared/main.go @@ -42,6 +42,9 @@ func Start(additionalJobs map[string]job.Job, registerEnterpriseMigrations func( builtins := map[string]job.Job{ "webhook-log-janitor": webhooks.NewJanitor(), "out-of-band-migrations": migrations.NewMigrator(registerMigrations), + "codeintel-upload-janitor": codeintel.NewUploadJanitorJob(), + "codeintel-upload-expirer": codeintel.NewUploadExpirerJob(), + "codeintel-commitgraph-updater": codeintel.NewCommitGraphUpdaterJob(), "codeintel-documents-indexer": codeintel.NewDocumentsIndexerJob(), "codeintel-autoindexing-scheduler": codeintel.NewAutoindexingSchedulerJob(), "codeintel-dependencies-indexer": codeintel.NewDependenciesIndexerJob(), diff --git a/doc/admin/workers.md b/doc/admin/workers.md index 8e9ca391536..6732d463b0c 100644 --- a/doc/admin/workers.md +++ b/doc/admin/workers.md @@ -10,6 +10,18 @@ The following jobs are defined by the `worker` service. This job runs [out of band migrations](migration.md#mout-of-band-migrations), which perform large data migrations in the background over time instead of synchronously during Sourcegraph instance updates. +#### `codeintel-upload-janitor` + +This job will eventually (and partially) replace `codeintel-janitor`. + +#### `codeintel-upload-expirer` + +This job will eventually (and partially) replace `codeintel-janitor` + +#### `codeintel-commitgraph-updater` + +This job will eventually replace `codeintel-commitgraph`. + #### `codeintel-documents-indexer` This job periodically indexes file contents at a syntactic level to build an index of search-based code intelligence. diff --git a/enterprise/cmd/frontend/internal/codeintel/services.go b/enterprise/cmd/frontend/internal/codeintel/services.go index 85253735db1..c7a9cd587b7 100644 --- a/enterprise/cmd/frontend/internal/codeintel/services.go +++ b/enterprise/cmd/frontend/internal/codeintel/services.go @@ -17,6 +17,8 @@ import ( store "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/dbstore" "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/lsifstore" "github.com/sourcegraph/sourcegraph/enterprise/internal/codeintel/stores/lsifuploadstore" + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads" + uploadshttp "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/transport/http" "github.com/sourcegraph/sourcegraph/internal/conf" "github.com/sourcegraph/sourcegraph/internal/conf/conftypes" "github.com/sourcegraph/sourcegraph/internal/database" @@ -70,6 +72,15 @@ func NewServices(ctx context.Context, config *Config, siteConfig conftypes.Watch // Initialize http endpoints operations := httpapi.NewOperations(observationContext) newUploadHandler := func(internal bool) http.Handler { + if false { + // Until this handler has been implemented, we retain the origial + // LSIF update handler. + // + // See https://github.com/sourcegraph/sourcegraph/issues/33375 + + return uploadshttp.GetHandler(uploads.GetService(db)) + } + return httpapi.NewUploadHandler( db, &httpapi.DBStoreShim{Store: dbStore}, diff --git a/internal/codeintel/uploads/background/cleanup/config.go b/internal/codeintel/uploads/background/cleanup/config.go new file mode 100644 index 00000000000..543163c969f --- /dev/null +++ b/internal/codeintel/uploads/background/cleanup/config.go @@ -0,0 +1,19 @@ +package cleanup + +import ( + "time" + + "github.com/sourcegraph/sourcegraph/internal/env" +) + +type config struct { + env.BaseConfig + + Interval time.Duration +} + +var ConfigInst = &config{} + +func (c *config) Load() { + c.Interval = c.GetInterval("CODEINTEL_UPLOLAD_JANITOR_INTERVAL", "1s", "How frequently to run the updater janitor routine.") +} diff --git a/internal/codeintel/uploads/background/cleanup/init.go b/internal/codeintel/uploads/background/cleanup/init.go new file mode 100644 index 00000000000..663f2e152f9 --- /dev/null +++ b/internal/codeintel/uploads/background/cleanup/init.go @@ -0,0 +1,11 @@ +package cleanup + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +func NewJanitor() goroutine.BackgroundRoutine { + return goroutine.NewPeriodicGoroutine(context.Background(), ConfigInst.Interval, &janitor{}) +} diff --git a/internal/codeintel/uploads/background/cleanup/janitor.go b/internal/codeintel/uploads/background/cleanup/janitor.go new file mode 100644 index 00000000000..790f0a0b86a --- /dev/null +++ b/internal/codeintel/uploads/background/cleanup/janitor.go @@ -0,0 +1,20 @@ +package cleanup + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +type janitor struct{} + +var _ goroutine.Handler = &janitor{} +var _ goroutine.ErrorHandler = &janitor{} + +func (r *janitor) Handle(ctx context.Context) error { + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + return nil +} + +func (r *janitor) HandleError(err error) { +} diff --git a/internal/codeintel/uploads/background/commitgraph/config.go b/internal/codeintel/uploads/background/commitgraph/config.go new file mode 100644 index 00000000000..a4d50dadcef --- /dev/null +++ b/internal/codeintel/uploads/background/commitgraph/config.go @@ -0,0 +1,19 @@ +package commitgraph + +import ( + "time" + + "github.com/sourcegraph/sourcegraph/internal/env" +) + +type config struct { + env.BaseConfig + + Interval time.Duration +} + +var ConfigInst = &config{} + +func (c *config) Load() { + c.Interval = c.GetInterval("CODEINTEL_UPLOAD_COMMITGRAPH_UPDATER_INTERVAL", "1s", "How frequently to run the upload commitgraph updater routine.") +} diff --git a/internal/codeintel/uploads/background/commitgraph/init.go b/internal/codeintel/uploads/background/commitgraph/init.go new file mode 100644 index 00000000000..28ff18c3d5c --- /dev/null +++ b/internal/codeintel/uploads/background/commitgraph/init.go @@ -0,0 +1,11 @@ +package commitgraph + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +func NewUpdater() goroutine.BackgroundRoutine { + return goroutine.NewPeriodicGoroutine(context.Background(), ConfigInst.Interval, &updater{}) +} diff --git a/internal/codeintel/uploads/background/commitgraph/updater.go b/internal/codeintel/uploads/background/commitgraph/updater.go new file mode 100644 index 00000000000..2c3691c09ce --- /dev/null +++ b/internal/codeintel/uploads/background/commitgraph/updater.go @@ -0,0 +1,20 @@ +package commitgraph + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +type updater struct{} + +var _ goroutine.Handler = &updater{} +var _ goroutine.ErrorHandler = &updater{} + +func (r *updater) Handle(ctx context.Context) error { + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + return nil +} + +func (r *updater) HandleError(err error) { +} diff --git a/internal/codeintel/uploads/background/expiration/config.go b/internal/codeintel/uploads/background/expiration/config.go new file mode 100644 index 00000000000..974a33050ea --- /dev/null +++ b/internal/codeintel/uploads/background/expiration/config.go @@ -0,0 +1,19 @@ +package expiration + +import ( + "time" + + "github.com/sourcegraph/sourcegraph/internal/env" +) + +type config struct { + env.BaseConfig + + Interval time.Duration +} + +var ConfigInst = &config{} + +func (c *config) Load() { + c.Interval = c.GetInterval("CODEINTEL_UPLOAD_EXPIRER_INTERVAL", "1s", "How frequently to run the upload expirer routine.") +} diff --git a/internal/codeintel/uploads/background/expiration/expirer.go b/internal/codeintel/uploads/background/expiration/expirer.go new file mode 100644 index 00000000000..6344826e7bb --- /dev/null +++ b/internal/codeintel/uploads/background/expiration/expirer.go @@ -0,0 +1,20 @@ +package expiration + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +type expirer struct{} + +var _ goroutine.Handler = &expirer{} +var _ goroutine.ErrorHandler = &expirer{} + +func (r *expirer) Handle(ctx context.Context) error { + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + return nil +} + +func (r *expirer) HandleError(err error) { +} diff --git a/internal/codeintel/uploads/background/expiration/init.go b/internal/codeintel/uploads/background/expiration/init.go new file mode 100644 index 00000000000..4d8fae189a8 --- /dev/null +++ b/internal/codeintel/uploads/background/expiration/init.go @@ -0,0 +1,11 @@ +package expiration + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/goroutine" +) + +func NewExpirer() goroutine.BackgroundRoutine { + return goroutine.NewPeriodicGoroutine(context.Background(), ConfigInst.Interval, &expirer{}) +} diff --git a/internal/codeintel/uploads/gen.go b/internal/codeintel/uploads/gen.go new file mode 100644 index 00000000000..bcabf117425 --- /dev/null +++ b/internal/codeintel/uploads/gen.go @@ -0,0 +1,3 @@ +package uploads + +//go:generate ../../../dev/mockgen.sh github.com/sourcegraph/sourcegraph/internal/codeintel/uploads -i Store -o mock_iface_test.go diff --git a/internal/codeintel/uploads/iface.go b/internal/codeintel/uploads/iface.go new file mode 100644 index 00000000000..18d86907828 --- /dev/null +++ b/internal/codeintel/uploads/iface.go @@ -0,0 +1,11 @@ +package uploads + +import ( + "context" + + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/store" +) + +type Store interface { + List(ctx context.Context, opts store.ListOpts) ([]Upload, error) +} diff --git a/internal/codeintel/uploads/init.go b/internal/codeintel/uploads/init.go new file mode 100644 index 00000000000..ced57ad9350 --- /dev/null +++ b/internal/codeintel/uploads/init.go @@ -0,0 +1,46 @@ +package uploads + +import ( + "sync" + + "github.com/inconshreveable/log15" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/store" + "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/internal/trace" +) + +var ( + svc *Service + svcOnce sync.Once +) + +// GetService creates or returns an already-initialized uplopads service. If the service is +// new, it will use the given database handle. +func GetService(db database.DB) *Service { + svcOnce.Do(func() { + observationContext := &observation.Context{ + Logger: log15.Root(), + Tracer: &trace.Tracer{Tracer: opentracing.GlobalTracer()}, + Registerer: prometheus.DefaultRegisterer, + } + + svc = newService( + store.GetStore(db), + observationContext, + ) + }) + + return svc +} + +// TestService creates a fresh uplopads service with the given database handle. +func TestService(db database.DB) *Service { + return newService( + store.GetStore(db), + &observation.TestContext, + ) +} diff --git a/internal/codeintel/uploads/internal/store/init.go b/internal/codeintel/uploads/internal/store/init.go new file mode 100644 index 00000000000..489a41d8b0f --- /dev/null +++ b/internal/codeintel/uploads/internal/store/init.go @@ -0,0 +1,36 @@ +package store + +import ( + "sync" + + "github.com/inconshreveable/log15" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + + "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/internal/trace" +) + +var ( + store *Store + storeOnce sync.Once +) + +func GetStore(db database.DB) *Store { + storeOnce.Do(func() { + observationContext := &observation.Context{ + Logger: log15.Root(), + Tracer: &trace.Tracer{Tracer: opentracing.GlobalTracer()}, + Registerer: prometheus.DefaultRegisterer, + } + + store = newStore(db, observationContext) + }) + + return store +} + +func TestStore(db database.DB) *Store { + return newStore(db, &observation.TestContext) +} diff --git a/internal/codeintel/uploads/internal/store/observability.go b/internal/codeintel/uploads/internal/store/observability.go new file mode 100644 index 00000000000..ea1cf22621c --- /dev/null +++ b/internal/codeintel/uploads/internal/store/observability.go @@ -0,0 +1,33 @@ +package store + +import ( + "fmt" + + "github.com/sourcegraph/sourcegraph/internal/metrics" + "github.com/sourcegraph/sourcegraph/internal/observation" +) + +type operations struct { + list *observation.Operation +} + +func newOperations(observationContext *observation.Context) *operations { + metrics := metrics.NewREDMetrics( + observationContext.Registerer, + "codeintel_uploads_store", + metrics.WithLabels("op"), + metrics.WithCountHelp("Total number of method invocations."), + ) + + op := func(name string) *observation.Operation { + return observationContext.Operation(observation.Op{ + Name: fmt.Sprintf("codeintel.uploads.store.%s", name), + MetricLabelValues: []string{name}, + Metrics: metrics, + }) + } + + return &operations{ + list: op("List"), + } +} diff --git a/internal/codeintel/uploads/internal/store/scan.go b/internal/codeintel/uploads/internal/store/scan.go new file mode 100644 index 00000000000..77872609c85 --- /dev/null +++ b/internal/codeintel/uploads/internal/store/scan.go @@ -0,0 +1,29 @@ +package store + +import ( + "database/sql" + + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared" + "github.com/sourcegraph/sourcegraph/internal/database/basestore" +) + +func scanUploads(rows *sql.Rows, queryErr error) (uploads []shared.Upload, err error) { + if queryErr != nil { + return nil, queryErr + } + defer func() { err = basestore.CloseRows(rows, err) }() + + for rows.Next() { + var upload shared.Upload + + if err = rows.Scan( + &upload.ID, + ); err != nil { + return nil, err + } + + uploads = append(uploads, upload) + } + + return uploads, nil +} diff --git a/internal/codeintel/uploads/internal/store/store.go b/internal/codeintel/uploads/internal/store/store.go new file mode 100644 index 00000000000..2c438a851db --- /dev/null +++ b/internal/codeintel/uploads/internal/store/store.go @@ -0,0 +1,70 @@ +package store + +import ( + "context" + "database/sql" + + "github.com/keegancsmith/sqlf" + "github.com/opentracing/opentracing-go/log" + + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared" + "github.com/sourcegraph/sourcegraph/internal/database/basestore" + "github.com/sourcegraph/sourcegraph/internal/database/dbutil" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +type Store struct { + *basestore.Store + operations *operations +} + +func newStore(db dbutil.DB, observationContext *observation.Context) *Store { + return &Store{ + Store: basestore.NewWithDB(db, sql.TxOptions{}), + operations: newOperations(observationContext), + } +} + +func (s *Store) With(other basestore.ShareableStore) *Store { + return &Store{ + Store: s.Store.With(other), + operations: s.operations, + } +} + +func (s *Store) Transact(ctx context.Context) (*Store, error) { + txBase, err := s.Store.Transact(ctx) + if err != nil { + return nil, err + } + + return &Store{ + Store: txBase, + operations: s.operations, + }, nil +} + +type ListOpts struct { + Limit int +} + +func (s *Store) List(ctx context.Context, opts ListOpts) (uploads []shared.Upload, err error) { + ctx, endObservation := s.operations.list.With(ctx, &err, observation.Args{}) + defer func() { + endObservation(1, observation.Args{LogFields: []log.Field{ + log.Int("numUploads", len(uploads)), + }}) + }() + + // This is only a stub and will be replaced or significantly modified + // in https://github.com/sourcegraph/sourcegraph/issues/33375 + _, _ = scanUploads(s.Query(ctx, sqlf.Sprintf(listQuery, opts.Limit))) + return nil, errors.Newf("unimplemented: uploads.store.List") +} + +const listQuery = ` +-- source: internal/codeintel/uploads/store/store.go:List +SELECT id FROM TODO +LIMIT %s +` diff --git a/internal/codeintel/uploads/mock_iface_test.go b/internal/codeintel/uploads/mock_iface_test.go new file mode 100644 index 00000000000..65020bf880d --- /dev/null +++ b/internal/codeintel/uploads/mock_iface_test.go @@ -0,0 +1,161 @@ +// Code generated by go-mockgen 1.1.5; DO NOT EDIT. + +package uploads + +import ( + "context" + "sync" + + store "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/store" + shared "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared" +) + +// MockStore is a mock implementation of the Store interface (from the +// package github.com/sourcegraph/sourcegraph/internal/codeintel/uploads) +// used for unit testing. +type MockStore struct { + // ListFunc is an instance of a mock function object controlling the + // behavior of the method List. + ListFunc *StoreListFunc +} + +// NewMockStore creates a new mock of the Store interface. All methods +// return zero values for all results, unless overwritten. +func NewMockStore() *MockStore { + return &MockStore{ + ListFunc: &StoreListFunc{ + defaultHook: func(context.Context, store.ListOpts) ([]shared.Upload, error) { + return nil, nil + }, + }, + } +} + +// NewStrictMockStore creates a new mock of the Store interface. All methods +// panic on invocation, unless overwritten. +func NewStrictMockStore() *MockStore { + return &MockStore{ + ListFunc: &StoreListFunc{ + defaultHook: func(context.Context, store.ListOpts) ([]shared.Upload, error) { + panic("unexpected invocation of MockStore.List") + }, + }, + } +} + +// NewMockStoreFrom creates a new mock of the MockStore interface. All +// methods delegate to the given implementation, unless overwritten. +func NewMockStoreFrom(i Store) *MockStore { + return &MockStore{ + ListFunc: &StoreListFunc{ + defaultHook: i.List, + }, + } +} + +// StoreListFunc describes the behavior when the List method of the parent +// MockStore instance is invoked. +type StoreListFunc struct { + defaultHook func(context.Context, store.ListOpts) ([]shared.Upload, error) + hooks []func(context.Context, store.ListOpts) ([]shared.Upload, error) + history []StoreListFuncCall + mutex sync.Mutex +} + +// List delegates to the next hook function in the queue and stores the +// parameter and result values of this invocation. +func (m *MockStore) List(v0 context.Context, v1 store.ListOpts) ([]shared.Upload, error) { + r0, r1 := m.ListFunc.nextHook()(v0, v1) + m.ListFunc.appendCall(StoreListFuncCall{v0, v1, r0, r1}) + return r0, r1 +} + +// SetDefaultHook sets function that is called when the List method of the +// parent MockStore instance is invoked and the hook queue is empty. +func (f *StoreListFunc) SetDefaultHook(hook func(context.Context, store.ListOpts) ([]shared.Upload, error)) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// List method of the parent MockStore instance invokes the hook at the +// front of the queue and discards it. After the queue is empty, the default +// hook function is invoked for any future action. +func (f *StoreListFunc) PushHook(hook func(context.Context, store.ListOpts) ([]shared.Upload, error)) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *StoreListFunc) SetDefaultReturn(r0 []shared.Upload, r1 error) { + f.SetDefaultHook(func(context.Context, store.ListOpts) ([]shared.Upload, error) { + return r0, r1 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *StoreListFunc) PushReturn(r0 []shared.Upload, r1 error) { + f.PushHook(func(context.Context, store.ListOpts) ([]shared.Upload, error) { + return r0, r1 + }) +} + +func (f *StoreListFunc) nextHook() func(context.Context, store.ListOpts) ([]shared.Upload, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *StoreListFunc) appendCall(r0 StoreListFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of StoreListFuncCall objects describing the +// invocations of this function. +func (f *StoreListFunc) History() []StoreListFuncCall { + f.mutex.Lock() + history := make([]StoreListFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// StoreListFuncCall is an object that describes an invocation of method +// List on an instance of MockStore. +type StoreListFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 context.Context + // Arg1 is the value of the 2nd argument passed to this method + // invocation. + Arg1 store.ListOpts + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 []shared.Upload + // Result1 is the value of the 2nd result returned from this method + // invocation. + Result1 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c StoreListFuncCall) Args() []interface{} { + return []interface{}{c.Arg0, c.Arg1} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c StoreListFuncCall) Results() []interface{} { + return []interface{}{c.Result0, c.Result1} +} diff --git a/internal/codeintel/uploads/observability.go b/internal/codeintel/uploads/observability.go new file mode 100644 index 00000000000..2b59fc764b3 --- /dev/null +++ b/internal/codeintel/uploads/observability.go @@ -0,0 +1,45 @@ +package uploads + +import ( + "fmt" + + "github.com/sourcegraph/sourcegraph/internal/metrics" + "github.com/sourcegraph/sourcegraph/internal/observation" +) + +type operations struct { + list *observation.Operation + get *observation.Operation + getBatch *observation.Operation + enqueue *observation.Operation + delete *observation.Operation + commitsVisibleTo *observation.Operation + uploadsVisibleTo *observation.Operation +} + +func newOperations(observationContext *observation.Context) *operations { + metrics := metrics.NewREDMetrics( + observationContext.Registerer, + "codeintel_uploads", + metrics.WithLabels("op"), + metrics.WithCountHelp("Total number of method invocations."), + ) + + op := func(name string) *observation.Operation { + return observationContext.Operation(observation.Op{ + Name: fmt.Sprintf("codeintel.uploads.%s", name), + MetricLabelValues: []string{name}, + Metrics: metrics, + }) + } + + return &operations{ + list: op("List"), + get: op("Get"), + getBatch: op("GetBatch"), + enqueue: op("Enqueue"), + delete: op("Delete"), + commitsVisibleTo: op("CommitsVisibleTo"), + uploadsVisibleTo: op("UploadsVisibleTo"), + } +} diff --git a/internal/codeintel/uploads/service.go b/internal/codeintel/uploads/service.go new file mode 100644 index 00000000000..9e1781c8c44 --- /dev/null +++ b/internal/codeintel/uploads/service.go @@ -0,0 +1,93 @@ +package uploads + +import ( + "context" + "io" + + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/internal/store" + "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads/shared" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/lib/errors" +) + +type Service struct { + uploadsStore Store + operations *operations +} + +func newService(uploadsStore Store, observationContext *observation.Context) *Service { + return &Service{ + uploadsStore: uploadsStore, + operations: newOperations(observationContext), + } +} + +type Upload = shared.Upload + +type ListOpts struct { + Limit int +} + +func (s *Service) List(ctx context.Context, opts ListOpts) (uploads []Upload, err error) { + ctx, endObservation := s.operations.list.With(ctx, &err, observation.Args{}) + defer endObservation(1, observation.Args{}) + + return s.uploadsStore.List(ctx, store.ListOpts(opts)) +} + +func (s *Service) Get(ctx context.Context, id int) (upload Upload, ok bool, err error) { + ctx, endObservation := s.operations.get.With(ctx, &err, observation.Args{}) + defer endObservation(1, observation.Args{}) + + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + _ = ctx + return Upload{}, false, errors.Newf("unimplemented: uploads.Get") +} + +func (s *Service) GetBatch(ctx context.Context, ids ...int) (uploads []Upload, err error) { + ctx, endObservation := s.operations.getBatch.With(ctx, &err, observation.Args{}) + defer endObservation(1, observation.Args{}) + + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + _ = ctx + return nil, errors.Newf("unimplemented: uploads.GetBatch") +} + +type UploadState struct { +} + +func (s *Service) Enqueue(ctx context.Context, state UploadState, reader io.Reader) (err error) { + ctx, endObservation := s.operations.enqueue.With(ctx, &err, observation.Args{}) + defer endObservation(1, observation.Args{}) + + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + _ = ctx + return errors.Newf("unimplemented: uploads.Enqueue") +} + +func (s *Service) Delete(ctx context.Context, id int) (err error) { + ctx, endObservation := s.operations.delete.With(ctx, &err, observation.Args{}) + defer endObservation(1, observation.Args{}) + + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + _ = ctx + return errors.Newf("unimplemented: uploads.Delete") +} + +func (s *Service) CommitsVisibleToUpload(ctx context.Context, id int) (commits []string, err error) { + ctx, endObservation := s.operations.commitsVisibleTo.With(ctx, &err, observation.Args{}) + defer endObservation(1, observation.Args{}) + + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + _ = ctx + return nil, errors.Newf("unimplemented: uploads.CommitsVisibleToUpload") +} + +func (s *Service) UploadsVisibleToCommit(ctx context.Context, commit string) (uploads []Upload, err error) { + ctx, endObservation := s.operations.uploadsVisibleTo.With(ctx, &err, observation.Args{}) + defer endObservation(1, observation.Args{}) + + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + _ = ctx + return nil, errors.Newf("unimplemented: uploads.UploadsVisibleToCommit") +} diff --git a/internal/codeintel/uploads/shared/types.go b/internal/codeintel/uploads/shared/types.go new file mode 100644 index 00000000000..cc2f0bd1c5f --- /dev/null +++ b/internal/codeintel/uploads/shared/types.go @@ -0,0 +1,5 @@ +package shared + +type Upload struct { + ID int +} diff --git a/internal/codeintel/uploads/transport/http/init.go b/internal/codeintel/uploads/transport/http/init.go new file mode 100644 index 00000000000..f3d8c11f272 --- /dev/null +++ b/internal/codeintel/uploads/transport/http/init.go @@ -0,0 +1,32 @@ +package http + +import ( + "sync" + + "github.com/inconshreveable/log15" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + + uploads "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads" + "github.com/sourcegraph/sourcegraph/internal/observation" + "github.com/sourcegraph/sourcegraph/internal/trace" +) + +var ( + handler *Handler + handlerOnce sync.Once +) + +func GetHandler(svc *uploads.Service) *Handler { + handlerOnce.Do(func() { + observationContext := &observation.Context{ + Logger: log15.Root(), + Tracer: &trace.Tracer{Tracer: opentracing.GlobalTracer()}, + Registerer: prometheus.DefaultRegisterer, + } + + handler = newHandler(svc, observationContext) + }) + + return handler +} diff --git a/internal/codeintel/uploads/transport/http/observability.go b/internal/codeintel/uploads/transport/http/observability.go new file mode 100644 index 00000000000..676c54d0e9e --- /dev/null +++ b/internal/codeintel/uploads/transport/http/observability.go @@ -0,0 +1,33 @@ +package http + +import ( + "fmt" + + "github.com/sourcegraph/sourcegraph/internal/metrics" + "github.com/sourcegraph/sourcegraph/internal/observation" +) + +type operations struct { + todo *observation.Operation +} + +func newOperations(observationContext *observation.Context) *operations { + metrics := metrics.NewREDMetrics( + observationContext.Registerer, + "codeintel_uploads_transport_http", + metrics.WithLabels("op"), + metrics.WithCountHelp("Total number of method invocations."), + ) + + op := func(name string) *observation.Operation { + return observationContext.Operation(observation.Op{ + Name: fmt.Sprintf("codeintel.uploads.transport.http.%s", name), + MetricLabelValues: []string{name}, + Metrics: metrics, + }) + } + + return &operations{ + todo: op("Todo"), + } +} diff --git a/internal/codeintel/uploads/transport/http/resolver.go b/internal/codeintel/uploads/transport/http/resolver.go new file mode 100644 index 00000000000..12ca6c8544d --- /dev/null +++ b/internal/codeintel/uploads/transport/http/resolver.go @@ -0,0 +1,69 @@ +package http + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + + "github.com/opentracing/opentracing-go/log" + + uploads "github.com/sourcegraph/sourcegraph/internal/codeintel/uploads" + "github.com/sourcegraph/sourcegraph/internal/observation" +) + +type Handler struct { + svc *uploads.Service + operations *operations +} + +var _ http.Handler = &Handler{} + +func newHandler(svc *uploads.Service, observationContext *observation.Context) *Handler { + return &Handler{ + svc: svc, + operations: newOperations(observationContext), + } +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.serveJSON(w, r) +} + +func (h *Handler) serveJSON(w http.ResponseWriter, r *http.Request) { + payload, statusCode, err := h.handleRequest(r) + if err != nil { + handleErr(w, err, "request failed", statusCode) + return + } + if payload == nil { + w.WriteHeader(http.StatusNoContent) + return + } + + data, err := json.Marshal(payload) + if err != nil { + handleErr(w, err, "failed to serialize result", http.StatusInternalServerError) + return + } + + w.WriteHeader(statusCode) + + if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { + handleErr(nil, err, "failed to write payload to client", http.StatusInternalServerError) + return + } +} + +func (h *Handler) handleRequest(r *http.Request) (payload interface{}, statusCode int, err error) { + ctx, trace, endObservation := h.operations.todo.WithAndLogger(r.Context(), &err, observation.Args{}) + defer func() { + endObservation(1, observation.Args{LogFields: []log.Field{ + log.Int("statusCode", statusCode), + }}) + }() + + // To be implemented in https://github.com/sourcegraph/sourcegraph/issues/33375 + _, _ = ctx, trace + return nil, 0, nil +} diff --git a/internal/codeintel/uploads/transport/http/util.go b/internal/codeintel/uploads/transport/http/util.go new file mode 100644 index 00000000000..40a60c6db08 --- /dev/null +++ b/internal/codeintel/uploads/transport/http/util.go @@ -0,0 +1,33 @@ +package http + +import ( + "fmt" + "net/http" + + "github.com/inconshreveable/log15" +) + +// func hasQuery(r *http.Request, name string) bool { +// return r.URL.Query().Get(name) != "" +// } + +// func getQuery(r *http.Request, name string) string { +// return r.URL.Query().Get(name) +// } + +// func getQueryInt(r *http.Request, name string) int { +// value, _ := strconv.Atoi(r.URL.Query().Get(name)) +// return value +// } + +const logPrefix = "codeintel.uploads.transport.http" + +func handleErr(w http.ResponseWriter, err error, logMessage string, statusCode int) { + if statusCode >= 500 { + log15.Error(fmt.Sprintf("%s: %s", logPrefix, logMessage), "error", err) + } + + if w != nil { + http.Error(w, fmt.Sprintf("%s: %s: %s", logPrefix, logMessage, err), statusCode) + } +}