From c205dedff27fc31e7089898cdca1f37ce3a48a85 Mon Sep 17 00:00:00 2001 From: Erzhan Torokulov Date: Wed, 24 May 2023 13:25:18 +0600 Subject: [PATCH] (analytics) add repo meta usage ping (#51921) * (analytics) adds backend event logs on repo metadata create/update/delete --- .../web/src/site-admin/SiteAdminPingsPage.tsx | 1 + cmd/frontend/graphqlbackend/BUILD.bazel | 1 + .../graphqlbackend/repository_metadata.go | 37 ++++- .../internal/app/updatecheck/client.go | 17 +++ .../internal/app/updatecheck/handler.go | 3 + .../internal/app/updatecheck/handler_test.go | 115 +++++++++++++- doc/admin/pings.md | 3 +- internal/database/event_logs.go | 94 ++++++++++++ internal/database/event_logs_test.go | 142 ++++++++++++++++++ internal/database/mocks_temp.go | 132 ++++++++++++++++ internal/types/types.go | 31 ++++ internal/usagestats/BUILD.bazel | 1 + .../usagestats/aggregated_repo_metadata.go | 60 ++++++++ 13 files changed, 632 insertions(+), 5 deletions(-) create mode 100644 internal/usagestats/aggregated_repo_metadata.go diff --git a/client/web/src/site-admin/SiteAdminPingsPage.tsx b/client/web/src/site-admin/SiteAdminPingsPage.tsx index 1c1a8a805ba..5a3256386fa 100644 --- a/client/web/src/site-admin/SiteAdminPingsPage.tsx +++ b/client/web/src/site-admin/SiteAdminPingsPage.tsx @@ -407,6 +407,7 @@ export const SiteAdminPingsPage: React.FunctionComponent
  • Histogram of cloned repository sizes
    • +
  • Aggregate daily, weekly, monthly repository metadata usage statistics
    • {updatesDisabled && All telemetry is disabled.} diff --git a/cmd/frontend/graphqlbackend/BUILD.bazel b/cmd/frontend/graphqlbackend/BUILD.bazel index ead48546fed..118fb8266fc 100644 --- a/cmd/frontend/graphqlbackend/BUILD.bazel +++ b/cmd/frontend/graphqlbackend/BUILD.bazel @@ -252,6 +252,7 @@ go_library( "//internal/database", "//internal/database/migration/cliutil", "//internal/database/migration/schemas", + "//internal/deviceid", "//internal/encryption", "//internal/encryption/keyring", "//internal/env", diff --git a/cmd/frontend/graphqlbackend/repository_metadata.go b/cmd/frontend/graphqlbackend/repository_metadata.go index f8c7cd8208a..97dc74e3dc8 100644 --- a/cmd/frontend/graphqlbackend/repository_metadata.go +++ b/cmd/frontend/graphqlbackend/repository_metadata.go @@ -9,9 +9,12 @@ import ( "github.com/graph-gophers/graphql-go/relay" "github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/graphqlutil" + "github.com/sourcegraph/sourcegraph/internal/actor" "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/deviceid" "github.com/sourcegraph/sourcegraph/internal/featureflag" "github.com/sourcegraph/sourcegraph/internal/rbac" + "github.com/sourcegraph/sourcegraph/internal/usagestats" "github.com/sourcegraph/sourcegraph/lib/errors" ) @@ -69,7 +72,12 @@ func (r *schemaResolver) AddRepoMetadata(ctx context.Context, args struct { return &EmptyResponse{}, emptyNonNilValueError{value: *args.Value} } - return &EmptyResponse{}, r.db.RepoKVPs().Create(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value}) + err = r.db.RepoKVPs().Create(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value}) + if err == nil { + r.logBackendEvent(ctx, "RepoMetadataAdded") + } + + return &EmptyResponse{}, err } // Deprecated: Use UpdateRepoMetadata instead. @@ -106,6 +114,9 @@ func (r *schemaResolver) UpdateRepoMetadata(ctx context.Context, args struct { } _, err = r.db.RepoKVPs().Update(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value}) + if err == nil { + r.logBackendEvent(ctx, "RepoMetadataUpdated") + } return &EmptyResponse{}, err } @@ -136,7 +147,29 @@ func (r *schemaResolver) DeleteRepoMetadata(ctx context.Context, args struct { return &EmptyResponse{}, err } - return &EmptyResponse{}, r.db.RepoKVPs().Delete(ctx, repoID, args.Key) + err = r.db.RepoKVPs().Delete(ctx, repoID, args.Key) + if err == nil { + r.logBackendEvent(ctx, "RepoMetadataDeleted") + } + return &EmptyResponse{}, err +} + +func (r *schemaResolver) logBackendEvent(ctx context.Context, eventName string) { + a := actor.FromContext(ctx) + if a.IsAuthenticated() && !a.IsMockUser() { + if err := usagestats.LogBackendEvent( + r.db, + a.UID, + deviceid.FromContext(ctx), + eventName, + nil, + nil, + featureflag.GetEvaluatedFlagSet(ctx), + nil, + ); err != nil { + r.logger.Warn("Could not log " + eventName) + } + } } type repoMetaResolver struct { diff --git a/cmd/frontend/internal/app/updatecheck/client.go b/cmd/frontend/internal/app/updatecheck/client.go index 9ef7ace88fe..b4f33922354 100644 --- a/cmd/frontend/internal/app/updatecheck/client.go +++ b/cmd/frontend/internal/app/updatecheck/client.go @@ -363,6 +363,17 @@ func getAndMarshalCodyUsageJSON(ctx context.Context, db database.DB) (_ json.Raw return json.Marshal(codyUsage) } +func getAndMarshalRepoMetadataUsageJSON(ctx context.Context, db database.DB) (_ json.RawMessage, err error) { + defer recordOperation("getAndMarshalRepoMetadataUsageJSON")(&err) + + repoMetadataUsage, err := usagestats.GetAggregatedRepoMetadataStats(ctx, db) + if err != nil { + return nil, err + } + + return json.Marshal(repoMetadataUsage) +} + func getDependencyVersions(ctx context.Context, db database.DB, logger log.Logger) (json.RawMessage, error) { logFunc := logFuncFrom(logger.Scoped("getDependencyVersions", "gets the version of various dependency services")) var ( @@ -519,6 +530,7 @@ func updateBody(ctx context.Context, logger log.Logger, db database.DB) (io.Read IDEExtensionsUsage: []byte("{}"), MigratedExtensionsUsage: []byte("{}"), CodyUsage: []byte("{}"), + RepoMetadataUsage: []byte("{}"), } totalUsers, err := getTotalUsersCount(ctx, db) @@ -668,6 +680,11 @@ func updateBody(ctx context.Context, logger log.Logger, db database.DB) (io.Read logFunc("codyUsage failed", log.Error(err)) } + r.RepoMetadataUsage, err = getAndMarshalRepoMetadataUsageJSON(ctx, db) + if err != nil { + logFunc("repoMetadataUsage failed", log.Error(err)) + } + r.HasExtURL = conf.UsingExternalURL() r.BuiltinSignupAllowed = conf.IsBuiltinSignupAllowed() r.AccessRequestEnabled = conf.IsAccessRequestEnabled() diff --git a/cmd/frontend/internal/app/updatecheck/handler.go b/cmd/frontend/internal/app/updatecheck/handler.go index 68fe3fab613..02ba4f3b933 100644 --- a/cmd/frontend/internal/app/updatecheck/handler.go +++ b/cmd/frontend/internal/app/updatecheck/handler.go @@ -247,6 +247,7 @@ type pingRequest struct { ActiveToday bool `json:"activeToday,omitempty"` // Only used in Sourcegraph App HasCodyEnabled bool `json:"hasCodyEnabled,omitempty"` CodyUsage json.RawMessage `json:"codyUsage,omitempty"` + RepoMetadataUsage json.RawMessage `json:"repoMetadataUsage,omitempty"` } type dependencyVersions struct { @@ -373,6 +374,7 @@ type pingPayload struct { Timestamp string `json:"timestamp"` HasCodyEnabled string `json:"has_cody_enabled"` CodyUsage json.RawMessage `json:"cody_usage"` + RepoMetadataUsage json.RawMessage `json:"repo_metadata_usage"` } func logPing(logger log.Logger, r *http.Request, pr *pingRequest, hasUpdate bool) { @@ -475,6 +477,7 @@ func marshalPing(pr *pingRequest, hasUpdate bool, clientAddr string, now time.Ti Timestamp: now.UTC().Format(time.RFC3339), HasCodyEnabled: strconv.FormatBool(codyFeatureFlag()), CodyUsage: codyUsage, + RepoMetadataUsage: pr.RepoMetadataUsage, }) } diff --git a/cmd/frontend/internal/app/updatecheck/handler_test.go b/cmd/frontend/internal/app/updatecheck/handler_test.go index f3e5a0aea85..9ed3c9a8fa5 100644 --- a/cmd/frontend/internal/app/updatecheck/handler_test.go +++ b/cmd/frontend/internal/app/updatecheck/handler_test.go @@ -207,6 +207,7 @@ func TestSerializeBasic(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "mylicense", "has_update": "true", @@ -282,6 +283,7 @@ func TestSerializeLimited(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "2023.03.23+205275.dd37e7", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "", "has_update": "true", @@ -359,6 +361,7 @@ func TestSerializeFromQuery(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "", "has_update": "true", @@ -419,6 +422,7 @@ func TestSerializeBatchChangesUsage(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "mylicense", "has_update": "true", @@ -479,6 +483,7 @@ func TestSerializeGrowthStatistics(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "mylicense", "has_update": "true", @@ -505,7 +510,7 @@ func TestSerializeGrowthStatistics(t *testing.T) { "search_onboarding": null, "homepage_panels": null, "repositories": null, -"repository_size_histogram": null, + "repository_size_histogram": null, "retention_statistics": null, "installer_email": "test@sourcegraph.com", "auth_providers": "foo,bar", @@ -640,6 +645,7 @@ func TestSerializeCodeIntelUsage(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "mylicense", "has_update": "true", @@ -822,6 +828,7 @@ func TestSerializeOldCodeIntelUsage(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "mylicense", "has_update": "true", @@ -952,6 +959,7 @@ func TestSerializeCodeHostVersions(t *testing.T) { compareJSON(t, payload, `{ "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "mylicense", "has_update": "true", @@ -1048,6 +1056,7 @@ func TestSerializeOwn(t *testing.T) { "access_request_enabled": "false", "remote_ip": "127.0.0.1", "remote_site_version": "3.12.6", + "repo_metadata_usage": null, "remote_site_id": "0101-0101", "license_key": "", "has_update": "true", @@ -1095,7 +1104,109 @@ func TestSerializeOwn(t *testing.T) { "homepage_panels": null, "search_onboarding": null, "repositories": null, -"repository_size_histogram": null, + "repository_size_histogram": null, + "retention_statistics": null, + "installer_email": "test@sourcegraph.com", + "auth_providers": "foo,bar", + "ext_services": "GITHUB,GITLAB", + "code_host_versions": null, + "builtin_signup_allowed": "true", + "deploy_type": "server", + "total_user_accounts": "234", + "has_external_url": "false", + "has_repos": "true", + "ever_searched": "false", + "ever_find_refs": "true", + "total_repos": "0", + "active_today": "false", + "os": "", + "timestamp": "`+now.UTC().Format(time.RFC3339)+`" + }`) +} + +func TestSerializeRepoMetadataUsage(t *testing.T) { + pr := &pingRequest{ + ClientSiteID: "0101-0101", + DeployType: "server", + ClientVersionString: "3.12.6", + AuthProviders: []string{"foo", "bar"}, + ExternalServices: []string{extsvc.KindGitHub, extsvc.KindGitLab}, + BuiltinSignupAllowed: true, + HasExtURL: false, + UniqueUsers: 123, + InitialAdminEmail: "test@sourcegraph.com", + TotalUsers: 234, + HasRepos: true, + EverSearched: false, + EverFindRefs: true, + RepoMetadataUsage: json.RawMessage(`{ + "summary": { + "is_enabled": true, + "repos_with_metadata_count": 10, + "repo_metadata_count": 100 + }, + "daily": { + "start_time": "2020-01-01T00:00:00Z", + "create_repo_metadata": { + "events_count": 10, + "users_count": 5 + } + } + }`), + } + + now := time.Now() + payload, err := marshalPing(pr, true, "127.0.0.1", now) + if err != nil { + t.Fatalf("unexpected error %s", err) + } + + compareJSON(t, payload, `{ + "access_request_enabled": "false", + "remote_ip": "127.0.0.1", + "remote_site_version": "3.12.6", + "repo_metadata_usage": null, + "remote_site_id": "0101-0101", + "license_key": "", + "has_update": "true", + "unique_users_today": "123", + "site_activity": null, + "batch_changes_usage": null, + "code_intel_usage": null, + "new_code_intel_usage": null, + "dependency_versions": null, + "extensions_usage": null, + "code_insights_usage": null, + "code_insights_critical_telemetry": null, + "code_monitoring_usage": null, + "cody_usage": null, + "notebooks_usage": null, + "code_host_integration_usage": null, + "ide_extensions_usage": null, + "migrated_extensions_usage": null, + "own_usage": null, + "repo_metadata_usage": { + "summary": { + "is_enabled": true, + "repos_with_metadata_count": 10, + "repo_metadata_count": 100 + }, + "daily": { + "start_time": "2020-01-01T00:00:00Z", + "create_repo_metadata": { + "events_count": 10, + "users_count": 5 + } + } + }, + "search_usage": null, + "growth_statistics": null, + "has_cody_enabled": "false", + "saved_searches": null, + "homepage_panels": null, + "search_onboarding": null, + "repositories": null, + "repository_size_histogram": null, "retention_statistics": null, "installer_email": "test@sourcegraph.com", "auth_providers": "foo,bar", diff --git a/doc/admin/pings.md b/doc/admin/pings.md index 4fd84ad9cca..a500e9f3620 100644 --- a/doc/admin/pings.md +++ b/doc/admin/pings.md @@ -191,11 +191,12 @@ Sourcegraph aggregates usage and performance metrics for some product features i - Sourcegraph Own usage data - Whether the `search-ownership` feature flag is turned on. - Number and ratio of repositories for which ownership data is available via CODEOWNERS file or the API. - - Aggregate monthly weekly and daily active users for the following activities: + - Narrowing search results by owner using `file:has.owners` predicate. - Selecting owner search result through `select:file.owners`. - Displaying ownership panel in file view. - Histogram of cloned repository sizes +- Aggregate daily, weekly, monthly repository metadata usage statistics ## Sourcegraph app telemetry diff --git a/internal/database/event_logs.go b/internal/database/event_logs.go index d9da88eef5a..77d97c92e9f 100644 --- a/internal/database/event_logs.go +++ b/internal/database/event_logs.go @@ -37,6 +37,9 @@ type EventLogStore interface { // AggregatedCodyEvents calculates CodyAggregatedEvent for each every unique event type related to Cody. AggregatedCodyEvents(ctx context.Context, now time.Time) ([]types.CodyAggregatedEvent, error) + // AggregatedRepoMetadataEvents calculates RepoMetadataAggregatedEvent for each every unique event type related to RepoMetadata. + AggregatedRepoMetadataEvents(ctx context.Context, now time.Time, period PeriodType) (*types.RepoMetadataAggregatedEvents, error) + // AggregatedSearchEvents calculates SearchAggregatedEvent for each every unique event type related to search. AggregatedSearchEvents(ctx context.Context, now time.Time) ([]types.SearchAggregatedEvent, error) @@ -1418,6 +1421,97 @@ func (l *eventLogStore) aggregatedCodyEvents(ctx context.Context, queryString st return events, nil } +func buildAggregatedRepoMetadataEventsQuery(period PeriodType) (string, error) { + unit := "" + switch period { + case Daily: + unit = "day" + case Weekly: + unit = "week" + case Monthly: + unit = "month" + default: + return "", ErrInvalidPeriodType + } + return ` + WITH events AS ( + SELECT + name, + ` + aggregatedUserIDQueryFragment + ` AS user_id, + argument + FROM event_logs + WHERE + timestamp >= ` + makeDateTruncExpression(unit, "%s::timestamp") + ` + AND name IN ('RepoMetadataAdded', 'RepoMetadataUpdated', 'RepoMetadataDeleted', 'SearchSubmitted') + ) + SELECT + ` + makeDateTruncExpression(unit, "%s::timestamp") + ` as start_time, + + COUNT(*) FILTER (WHERE name IN ('RepoMetadataAdded')) AS added_count, + COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataAdded')) AS added_unique_count, + + COUNT(*) FILTER (WHERE name IN ('RepoMetadataUpdated')) AS updated_count, + COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataUpdated')) AS updated_unique_count, + + COUNT(*) FILTER (WHERE name IN ('RepoMetadataDeleted')) AS deleted_count, + COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataDeleted')) AS deleted_unique_count, + + COUNT(*) FILTER ( + WHERE name IN ('SearchSubmitted') + AND ( + argument->>'query' ILIKE '%%repo:has(%%' + OR argument->>'query' ILIKE '%%repo:has.key(%%' + OR argument->>'query' ILIKE '%%repo:has.tag(%%' + OR argument->>'query' ILIKE '%%repo:has.meta(%%' + ) + ) AS searches_count, + COUNT(DISTINCT user_id) FILTER ( + WHERE name IN ('SearchSubmitted') + AND ( + argument->>'query' ILIKE '%%repo:has(%%' + OR argument->>'query' ILIKE '%%repo:has.key(%%' + OR argument->>'query' ILIKE '%%repo:has.tag(%%' + OR argument->>'query' ILIKE '%%repo:has.meta(%%' + ) + ) AS searches_unique_count + FROM events; + `, nil +} + +func (l *eventLogStore) AggregatedRepoMetadataEvents(ctx context.Context, now time.Time, period PeriodType) (*types.RepoMetadataAggregatedEvents, error) { + query, err := buildAggregatedRepoMetadataEventsQuery(period) + if err != nil { + return nil, err + } + row := l.QueryRow(ctx, sqlf.Sprintf(query, now, now)) + var startTime time.Time + var createEvent types.EventStats + var updateEvent types.EventStats + var deleteEvent types.EventStats + var searchEvent types.EventStats + if err := row.Scan( + &startTime, + &createEvent.EventsCount, + &createEvent.UsersCount, + &updateEvent.EventsCount, + &updateEvent.UsersCount, + &deleteEvent.EventsCount, + &deleteEvent.UsersCount, + &searchEvent.EventsCount, + &searchEvent.UsersCount, + ); err != nil { + return nil, err + } + + return &types.RepoMetadataAggregatedEvents{ + StartTime: startTime, + CreateRepoMetadata: &createEvent, + UpdateRepoMetadata: &updateEvent, + DeleteRepoMetadata: &deleteEvent, + SearchFilterUsage: &searchEvent, + }, nil +} + func (l *eventLogStore) AggregatedSearchEvents(ctx context.Context, now time.Time) ([]types.SearchAggregatedEvent, error) { latencyEvents, err := l.aggregatedSearchEvents(ctx, aggregatedSearchLatencyEventsQuery, now) if err != nil { diff --git a/internal/database/event_logs_test.go b/internal/database/event_logs_test.go index b65fbf2fc53..e643ef5dc91 100644 --- a/internal/database/event_logs_test.go +++ b/internal/database/event_logs_test.go @@ -1885,3 +1885,145 @@ func TestEventLogs_OwnershipFeatureActivity(t *testing.T) { }) } } + +func TestEventLogs_AggregatedRepoMetadataStats(t *testing.T) { + if testing.Short() { + t.Skip() + } + t.Parallel() + ptr := func(i int32) *int32 { return &i } + now := time.Date(2000, time.January, 20, 12, 0, 0, 0, time.UTC) + events := []*Event{ + { + UserID: 1, + Name: "RepoMetadataAdded", + Source: "BACKEND", + Timestamp: now, + }, + { + UserID: 1, + Name: "RepoMetadataAdded", + Source: "BACKEND", + Timestamp: now, + }, + { + UserID: 1, + Name: "RepoMetadataAdded", + Source: "BACKEND", + Timestamp: time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.UTC), + }, + { + UserID: 1, + Name: "RepoMetadataUpdated", + Source: "BACKEND", + Timestamp: now, + }, + { + UserID: 1, + Name: "RepoMetadataDeleted", + Source: "BACKEND", + Timestamp: now, + }, + { + UserID: 1, + Name: "SearchSubmitted", + Argument: json.RawMessage(`{"query": "repo:has(some:meta)"}`), + Source: "BACKEND", + Timestamp: now, + }, + } + logger := logtest.Scoped(t) + db := NewDB(logger, dbtest.NewDB(logger, t)) + ctx := context.Background() + for _, e := range events { + if err := db.EventLogs().Insert(ctx, e); err != nil { + t.Fatalf("failed inserting test data: %s", err) + } + } + + for name, testCase := range map[string]struct { + now time.Time + period PeriodType + stats *types.RepoMetadataAggregatedEvents + }{ + "daily": { + now: now, + period: Daily, + stats: &types.RepoMetadataAggregatedEvents{ + StartTime: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC), + CreateRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(2), + }, + UpdateRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + DeleteRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + SearchFilterUsage: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + }, + }, + "weekly": { + now: now, + period: Weekly, + stats: &types.RepoMetadataAggregatedEvents{ + StartTime: time.Date(now.Year(), now.Month(), now.Day()-int(now.Weekday()), 0, 0, 0, 0, time.UTC), + CreateRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(3), + }, + UpdateRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + DeleteRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + SearchFilterUsage: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + }, + }, + "monthly": { + now: now, + period: Monthly, + stats: &types.RepoMetadataAggregatedEvents{ + StartTime: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC), + CreateRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(3), + }, + UpdateRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + DeleteRepoMetadata: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + SearchFilterUsage: &types.EventStats{ + UsersCount: ptr(1), + EventsCount: ptr(1), + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + stats, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, testCase.now, testCase.period) + if err != nil { + t.Fatalf("querying activity failed: %s", err) + } + if diff := cmp.Diff(testCase.stats, stats); diff != "" { + t.Errorf("unexpected statistics returned:\n%s", diff) + } + }) + } +} diff --git a/internal/database/mocks_temp.go b/internal/database/mocks_temp.go index 06cd1cf89cf..53708155a3b 100644 --- a/internal/database/mocks_temp.go +++ b/internal/database/mocks_temp.go @@ -11835,6 +11835,10 @@ type MockEventLogStore struct { // AggregatedCodyEventsFunc is an instance of a mock function object // controlling the behavior of the method AggregatedCodyEvents. AggregatedCodyEventsFunc *EventLogStoreAggregatedCodyEventsFunc + // AggregatedRepoMetadataEventsFunc is an instance of a mock function + // object controlling the behavior of the method + // AggregatedRepoMetadataEvents. + AggregatedRepoMetadataEventsFunc *EventLogStoreAggregatedRepoMetadataEventsFunc // AggregatedSearchEventsFunc is an instance of a mock function object // controlling the behavior of the method AggregatedSearchEvents. AggregatedSearchEventsFunc *EventLogStoreAggregatedSearchEventsFunc @@ -11976,6 +11980,11 @@ func NewMockEventLogStore() *MockEventLogStore { return }, }, + AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{ + defaultHook: func(context.Context, time.Time, PeriodType) (r0 *types.RepoMetadataAggregatedEvents, r1 error) { + return + }, + }, AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{ defaultHook: func(context.Context, time.Time) (r0 []types.SearchAggregatedEvent, r1 error) { return @@ -12173,6 +12182,11 @@ func NewStrictMockEventLogStore() *MockEventLogStore { panic("unexpected invocation of MockEventLogStore.AggregatedCodyEvents") }, }, + AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{ + defaultHook: func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) { + panic("unexpected invocation of MockEventLogStore.AggregatedRepoMetadataEvents") + }, + }, AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{ defaultHook: func(context.Context, time.Time) ([]types.SearchAggregatedEvent, error) { panic("unexpected invocation of MockEventLogStore.AggregatedSearchEvents") @@ -12365,6 +12379,9 @@ func NewMockEventLogStoreFrom(i EventLogStore) *MockEventLogStore { AggregatedCodyEventsFunc: &EventLogStoreAggregatedCodyEventsFunc{ defaultHook: i.AggregatedCodyEvents, }, + AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{ + defaultHook: i.AggregatedRepoMetadataEvents, + }, AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{ defaultHook: i.AggregatedSearchEvents, }, @@ -12804,6 +12821,121 @@ func (c EventLogStoreAggregatedCodyEventsFuncCall) Results() []interface{} { return []interface{}{c.Result0, c.Result1} } +// EventLogStoreAggregatedRepoMetadataEventsFunc describes the behavior when +// the AggregatedRepoMetadataEvents method of the parent MockEventLogStore +// instance is invoked. +type EventLogStoreAggregatedRepoMetadataEventsFunc struct { + defaultHook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) + hooks []func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) + history []EventLogStoreAggregatedRepoMetadataEventsFuncCall + mutex sync.Mutex +} + +// AggregatedRepoMetadataEvents delegates to the next hook function in the +// queue and stores the parameter and result values of this invocation. +func (m *MockEventLogStore) AggregatedRepoMetadataEvents(v0 context.Context, v1 time.Time, v2 PeriodType) (*types.RepoMetadataAggregatedEvents, error) { + r0, r1 := m.AggregatedRepoMetadataEventsFunc.nextHook()(v0, v1, v2) + m.AggregatedRepoMetadataEventsFunc.appendCall(EventLogStoreAggregatedRepoMetadataEventsFuncCall{v0, v1, v2, r0, r1}) + return r0, r1 +} + +// SetDefaultHook sets function that is called when the +// AggregatedRepoMetadataEvents method of the parent MockEventLogStore +// instance is invoked and the hook queue is empty. +func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) SetDefaultHook(hook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)) { + f.defaultHook = hook +} + +// PushHook adds a function to the end of hook queue. Each invocation of the +// AggregatedRepoMetadataEvents method of the parent MockEventLogStore +// instance invokes the hook at the front of the queue and discards it. +// After the queue is empty, the default hook function is invoked for any +// future action. +func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) PushHook(hook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)) { + f.mutex.Lock() + f.hooks = append(f.hooks, hook) + f.mutex.Unlock() +} + +// SetDefaultReturn calls SetDefaultHook with a function that returns the +// given values. +func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) SetDefaultReturn(r0 *types.RepoMetadataAggregatedEvents, r1 error) { + f.SetDefaultHook(func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) { + return r0, r1 + }) +} + +// PushReturn calls PushHook with a function that returns the given values. +func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) PushReturn(r0 *types.RepoMetadataAggregatedEvents, r1 error) { + f.PushHook(func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) { + return r0, r1 + }) +} + +func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) nextHook() func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + if len(f.hooks) == 0 { + return f.defaultHook + } + + hook := f.hooks[0] + f.hooks = f.hooks[1:] + return hook +} + +func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) appendCall(r0 EventLogStoreAggregatedRepoMetadataEventsFuncCall) { + f.mutex.Lock() + f.history = append(f.history, r0) + f.mutex.Unlock() +} + +// History returns a sequence of +// EventLogStoreAggregatedRepoMetadataEventsFuncCall objects describing the +// invocations of this function. +func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) History() []EventLogStoreAggregatedRepoMetadataEventsFuncCall { + f.mutex.Lock() + history := make([]EventLogStoreAggregatedRepoMetadataEventsFuncCall, len(f.history)) + copy(history, f.history) + f.mutex.Unlock() + + return history +} + +// EventLogStoreAggregatedRepoMetadataEventsFuncCall is an object that +// describes an invocation of method AggregatedRepoMetadataEvents on an +// instance of MockEventLogStore. +type EventLogStoreAggregatedRepoMetadataEventsFuncCall struct { + // Arg0 is the value of the 1st argument passed to this method + // invocation. + Arg0 context.Context + // Arg1 is the value of the 2nd argument passed to this method + // invocation. + Arg1 time.Time + // Arg2 is the value of the 3rd argument passed to this method + // invocation. + Arg2 PeriodType + // Result0 is the value of the 1st result returned from this method + // invocation. + Result0 *types.RepoMetadataAggregatedEvents + // Result1 is the value of the 2nd result returned from this method + // invocation. + Result1 error +} + +// Args returns an interface slice containing the arguments of this +// invocation. +func (c EventLogStoreAggregatedRepoMetadataEventsFuncCall) Args() []interface{} { + return []interface{}{c.Arg0, c.Arg1, c.Arg2} +} + +// Results returns an interface slice containing the results of this +// invocation. +func (c EventLogStoreAggregatedRepoMetadataEventsFuncCall) Results() []interface{} { + return []interface{}{c.Result0, c.Result1} +} + // EventLogStoreAggregatedSearchEventsFunc describes the behavior when the // AggregatedSearchEvents method of the parent MockEventLogStore instance is // invoked. diff --git a/internal/types/types.go b/internal/types/types.go index 6371432568b..04d83fcb08c 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1021,6 +1021,37 @@ type CodyAggregatedEvent struct { InvalidDay int32 } +// NOTE: DO NOT alter this struct without making a symmetric change +// to the updatecheck handler. +// RepoMetadataAggregatedStats represents the total number of repo metadata, +// number of repositories with any metadata, total and unique number of +// events for repo metadata usage related events over the current day, week, month. +type RepoMetadataAggregatedStats struct { + Summary *RepoMetadataAggregatedSummary + Daily *RepoMetadataAggregatedEvents + Weekly *RepoMetadataAggregatedEvents + Monthly *RepoMetadataAggregatedEvents +} + +type RepoMetadataAggregatedSummary struct { + IsEnabled bool + RepoMetadataCount *int32 + ReposWithMetadataCount *int32 +} + +type RepoMetadataAggregatedEvents struct { + StartTime time.Time + CreateRepoMetadata *EventStats + UpdateRepoMetadata *EventStats + DeleteRepoMetadata *EventStats + SearchFilterUsage *EventStats +} + +type EventStats struct { + UsersCount *int32 + EventsCount *int32 +} + // NOTE: DO NOT alter this struct without making a symmetric change // to the updatecheck handler. This struct is marshalled and sent to // BigQuery, which requires the input match its schema exactly. diff --git a/internal/usagestats/BUILD.bazel b/internal/usagestats/BUILD.bazel index 973fda501e9..89033c0238c 100644 --- a/internal/usagestats/BUILD.bazel +++ b/internal/usagestats/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "aggregated.go", "aggregated_codeintel.go", + "aggregated_repo_metadata.go", "aggregated_search.go", "all_time_stats.go", "batches.go", diff --git a/internal/usagestats/aggregated_repo_metadata.go b/internal/usagestats/aggregated_repo_metadata.go new file mode 100644 index 00000000000..5e8f287276d --- /dev/null +++ b/internal/usagestats/aggregated_repo_metadata.go @@ -0,0 +1,60 @@ +package usagestats + +import ( + "context" + "time" + + "github.com/sourcegraph/sourcegraph/internal/database" + "github.com/sourcegraph/sourcegraph/internal/types" +) + +func GetAggregatedRepoMetadataStats(ctx context.Context, db database.DB) (*types.RepoMetadataAggregatedStats, error) { + now := time.Now().UTC() + daily, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Daily) + if err != nil { + return nil, err + } + + weekly, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Weekly) + if err != nil { + return nil, err + } + + monthly, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Monthly) + if err != nil { + return nil, err + } + + summary, err := getAggregatedRepoMetadataSummary(ctx, db) + if err != nil { + return nil, err + } + + return &types.RepoMetadataAggregatedStats{ + Summary: summary, + Daily: daily, + Weekly: weekly, + Monthly: monthly, + }, nil +} + +func getAggregatedRepoMetadataSummary(ctx context.Context, db database.DB) (*types.RepoMetadataAggregatedSummary, error) { + q := ` + SELECT + COUNT(*) AS total_count, + COUNT(DISTINCT repo_id) AS total_repos_count + FROM repo_kvps + ` + var summary types.RepoMetadataAggregatedSummary + err := db.QueryRowContext(ctx, q).Scan(&summary.RepoMetadataCount, &summary.ReposWithMetadataCount) + if err != nil { + return nil, err + } + flag, err := db.FeatureFlags().GetFeatureFlag(ctx, "repository-metadata") + if err != nil { + return nil, err + } + summary.IsEnabled = flag != nil && flag.Bool.Value + + return &summary, nil +}