(analytics) add repo meta usage ping (#51921)

* (analytics) adds backend event logs on repo metadata create/update/delete
This commit is contained in:
Erzhan Torokulov 2023-05-24 13:25:18 +06:00 committed by GitHub
parent 77ae92fc11
commit c205dedff2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 632 additions and 5 deletions

View File

@ -407,6 +407,7 @@ export const SiteAdminPingsPage: React.FunctionComponent<React.PropsWithChildren
</ul>
</li>
<li>Histogram of cloned repository sizes</li>
<li>Aggregate daily, weekly, monthly repository metadata usage statistics</li>
</ul>
{updatesDisabled && <Text>All telemetry is disabled.</Text>}
</Container>

View File

@ -252,6 +252,7 @@ go_library(
"//internal/database",
"//internal/database/migration/cliutil",
"//internal/database/migration/schemas",
"//internal/deviceid",
"//internal/encryption",
"//internal/encryption/keyring",
"//internal/env",

View File

@ -9,9 +9,12 @@ import (
"github.com/graph-gophers/graphql-go/relay"
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/graphqlutil"
"github.com/sourcegraph/sourcegraph/internal/actor"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/deviceid"
"github.com/sourcegraph/sourcegraph/internal/featureflag"
"github.com/sourcegraph/sourcegraph/internal/rbac"
"github.com/sourcegraph/sourcegraph/internal/usagestats"
"github.com/sourcegraph/sourcegraph/lib/errors"
)
@ -69,7 +72,12 @@ func (r *schemaResolver) AddRepoMetadata(ctx context.Context, args struct {
return &EmptyResponse{}, emptyNonNilValueError{value: *args.Value}
}
return &EmptyResponse{}, r.db.RepoKVPs().Create(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value})
err = r.db.RepoKVPs().Create(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value})
if err == nil {
r.logBackendEvent(ctx, "RepoMetadataAdded")
}
return &EmptyResponse{}, err
}
// Deprecated: Use UpdateRepoMetadata instead.
@ -106,6 +114,9 @@ func (r *schemaResolver) UpdateRepoMetadata(ctx context.Context, args struct {
}
_, err = r.db.RepoKVPs().Update(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value})
if err == nil {
r.logBackendEvent(ctx, "RepoMetadataUpdated")
}
return &EmptyResponse{}, err
}
@ -136,7 +147,29 @@ func (r *schemaResolver) DeleteRepoMetadata(ctx context.Context, args struct {
return &EmptyResponse{}, err
}
return &EmptyResponse{}, r.db.RepoKVPs().Delete(ctx, repoID, args.Key)
err = r.db.RepoKVPs().Delete(ctx, repoID, args.Key)
if err == nil {
r.logBackendEvent(ctx, "RepoMetadataDeleted")
}
return &EmptyResponse{}, err
}
func (r *schemaResolver) logBackendEvent(ctx context.Context, eventName string) {
a := actor.FromContext(ctx)
if a.IsAuthenticated() && !a.IsMockUser() {
if err := usagestats.LogBackendEvent(
r.db,
a.UID,
deviceid.FromContext(ctx),
eventName,
nil,
nil,
featureflag.GetEvaluatedFlagSet(ctx),
nil,
); err != nil {
r.logger.Warn("Could not log " + eventName)
}
}
}
type repoMetaResolver struct {

View File

@ -363,6 +363,17 @@ func getAndMarshalCodyUsageJSON(ctx context.Context, db database.DB) (_ json.Raw
return json.Marshal(codyUsage)
}
func getAndMarshalRepoMetadataUsageJSON(ctx context.Context, db database.DB) (_ json.RawMessage, err error) {
defer recordOperation("getAndMarshalRepoMetadataUsageJSON")(&err)
repoMetadataUsage, err := usagestats.GetAggregatedRepoMetadataStats(ctx, db)
if err != nil {
return nil, err
}
return json.Marshal(repoMetadataUsage)
}
func getDependencyVersions(ctx context.Context, db database.DB, logger log.Logger) (json.RawMessage, error) {
logFunc := logFuncFrom(logger.Scoped("getDependencyVersions", "gets the version of various dependency services"))
var (
@ -519,6 +530,7 @@ func updateBody(ctx context.Context, logger log.Logger, db database.DB) (io.Read
IDEExtensionsUsage: []byte("{}"),
MigratedExtensionsUsage: []byte("{}"),
CodyUsage: []byte("{}"),
RepoMetadataUsage: []byte("{}"),
}
totalUsers, err := getTotalUsersCount(ctx, db)
@ -668,6 +680,11 @@ func updateBody(ctx context.Context, logger log.Logger, db database.DB) (io.Read
logFunc("codyUsage failed", log.Error(err))
}
r.RepoMetadataUsage, err = getAndMarshalRepoMetadataUsageJSON(ctx, db)
if err != nil {
logFunc("repoMetadataUsage failed", log.Error(err))
}
r.HasExtURL = conf.UsingExternalURL()
r.BuiltinSignupAllowed = conf.IsBuiltinSignupAllowed()
r.AccessRequestEnabled = conf.IsAccessRequestEnabled()

View File

@ -247,6 +247,7 @@ type pingRequest struct {
ActiveToday bool `json:"activeToday,omitempty"` // Only used in Sourcegraph App
HasCodyEnabled bool `json:"hasCodyEnabled,omitempty"`
CodyUsage json.RawMessage `json:"codyUsage,omitempty"`
RepoMetadataUsage json.RawMessage `json:"repoMetadataUsage,omitempty"`
}
type dependencyVersions struct {
@ -373,6 +374,7 @@ type pingPayload struct {
Timestamp string `json:"timestamp"`
HasCodyEnabled string `json:"has_cody_enabled"`
CodyUsage json.RawMessage `json:"cody_usage"`
RepoMetadataUsage json.RawMessage `json:"repo_metadata_usage"`
}
func logPing(logger log.Logger, r *http.Request, pr *pingRequest, hasUpdate bool) {
@ -475,6 +477,7 @@ func marshalPing(pr *pingRequest, hasUpdate bool, clientAddr string, now time.Ti
Timestamp: now.UTC().Format(time.RFC3339),
HasCodyEnabled: strconv.FormatBool(codyFeatureFlag()),
CodyUsage: codyUsage,
RepoMetadataUsage: pr.RepoMetadataUsage,
})
}

View File

@ -207,6 +207,7 @@ func TestSerializeBasic(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "mylicense",
"has_update": "true",
@ -282,6 +283,7 @@ func TestSerializeLimited(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "2023.03.23+205275.dd37e7",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "",
"has_update": "true",
@ -359,6 +361,7 @@ func TestSerializeFromQuery(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "",
"has_update": "true",
@ -419,6 +422,7 @@ func TestSerializeBatchChangesUsage(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "mylicense",
"has_update": "true",
@ -479,6 +483,7 @@ func TestSerializeGrowthStatistics(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "mylicense",
"has_update": "true",
@ -505,7 +510,7 @@ func TestSerializeGrowthStatistics(t *testing.T) {
"search_onboarding": null,
"homepage_panels": null,
"repositories": null,
"repository_size_histogram": null,
"repository_size_histogram": null,
"retention_statistics": null,
"installer_email": "test@sourcegraph.com",
"auth_providers": "foo,bar",
@ -640,6 +645,7 @@ func TestSerializeCodeIntelUsage(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "mylicense",
"has_update": "true",
@ -822,6 +828,7 @@ func TestSerializeOldCodeIntelUsage(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "mylicense",
"has_update": "true",
@ -952,6 +959,7 @@ func TestSerializeCodeHostVersions(t *testing.T) {
compareJSON(t, payload, `{
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "mylicense",
"has_update": "true",
@ -1048,6 +1056,7 @@ func TestSerializeOwn(t *testing.T) {
"access_request_enabled": "false",
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "",
"has_update": "true",
@ -1095,7 +1104,109 @@ func TestSerializeOwn(t *testing.T) {
"homepage_panels": null,
"search_onboarding": null,
"repositories": null,
"repository_size_histogram": null,
"repository_size_histogram": null,
"retention_statistics": null,
"installer_email": "test@sourcegraph.com",
"auth_providers": "foo,bar",
"ext_services": "GITHUB,GITLAB",
"code_host_versions": null,
"builtin_signup_allowed": "true",
"deploy_type": "server",
"total_user_accounts": "234",
"has_external_url": "false",
"has_repos": "true",
"ever_searched": "false",
"ever_find_refs": "true",
"total_repos": "0",
"active_today": "false",
"os": "",
"timestamp": "`+now.UTC().Format(time.RFC3339)+`"
}`)
}
func TestSerializeRepoMetadataUsage(t *testing.T) {
pr := &pingRequest{
ClientSiteID: "0101-0101",
DeployType: "server",
ClientVersionString: "3.12.6",
AuthProviders: []string{"foo", "bar"},
ExternalServices: []string{extsvc.KindGitHub, extsvc.KindGitLab},
BuiltinSignupAllowed: true,
HasExtURL: false,
UniqueUsers: 123,
InitialAdminEmail: "test@sourcegraph.com",
TotalUsers: 234,
HasRepos: true,
EverSearched: false,
EverFindRefs: true,
RepoMetadataUsage: json.RawMessage(`{
"summary": {
"is_enabled": true,
"repos_with_metadata_count": 10,
"repo_metadata_count": 100
},
"daily": {
"start_time": "2020-01-01T00:00:00Z",
"create_repo_metadata": {
"events_count": 10,
"users_count": 5
}
}
}`),
}
now := time.Now()
payload, err := marshalPing(pr, true, "127.0.0.1", now)
if err != nil {
t.Fatalf("unexpected error %s", err)
}
compareJSON(t, payload, `{
"access_request_enabled": "false",
"remote_ip": "127.0.0.1",
"remote_site_version": "3.12.6",
"repo_metadata_usage": null,
"remote_site_id": "0101-0101",
"license_key": "",
"has_update": "true",
"unique_users_today": "123",
"site_activity": null,
"batch_changes_usage": null,
"code_intel_usage": null,
"new_code_intel_usage": null,
"dependency_versions": null,
"extensions_usage": null,
"code_insights_usage": null,
"code_insights_critical_telemetry": null,
"code_monitoring_usage": null,
"cody_usage": null,
"notebooks_usage": null,
"code_host_integration_usage": null,
"ide_extensions_usage": null,
"migrated_extensions_usage": null,
"own_usage": null,
"repo_metadata_usage": {
"summary": {
"is_enabled": true,
"repos_with_metadata_count": 10,
"repo_metadata_count": 100
},
"daily": {
"start_time": "2020-01-01T00:00:00Z",
"create_repo_metadata": {
"events_count": 10,
"users_count": 5
}
}
},
"search_usage": null,
"growth_statistics": null,
"has_cody_enabled": "false",
"saved_searches": null,
"homepage_panels": null,
"search_onboarding": null,
"repositories": null,
"repository_size_histogram": null,
"retention_statistics": null,
"installer_email": "test@sourcegraph.com",
"auth_providers": "foo,bar",

View File

@ -191,11 +191,12 @@ Sourcegraph aggregates usage and performance metrics for some product features i
- Sourcegraph Own usage data
- Whether the `search-ownership` feature flag is turned on.
- Number and ratio of repositories for which ownership data is available via CODEOWNERS file or the API.
- Aggregate monthly weekly and daily active users for the following activities:
<!-- - Aggregate monthly weekly and daily active users for the following activities: -->
- Narrowing search results by owner using `file:has.owners` predicate.
- Selecting owner search result through `select:file.owners`.
- Displaying ownership panel in file view.
- Histogram of cloned repository sizes
- Aggregate daily, weekly, monthly repository metadata usage statistics
</details>
## Sourcegraph app telemetry

View File

@ -37,6 +37,9 @@ type EventLogStore interface {
// AggregatedCodyEvents calculates CodyAggregatedEvent for each every unique event type related to Cody.
AggregatedCodyEvents(ctx context.Context, now time.Time) ([]types.CodyAggregatedEvent, error)
// AggregatedRepoMetadataEvents calculates RepoMetadataAggregatedEvent for each every unique event type related to RepoMetadata.
AggregatedRepoMetadataEvents(ctx context.Context, now time.Time, period PeriodType) (*types.RepoMetadataAggregatedEvents, error)
// AggregatedSearchEvents calculates SearchAggregatedEvent for each every unique event type related to search.
AggregatedSearchEvents(ctx context.Context, now time.Time) ([]types.SearchAggregatedEvent, error)
@ -1418,6 +1421,97 @@ func (l *eventLogStore) aggregatedCodyEvents(ctx context.Context, queryString st
return events, nil
}
func buildAggregatedRepoMetadataEventsQuery(period PeriodType) (string, error) {
unit := ""
switch period {
case Daily:
unit = "day"
case Weekly:
unit = "week"
case Monthly:
unit = "month"
default:
return "", ErrInvalidPeriodType
}
return `
WITH events AS (
SELECT
name,
` + aggregatedUserIDQueryFragment + ` AS user_id,
argument
FROM event_logs
WHERE
timestamp >= ` + makeDateTruncExpression(unit, "%s::timestamp") + `
AND name IN ('RepoMetadataAdded', 'RepoMetadataUpdated', 'RepoMetadataDeleted', 'SearchSubmitted')
)
SELECT
` + makeDateTruncExpression(unit, "%s::timestamp") + ` as start_time,
COUNT(*) FILTER (WHERE name IN ('RepoMetadataAdded')) AS added_count,
COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataAdded')) AS added_unique_count,
COUNT(*) FILTER (WHERE name IN ('RepoMetadataUpdated')) AS updated_count,
COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataUpdated')) AS updated_unique_count,
COUNT(*) FILTER (WHERE name IN ('RepoMetadataDeleted')) AS deleted_count,
COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataDeleted')) AS deleted_unique_count,
COUNT(*) FILTER (
WHERE name IN ('SearchSubmitted')
AND (
argument->>'query' ILIKE '%%repo:has(%%'
OR argument->>'query' ILIKE '%%repo:has.key(%%'
OR argument->>'query' ILIKE '%%repo:has.tag(%%'
OR argument->>'query' ILIKE '%%repo:has.meta(%%'
)
) AS searches_count,
COUNT(DISTINCT user_id) FILTER (
WHERE name IN ('SearchSubmitted')
AND (
argument->>'query' ILIKE '%%repo:has(%%'
OR argument->>'query' ILIKE '%%repo:has.key(%%'
OR argument->>'query' ILIKE '%%repo:has.tag(%%'
OR argument->>'query' ILIKE '%%repo:has.meta(%%'
)
) AS searches_unique_count
FROM events;
`, nil
}
func (l *eventLogStore) AggregatedRepoMetadataEvents(ctx context.Context, now time.Time, period PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
query, err := buildAggregatedRepoMetadataEventsQuery(period)
if err != nil {
return nil, err
}
row := l.QueryRow(ctx, sqlf.Sprintf(query, now, now))
var startTime time.Time
var createEvent types.EventStats
var updateEvent types.EventStats
var deleteEvent types.EventStats
var searchEvent types.EventStats
if err := row.Scan(
&startTime,
&createEvent.EventsCount,
&createEvent.UsersCount,
&updateEvent.EventsCount,
&updateEvent.UsersCount,
&deleteEvent.EventsCount,
&deleteEvent.UsersCount,
&searchEvent.EventsCount,
&searchEvent.UsersCount,
); err != nil {
return nil, err
}
return &types.RepoMetadataAggregatedEvents{
StartTime: startTime,
CreateRepoMetadata: &createEvent,
UpdateRepoMetadata: &updateEvent,
DeleteRepoMetadata: &deleteEvent,
SearchFilterUsage: &searchEvent,
}, nil
}
func (l *eventLogStore) AggregatedSearchEvents(ctx context.Context, now time.Time) ([]types.SearchAggregatedEvent, error) {
latencyEvents, err := l.aggregatedSearchEvents(ctx, aggregatedSearchLatencyEventsQuery, now)
if err != nil {

View File

@ -1885,3 +1885,145 @@ func TestEventLogs_OwnershipFeatureActivity(t *testing.T) {
})
}
}
func TestEventLogs_AggregatedRepoMetadataStats(t *testing.T) {
if testing.Short() {
t.Skip()
}
t.Parallel()
ptr := func(i int32) *int32 { return &i }
now := time.Date(2000, time.January, 20, 12, 0, 0, 0, time.UTC)
events := []*Event{
{
UserID: 1,
Name: "RepoMetadataAdded",
Source: "BACKEND",
Timestamp: now,
},
{
UserID: 1,
Name: "RepoMetadataAdded",
Source: "BACKEND",
Timestamp: now,
},
{
UserID: 1,
Name: "RepoMetadataAdded",
Source: "BACKEND",
Timestamp: time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.UTC),
},
{
UserID: 1,
Name: "RepoMetadataUpdated",
Source: "BACKEND",
Timestamp: now,
},
{
UserID: 1,
Name: "RepoMetadataDeleted",
Source: "BACKEND",
Timestamp: now,
},
{
UserID: 1,
Name: "SearchSubmitted",
Argument: json.RawMessage(`{"query": "repo:has(some:meta)"}`),
Source: "BACKEND",
Timestamp: now,
},
}
logger := logtest.Scoped(t)
db := NewDB(logger, dbtest.NewDB(logger, t))
ctx := context.Background()
for _, e := range events {
if err := db.EventLogs().Insert(ctx, e); err != nil {
t.Fatalf("failed inserting test data: %s", err)
}
}
for name, testCase := range map[string]struct {
now time.Time
period PeriodType
stats *types.RepoMetadataAggregatedEvents
}{
"daily": {
now: now,
period: Daily,
stats: &types.RepoMetadataAggregatedEvents{
StartTime: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC),
CreateRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(2),
},
UpdateRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
DeleteRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
SearchFilterUsage: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
},
},
"weekly": {
now: now,
period: Weekly,
stats: &types.RepoMetadataAggregatedEvents{
StartTime: time.Date(now.Year(), now.Month(), now.Day()-int(now.Weekday()), 0, 0, 0, 0, time.UTC),
CreateRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(3),
},
UpdateRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
DeleteRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
SearchFilterUsage: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
},
},
"monthly": {
now: now,
period: Monthly,
stats: &types.RepoMetadataAggregatedEvents{
StartTime: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC),
CreateRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(3),
},
UpdateRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
DeleteRepoMetadata: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
SearchFilterUsage: &types.EventStats{
UsersCount: ptr(1),
EventsCount: ptr(1),
},
},
},
} {
t.Run(name, func(t *testing.T) {
stats, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, testCase.now, testCase.period)
if err != nil {
t.Fatalf("querying activity failed: %s", err)
}
if diff := cmp.Diff(testCase.stats, stats); diff != "" {
t.Errorf("unexpected statistics returned:\n%s", diff)
}
})
}
}

View File

@ -11835,6 +11835,10 @@ type MockEventLogStore struct {
// AggregatedCodyEventsFunc is an instance of a mock function object
// controlling the behavior of the method AggregatedCodyEvents.
AggregatedCodyEventsFunc *EventLogStoreAggregatedCodyEventsFunc
// AggregatedRepoMetadataEventsFunc is an instance of a mock function
// object controlling the behavior of the method
// AggregatedRepoMetadataEvents.
AggregatedRepoMetadataEventsFunc *EventLogStoreAggregatedRepoMetadataEventsFunc
// AggregatedSearchEventsFunc is an instance of a mock function object
// controlling the behavior of the method AggregatedSearchEvents.
AggregatedSearchEventsFunc *EventLogStoreAggregatedSearchEventsFunc
@ -11976,6 +11980,11 @@ func NewMockEventLogStore() *MockEventLogStore {
return
},
},
AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{
defaultHook: func(context.Context, time.Time, PeriodType) (r0 *types.RepoMetadataAggregatedEvents, r1 error) {
return
},
},
AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{
defaultHook: func(context.Context, time.Time) (r0 []types.SearchAggregatedEvent, r1 error) {
return
@ -12173,6 +12182,11 @@ func NewStrictMockEventLogStore() *MockEventLogStore {
panic("unexpected invocation of MockEventLogStore.AggregatedCodyEvents")
},
},
AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{
defaultHook: func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
panic("unexpected invocation of MockEventLogStore.AggregatedRepoMetadataEvents")
},
},
AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{
defaultHook: func(context.Context, time.Time) ([]types.SearchAggregatedEvent, error) {
panic("unexpected invocation of MockEventLogStore.AggregatedSearchEvents")
@ -12365,6 +12379,9 @@ func NewMockEventLogStoreFrom(i EventLogStore) *MockEventLogStore {
AggregatedCodyEventsFunc: &EventLogStoreAggregatedCodyEventsFunc{
defaultHook: i.AggregatedCodyEvents,
},
AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{
defaultHook: i.AggregatedRepoMetadataEvents,
},
AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{
defaultHook: i.AggregatedSearchEvents,
},
@ -12804,6 +12821,121 @@ func (c EventLogStoreAggregatedCodyEventsFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1}
}
// EventLogStoreAggregatedRepoMetadataEventsFunc describes the behavior when
// the AggregatedRepoMetadataEvents method of the parent MockEventLogStore
// instance is invoked.
type EventLogStoreAggregatedRepoMetadataEventsFunc struct {
defaultHook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)
hooks []func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)
history []EventLogStoreAggregatedRepoMetadataEventsFuncCall
mutex sync.Mutex
}
// AggregatedRepoMetadataEvents delegates to the next hook function in the
// queue and stores the parameter and result values of this invocation.
func (m *MockEventLogStore) AggregatedRepoMetadataEvents(v0 context.Context, v1 time.Time, v2 PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
r0, r1 := m.AggregatedRepoMetadataEventsFunc.nextHook()(v0, v1, v2)
m.AggregatedRepoMetadataEventsFunc.appendCall(EventLogStoreAggregatedRepoMetadataEventsFuncCall{v0, v1, v2, r0, r1})
return r0, r1
}
// SetDefaultHook sets function that is called when the
// AggregatedRepoMetadataEvents method of the parent MockEventLogStore
// instance is invoked and the hook queue is empty.
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) SetDefaultHook(hook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)) {
f.defaultHook = hook
}
// PushHook adds a function to the end of hook queue. Each invocation of the
// AggregatedRepoMetadataEvents method of the parent MockEventLogStore
// instance invokes the hook at the front of the queue and discards it.
// After the queue is empty, the default hook function is invoked for any
// future action.
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) PushHook(hook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)) {
f.mutex.Lock()
f.hooks = append(f.hooks, hook)
f.mutex.Unlock()
}
// SetDefaultReturn calls SetDefaultHook with a function that returns the
// given values.
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) SetDefaultReturn(r0 *types.RepoMetadataAggregatedEvents, r1 error) {
f.SetDefaultHook(func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
return r0, r1
})
}
// PushReturn calls PushHook with a function that returns the given values.
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) PushReturn(r0 *types.RepoMetadataAggregatedEvents, r1 error) {
f.PushHook(func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
return r0, r1
})
}
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) nextHook() func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
f.mutex.Lock()
defer f.mutex.Unlock()
if len(f.hooks) == 0 {
return f.defaultHook
}
hook := f.hooks[0]
f.hooks = f.hooks[1:]
return hook
}
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) appendCall(r0 EventLogStoreAggregatedRepoMetadataEventsFuncCall) {
f.mutex.Lock()
f.history = append(f.history, r0)
f.mutex.Unlock()
}
// History returns a sequence of
// EventLogStoreAggregatedRepoMetadataEventsFuncCall objects describing the
// invocations of this function.
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) History() []EventLogStoreAggregatedRepoMetadataEventsFuncCall {
f.mutex.Lock()
history := make([]EventLogStoreAggregatedRepoMetadataEventsFuncCall, len(f.history))
copy(history, f.history)
f.mutex.Unlock()
return history
}
// EventLogStoreAggregatedRepoMetadataEventsFuncCall is an object that
// describes an invocation of method AggregatedRepoMetadataEvents on an
// instance of MockEventLogStore.
type EventLogStoreAggregatedRepoMetadataEventsFuncCall struct {
// Arg0 is the value of the 1st argument passed to this method
// invocation.
Arg0 context.Context
// Arg1 is the value of the 2nd argument passed to this method
// invocation.
Arg1 time.Time
// Arg2 is the value of the 3rd argument passed to this method
// invocation.
Arg2 PeriodType
// Result0 is the value of the 1st result returned from this method
// invocation.
Result0 *types.RepoMetadataAggregatedEvents
// Result1 is the value of the 2nd result returned from this method
// invocation.
Result1 error
}
// Args returns an interface slice containing the arguments of this
// invocation.
func (c EventLogStoreAggregatedRepoMetadataEventsFuncCall) Args() []interface{} {
return []interface{}{c.Arg0, c.Arg1, c.Arg2}
}
// Results returns an interface slice containing the results of this
// invocation.
func (c EventLogStoreAggregatedRepoMetadataEventsFuncCall) Results() []interface{} {
return []interface{}{c.Result0, c.Result1}
}
// EventLogStoreAggregatedSearchEventsFunc describes the behavior when the
// AggregatedSearchEvents method of the parent MockEventLogStore instance is
// invoked.

View File

@ -1021,6 +1021,37 @@ type CodyAggregatedEvent struct {
InvalidDay int32
}
// NOTE: DO NOT alter this struct without making a symmetric change
// to the updatecheck handler.
// RepoMetadataAggregatedStats represents the total number of repo metadata,
// number of repositories with any metadata, total and unique number of
// events for repo metadata usage related events over the current day, week, month.
type RepoMetadataAggregatedStats struct {
Summary *RepoMetadataAggregatedSummary
Daily *RepoMetadataAggregatedEvents
Weekly *RepoMetadataAggregatedEvents
Monthly *RepoMetadataAggregatedEvents
}
type RepoMetadataAggregatedSummary struct {
IsEnabled bool
RepoMetadataCount *int32
ReposWithMetadataCount *int32
}
type RepoMetadataAggregatedEvents struct {
StartTime time.Time
CreateRepoMetadata *EventStats
UpdateRepoMetadata *EventStats
DeleteRepoMetadata *EventStats
SearchFilterUsage *EventStats
}
type EventStats struct {
UsersCount *int32
EventsCount *int32
}
// NOTE: DO NOT alter this struct without making a symmetric change
// to the updatecheck handler. This struct is marshalled and sent to
// BigQuery, which requires the input match its schema exactly.

View File

@ -5,6 +5,7 @@ go_library(
srcs = [
"aggregated.go",
"aggregated_codeintel.go",
"aggregated_repo_metadata.go",
"aggregated_search.go",
"all_time_stats.go",
"batches.go",

View File

@ -0,0 +1,60 @@
package usagestats
import (
"context"
"time"
"github.com/sourcegraph/sourcegraph/internal/database"
"github.com/sourcegraph/sourcegraph/internal/types"
)
func GetAggregatedRepoMetadataStats(ctx context.Context, db database.DB) (*types.RepoMetadataAggregatedStats, error) {
now := time.Now().UTC()
daily, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Daily)
if err != nil {
return nil, err
}
weekly, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Weekly)
if err != nil {
return nil, err
}
monthly, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Monthly)
if err != nil {
return nil, err
}
summary, err := getAggregatedRepoMetadataSummary(ctx, db)
if err != nil {
return nil, err
}
return &types.RepoMetadataAggregatedStats{
Summary: summary,
Daily: daily,
Weekly: weekly,
Monthly: monthly,
}, nil
}
func getAggregatedRepoMetadataSummary(ctx context.Context, db database.DB) (*types.RepoMetadataAggregatedSummary, error) {
q := `
SELECT
COUNT(*) AS total_count,
COUNT(DISTINCT repo_id) AS total_repos_count
FROM repo_kvps
`
var summary types.RepoMetadataAggregatedSummary
err := db.QueryRowContext(ctx, q).Scan(&summary.RepoMetadataCount, &summary.ReposWithMetadataCount)
if err != nil {
return nil, err
}
flag, err := db.FeatureFlags().GetFeatureFlag(ctx, "repository-metadata")
if err != nil {
return nil, err
}
summary.IsEnabled = flag != nil && flag.Bool.Value
return &summary, nil
}