mirror of
https://github.com/sourcegraph/sourcegraph.git
synced 2026-02-06 18:31:54 +00:00
(analytics) add repo meta usage ping (#51921)
* (analytics) adds backend event logs on repo metadata create/update/delete
This commit is contained in:
parent
77ae92fc11
commit
c205dedff2
@ -407,6 +407,7 @@ export const SiteAdminPingsPage: React.FunctionComponent<React.PropsWithChildren
|
||||
</ul>
|
||||
</li>
|
||||
<li>Histogram of cloned repository sizes</li>
|
||||
<li>Aggregate daily, weekly, monthly repository metadata usage statistics</li>
|
||||
</ul>
|
||||
{updatesDisabled && <Text>All telemetry is disabled.</Text>}
|
||||
</Container>
|
||||
|
||||
1
cmd/frontend/graphqlbackend/BUILD.bazel
generated
1
cmd/frontend/graphqlbackend/BUILD.bazel
generated
@ -252,6 +252,7 @@ go_library(
|
||||
"//internal/database",
|
||||
"//internal/database/migration/cliutil",
|
||||
"//internal/database/migration/schemas",
|
||||
"//internal/deviceid",
|
||||
"//internal/encryption",
|
||||
"//internal/encryption/keyring",
|
||||
"//internal/env",
|
||||
|
||||
@ -9,9 +9,12 @@ import (
|
||||
"github.com/graph-gophers/graphql-go/relay"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/cmd/frontend/graphqlbackend/graphqlutil"
|
||||
"github.com/sourcegraph/sourcegraph/internal/actor"
|
||||
"github.com/sourcegraph/sourcegraph/internal/database"
|
||||
"github.com/sourcegraph/sourcegraph/internal/deviceid"
|
||||
"github.com/sourcegraph/sourcegraph/internal/featureflag"
|
||||
"github.com/sourcegraph/sourcegraph/internal/rbac"
|
||||
"github.com/sourcegraph/sourcegraph/internal/usagestats"
|
||||
"github.com/sourcegraph/sourcegraph/lib/errors"
|
||||
)
|
||||
|
||||
@ -69,7 +72,12 @@ func (r *schemaResolver) AddRepoMetadata(ctx context.Context, args struct {
|
||||
return &EmptyResponse{}, emptyNonNilValueError{value: *args.Value}
|
||||
}
|
||||
|
||||
return &EmptyResponse{}, r.db.RepoKVPs().Create(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value})
|
||||
err = r.db.RepoKVPs().Create(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value})
|
||||
if err == nil {
|
||||
r.logBackendEvent(ctx, "RepoMetadataAdded")
|
||||
}
|
||||
|
||||
return &EmptyResponse{}, err
|
||||
}
|
||||
|
||||
// Deprecated: Use UpdateRepoMetadata instead.
|
||||
@ -106,6 +114,9 @@ func (r *schemaResolver) UpdateRepoMetadata(ctx context.Context, args struct {
|
||||
}
|
||||
|
||||
_, err = r.db.RepoKVPs().Update(ctx, repoID, database.KeyValuePair{Key: args.Key, Value: args.Value})
|
||||
if err == nil {
|
||||
r.logBackendEvent(ctx, "RepoMetadataUpdated")
|
||||
}
|
||||
return &EmptyResponse{}, err
|
||||
}
|
||||
|
||||
@ -136,7 +147,29 @@ func (r *schemaResolver) DeleteRepoMetadata(ctx context.Context, args struct {
|
||||
return &EmptyResponse{}, err
|
||||
}
|
||||
|
||||
return &EmptyResponse{}, r.db.RepoKVPs().Delete(ctx, repoID, args.Key)
|
||||
err = r.db.RepoKVPs().Delete(ctx, repoID, args.Key)
|
||||
if err == nil {
|
||||
r.logBackendEvent(ctx, "RepoMetadataDeleted")
|
||||
}
|
||||
return &EmptyResponse{}, err
|
||||
}
|
||||
|
||||
func (r *schemaResolver) logBackendEvent(ctx context.Context, eventName string) {
|
||||
a := actor.FromContext(ctx)
|
||||
if a.IsAuthenticated() && !a.IsMockUser() {
|
||||
if err := usagestats.LogBackendEvent(
|
||||
r.db,
|
||||
a.UID,
|
||||
deviceid.FromContext(ctx),
|
||||
eventName,
|
||||
nil,
|
||||
nil,
|
||||
featureflag.GetEvaluatedFlagSet(ctx),
|
||||
nil,
|
||||
); err != nil {
|
||||
r.logger.Warn("Could not log " + eventName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type repoMetaResolver struct {
|
||||
|
||||
@ -363,6 +363,17 @@ func getAndMarshalCodyUsageJSON(ctx context.Context, db database.DB) (_ json.Raw
|
||||
return json.Marshal(codyUsage)
|
||||
}
|
||||
|
||||
func getAndMarshalRepoMetadataUsageJSON(ctx context.Context, db database.DB) (_ json.RawMessage, err error) {
|
||||
defer recordOperation("getAndMarshalRepoMetadataUsageJSON")(&err)
|
||||
|
||||
repoMetadataUsage, err := usagestats.GetAggregatedRepoMetadataStats(ctx, db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return json.Marshal(repoMetadataUsage)
|
||||
}
|
||||
|
||||
func getDependencyVersions(ctx context.Context, db database.DB, logger log.Logger) (json.RawMessage, error) {
|
||||
logFunc := logFuncFrom(logger.Scoped("getDependencyVersions", "gets the version of various dependency services"))
|
||||
var (
|
||||
@ -519,6 +530,7 @@ func updateBody(ctx context.Context, logger log.Logger, db database.DB) (io.Read
|
||||
IDEExtensionsUsage: []byte("{}"),
|
||||
MigratedExtensionsUsage: []byte("{}"),
|
||||
CodyUsage: []byte("{}"),
|
||||
RepoMetadataUsage: []byte("{}"),
|
||||
}
|
||||
|
||||
totalUsers, err := getTotalUsersCount(ctx, db)
|
||||
@ -668,6 +680,11 @@ func updateBody(ctx context.Context, logger log.Logger, db database.DB) (io.Read
|
||||
logFunc("codyUsage failed", log.Error(err))
|
||||
}
|
||||
|
||||
r.RepoMetadataUsage, err = getAndMarshalRepoMetadataUsageJSON(ctx, db)
|
||||
if err != nil {
|
||||
logFunc("repoMetadataUsage failed", log.Error(err))
|
||||
}
|
||||
|
||||
r.HasExtURL = conf.UsingExternalURL()
|
||||
r.BuiltinSignupAllowed = conf.IsBuiltinSignupAllowed()
|
||||
r.AccessRequestEnabled = conf.IsAccessRequestEnabled()
|
||||
|
||||
@ -247,6 +247,7 @@ type pingRequest struct {
|
||||
ActiveToday bool `json:"activeToday,omitempty"` // Only used in Sourcegraph App
|
||||
HasCodyEnabled bool `json:"hasCodyEnabled,omitempty"`
|
||||
CodyUsage json.RawMessage `json:"codyUsage,omitempty"`
|
||||
RepoMetadataUsage json.RawMessage `json:"repoMetadataUsage,omitempty"`
|
||||
}
|
||||
|
||||
type dependencyVersions struct {
|
||||
@ -373,6 +374,7 @@ type pingPayload struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
HasCodyEnabled string `json:"has_cody_enabled"`
|
||||
CodyUsage json.RawMessage `json:"cody_usage"`
|
||||
RepoMetadataUsage json.RawMessage `json:"repo_metadata_usage"`
|
||||
}
|
||||
|
||||
func logPing(logger log.Logger, r *http.Request, pr *pingRequest, hasUpdate bool) {
|
||||
@ -475,6 +477,7 @@ func marshalPing(pr *pingRequest, hasUpdate bool, clientAddr string, now time.Ti
|
||||
Timestamp: now.UTC().Format(time.RFC3339),
|
||||
HasCodyEnabled: strconv.FormatBool(codyFeatureFlag()),
|
||||
CodyUsage: codyUsage,
|
||||
RepoMetadataUsage: pr.RepoMetadataUsage,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -207,6 +207,7 @@ func TestSerializeBasic(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "mylicense",
|
||||
"has_update": "true",
|
||||
@ -282,6 +283,7 @@ func TestSerializeLimited(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "2023.03.23+205275.dd37e7",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "",
|
||||
"has_update": "true",
|
||||
@ -359,6 +361,7 @@ func TestSerializeFromQuery(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "",
|
||||
"has_update": "true",
|
||||
@ -419,6 +422,7 @@ func TestSerializeBatchChangesUsage(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "mylicense",
|
||||
"has_update": "true",
|
||||
@ -479,6 +483,7 @@ func TestSerializeGrowthStatistics(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "mylicense",
|
||||
"has_update": "true",
|
||||
@ -505,7 +510,7 @@ func TestSerializeGrowthStatistics(t *testing.T) {
|
||||
"search_onboarding": null,
|
||||
"homepage_panels": null,
|
||||
"repositories": null,
|
||||
"repository_size_histogram": null,
|
||||
"repository_size_histogram": null,
|
||||
"retention_statistics": null,
|
||||
"installer_email": "test@sourcegraph.com",
|
||||
"auth_providers": "foo,bar",
|
||||
@ -640,6 +645,7 @@ func TestSerializeCodeIntelUsage(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "mylicense",
|
||||
"has_update": "true",
|
||||
@ -822,6 +828,7 @@ func TestSerializeOldCodeIntelUsage(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "mylicense",
|
||||
"has_update": "true",
|
||||
@ -952,6 +959,7 @@ func TestSerializeCodeHostVersions(t *testing.T) {
|
||||
compareJSON(t, payload, `{
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "mylicense",
|
||||
"has_update": "true",
|
||||
@ -1048,6 +1056,7 @@ func TestSerializeOwn(t *testing.T) {
|
||||
"access_request_enabled": "false",
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "",
|
||||
"has_update": "true",
|
||||
@ -1095,7 +1104,109 @@ func TestSerializeOwn(t *testing.T) {
|
||||
"homepage_panels": null,
|
||||
"search_onboarding": null,
|
||||
"repositories": null,
|
||||
"repository_size_histogram": null,
|
||||
"repository_size_histogram": null,
|
||||
"retention_statistics": null,
|
||||
"installer_email": "test@sourcegraph.com",
|
||||
"auth_providers": "foo,bar",
|
||||
"ext_services": "GITHUB,GITLAB",
|
||||
"code_host_versions": null,
|
||||
"builtin_signup_allowed": "true",
|
||||
"deploy_type": "server",
|
||||
"total_user_accounts": "234",
|
||||
"has_external_url": "false",
|
||||
"has_repos": "true",
|
||||
"ever_searched": "false",
|
||||
"ever_find_refs": "true",
|
||||
"total_repos": "0",
|
||||
"active_today": "false",
|
||||
"os": "",
|
||||
"timestamp": "`+now.UTC().Format(time.RFC3339)+`"
|
||||
}`)
|
||||
}
|
||||
|
||||
func TestSerializeRepoMetadataUsage(t *testing.T) {
|
||||
pr := &pingRequest{
|
||||
ClientSiteID: "0101-0101",
|
||||
DeployType: "server",
|
||||
ClientVersionString: "3.12.6",
|
||||
AuthProviders: []string{"foo", "bar"},
|
||||
ExternalServices: []string{extsvc.KindGitHub, extsvc.KindGitLab},
|
||||
BuiltinSignupAllowed: true,
|
||||
HasExtURL: false,
|
||||
UniqueUsers: 123,
|
||||
InitialAdminEmail: "test@sourcegraph.com",
|
||||
TotalUsers: 234,
|
||||
HasRepos: true,
|
||||
EverSearched: false,
|
||||
EverFindRefs: true,
|
||||
RepoMetadataUsage: json.RawMessage(`{
|
||||
"summary": {
|
||||
"is_enabled": true,
|
||||
"repos_with_metadata_count": 10,
|
||||
"repo_metadata_count": 100
|
||||
},
|
||||
"daily": {
|
||||
"start_time": "2020-01-01T00:00:00Z",
|
||||
"create_repo_metadata": {
|
||||
"events_count": 10,
|
||||
"users_count": 5
|
||||
}
|
||||
}
|
||||
}`),
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
payload, err := marshalPing(pr, true, "127.0.0.1", now)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
compareJSON(t, payload, `{
|
||||
"access_request_enabled": "false",
|
||||
"remote_ip": "127.0.0.1",
|
||||
"remote_site_version": "3.12.6",
|
||||
"repo_metadata_usage": null,
|
||||
"remote_site_id": "0101-0101",
|
||||
"license_key": "",
|
||||
"has_update": "true",
|
||||
"unique_users_today": "123",
|
||||
"site_activity": null,
|
||||
"batch_changes_usage": null,
|
||||
"code_intel_usage": null,
|
||||
"new_code_intel_usage": null,
|
||||
"dependency_versions": null,
|
||||
"extensions_usage": null,
|
||||
"code_insights_usage": null,
|
||||
"code_insights_critical_telemetry": null,
|
||||
"code_monitoring_usage": null,
|
||||
"cody_usage": null,
|
||||
"notebooks_usage": null,
|
||||
"code_host_integration_usage": null,
|
||||
"ide_extensions_usage": null,
|
||||
"migrated_extensions_usage": null,
|
||||
"own_usage": null,
|
||||
"repo_metadata_usage": {
|
||||
"summary": {
|
||||
"is_enabled": true,
|
||||
"repos_with_metadata_count": 10,
|
||||
"repo_metadata_count": 100
|
||||
},
|
||||
"daily": {
|
||||
"start_time": "2020-01-01T00:00:00Z",
|
||||
"create_repo_metadata": {
|
||||
"events_count": 10,
|
||||
"users_count": 5
|
||||
}
|
||||
}
|
||||
},
|
||||
"search_usage": null,
|
||||
"growth_statistics": null,
|
||||
"has_cody_enabled": "false",
|
||||
"saved_searches": null,
|
||||
"homepage_panels": null,
|
||||
"search_onboarding": null,
|
||||
"repositories": null,
|
||||
"repository_size_histogram": null,
|
||||
"retention_statistics": null,
|
||||
"installer_email": "test@sourcegraph.com",
|
||||
"auth_providers": "foo,bar",
|
||||
|
||||
@ -191,11 +191,12 @@ Sourcegraph aggregates usage and performance metrics for some product features i
|
||||
- Sourcegraph Own usage data
|
||||
- Whether the `search-ownership` feature flag is turned on.
|
||||
- Number and ratio of repositories for which ownership data is available via CODEOWNERS file or the API.
|
||||
- Aggregate monthly weekly and daily active users for the following activities:
|
||||
<!-- - Aggregate monthly weekly and daily active users for the following activities: -->
|
||||
- Narrowing search results by owner using `file:has.owners` predicate.
|
||||
- Selecting owner search result through `select:file.owners`.
|
||||
- Displaying ownership panel in file view.
|
||||
- Histogram of cloned repository sizes
|
||||
- Aggregate daily, weekly, monthly repository metadata usage statistics
|
||||
</details>
|
||||
|
||||
## Sourcegraph app telemetry
|
||||
|
||||
@ -37,6 +37,9 @@ type EventLogStore interface {
|
||||
// AggregatedCodyEvents calculates CodyAggregatedEvent for each every unique event type related to Cody.
|
||||
AggregatedCodyEvents(ctx context.Context, now time.Time) ([]types.CodyAggregatedEvent, error)
|
||||
|
||||
// AggregatedRepoMetadataEvents calculates RepoMetadataAggregatedEvent for each every unique event type related to RepoMetadata.
|
||||
AggregatedRepoMetadataEvents(ctx context.Context, now time.Time, period PeriodType) (*types.RepoMetadataAggregatedEvents, error)
|
||||
|
||||
// AggregatedSearchEvents calculates SearchAggregatedEvent for each every unique event type related to search.
|
||||
AggregatedSearchEvents(ctx context.Context, now time.Time) ([]types.SearchAggregatedEvent, error)
|
||||
|
||||
@ -1418,6 +1421,97 @@ func (l *eventLogStore) aggregatedCodyEvents(ctx context.Context, queryString st
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func buildAggregatedRepoMetadataEventsQuery(period PeriodType) (string, error) {
|
||||
unit := ""
|
||||
switch period {
|
||||
case Daily:
|
||||
unit = "day"
|
||||
case Weekly:
|
||||
unit = "week"
|
||||
case Monthly:
|
||||
unit = "month"
|
||||
default:
|
||||
return "", ErrInvalidPeriodType
|
||||
}
|
||||
return `
|
||||
WITH events AS (
|
||||
SELECT
|
||||
name,
|
||||
` + aggregatedUserIDQueryFragment + ` AS user_id,
|
||||
argument
|
||||
FROM event_logs
|
||||
WHERE
|
||||
timestamp >= ` + makeDateTruncExpression(unit, "%s::timestamp") + `
|
||||
AND name IN ('RepoMetadataAdded', 'RepoMetadataUpdated', 'RepoMetadataDeleted', 'SearchSubmitted')
|
||||
)
|
||||
SELECT
|
||||
` + makeDateTruncExpression(unit, "%s::timestamp") + ` as start_time,
|
||||
|
||||
COUNT(*) FILTER (WHERE name IN ('RepoMetadataAdded')) AS added_count,
|
||||
COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataAdded')) AS added_unique_count,
|
||||
|
||||
COUNT(*) FILTER (WHERE name IN ('RepoMetadataUpdated')) AS updated_count,
|
||||
COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataUpdated')) AS updated_unique_count,
|
||||
|
||||
COUNT(*) FILTER (WHERE name IN ('RepoMetadataDeleted')) AS deleted_count,
|
||||
COUNT(DISTINCT user_id) FILTER (WHERE name IN ('RepoMetadataDeleted')) AS deleted_unique_count,
|
||||
|
||||
COUNT(*) FILTER (
|
||||
WHERE name IN ('SearchSubmitted')
|
||||
AND (
|
||||
argument->>'query' ILIKE '%%repo:has(%%'
|
||||
OR argument->>'query' ILIKE '%%repo:has.key(%%'
|
||||
OR argument->>'query' ILIKE '%%repo:has.tag(%%'
|
||||
OR argument->>'query' ILIKE '%%repo:has.meta(%%'
|
||||
)
|
||||
) AS searches_count,
|
||||
COUNT(DISTINCT user_id) FILTER (
|
||||
WHERE name IN ('SearchSubmitted')
|
||||
AND (
|
||||
argument->>'query' ILIKE '%%repo:has(%%'
|
||||
OR argument->>'query' ILIKE '%%repo:has.key(%%'
|
||||
OR argument->>'query' ILIKE '%%repo:has.tag(%%'
|
||||
OR argument->>'query' ILIKE '%%repo:has.meta(%%'
|
||||
)
|
||||
) AS searches_unique_count
|
||||
FROM events;
|
||||
`, nil
|
||||
}
|
||||
|
||||
func (l *eventLogStore) AggregatedRepoMetadataEvents(ctx context.Context, now time.Time, period PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
|
||||
query, err := buildAggregatedRepoMetadataEventsQuery(period)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row := l.QueryRow(ctx, sqlf.Sprintf(query, now, now))
|
||||
var startTime time.Time
|
||||
var createEvent types.EventStats
|
||||
var updateEvent types.EventStats
|
||||
var deleteEvent types.EventStats
|
||||
var searchEvent types.EventStats
|
||||
if err := row.Scan(
|
||||
&startTime,
|
||||
&createEvent.EventsCount,
|
||||
&createEvent.UsersCount,
|
||||
&updateEvent.EventsCount,
|
||||
&updateEvent.UsersCount,
|
||||
&deleteEvent.EventsCount,
|
||||
&deleteEvent.UsersCount,
|
||||
&searchEvent.EventsCount,
|
||||
&searchEvent.UsersCount,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &types.RepoMetadataAggregatedEvents{
|
||||
StartTime: startTime,
|
||||
CreateRepoMetadata: &createEvent,
|
||||
UpdateRepoMetadata: &updateEvent,
|
||||
DeleteRepoMetadata: &deleteEvent,
|
||||
SearchFilterUsage: &searchEvent,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *eventLogStore) AggregatedSearchEvents(ctx context.Context, now time.Time) ([]types.SearchAggregatedEvent, error) {
|
||||
latencyEvents, err := l.aggregatedSearchEvents(ctx, aggregatedSearchLatencyEventsQuery, now)
|
||||
if err != nil {
|
||||
|
||||
@ -1885,3 +1885,145 @@ func TestEventLogs_OwnershipFeatureActivity(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventLogs_AggregatedRepoMetadataStats(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip()
|
||||
}
|
||||
t.Parallel()
|
||||
ptr := func(i int32) *int32 { return &i }
|
||||
now := time.Date(2000, time.January, 20, 12, 0, 0, 0, time.UTC)
|
||||
events := []*Event{
|
||||
{
|
||||
UserID: 1,
|
||||
Name: "RepoMetadataAdded",
|
||||
Source: "BACKEND",
|
||||
Timestamp: now,
|
||||
},
|
||||
{
|
||||
UserID: 1,
|
||||
Name: "RepoMetadataAdded",
|
||||
Source: "BACKEND",
|
||||
Timestamp: now,
|
||||
},
|
||||
{
|
||||
UserID: 1,
|
||||
Name: "RepoMetadataAdded",
|
||||
Source: "BACKEND",
|
||||
Timestamp: time.Date(now.Year(), now.Month(), now.Day()-1, now.Hour(), 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
UserID: 1,
|
||||
Name: "RepoMetadataUpdated",
|
||||
Source: "BACKEND",
|
||||
Timestamp: now,
|
||||
},
|
||||
{
|
||||
UserID: 1,
|
||||
Name: "RepoMetadataDeleted",
|
||||
Source: "BACKEND",
|
||||
Timestamp: now,
|
||||
},
|
||||
{
|
||||
UserID: 1,
|
||||
Name: "SearchSubmitted",
|
||||
Argument: json.RawMessage(`{"query": "repo:has(some:meta)"}`),
|
||||
Source: "BACKEND",
|
||||
Timestamp: now,
|
||||
},
|
||||
}
|
||||
logger := logtest.Scoped(t)
|
||||
db := NewDB(logger, dbtest.NewDB(logger, t))
|
||||
ctx := context.Background()
|
||||
for _, e := range events {
|
||||
if err := db.EventLogs().Insert(ctx, e); err != nil {
|
||||
t.Fatalf("failed inserting test data: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for name, testCase := range map[string]struct {
|
||||
now time.Time
|
||||
period PeriodType
|
||||
stats *types.RepoMetadataAggregatedEvents
|
||||
}{
|
||||
"daily": {
|
||||
now: now,
|
||||
period: Daily,
|
||||
stats: &types.RepoMetadataAggregatedEvents{
|
||||
StartTime: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC),
|
||||
CreateRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(2),
|
||||
},
|
||||
UpdateRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
DeleteRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
SearchFilterUsage: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
"weekly": {
|
||||
now: now,
|
||||
period: Weekly,
|
||||
stats: &types.RepoMetadataAggregatedEvents{
|
||||
StartTime: time.Date(now.Year(), now.Month(), now.Day()-int(now.Weekday()), 0, 0, 0, 0, time.UTC),
|
||||
CreateRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(3),
|
||||
},
|
||||
UpdateRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
DeleteRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
SearchFilterUsage: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
"monthly": {
|
||||
now: now,
|
||||
period: Monthly,
|
||||
stats: &types.RepoMetadataAggregatedEvents{
|
||||
StartTime: time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC),
|
||||
CreateRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(3),
|
||||
},
|
||||
UpdateRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
DeleteRepoMetadata: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
SearchFilterUsage: &types.EventStats{
|
||||
UsersCount: ptr(1),
|
||||
EventsCount: ptr(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
stats, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, testCase.now, testCase.period)
|
||||
if err != nil {
|
||||
t.Fatalf("querying activity failed: %s", err)
|
||||
}
|
||||
if diff := cmp.Diff(testCase.stats, stats); diff != "" {
|
||||
t.Errorf("unexpected statistics returned:\n%s", diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -11835,6 +11835,10 @@ type MockEventLogStore struct {
|
||||
// AggregatedCodyEventsFunc is an instance of a mock function object
|
||||
// controlling the behavior of the method AggregatedCodyEvents.
|
||||
AggregatedCodyEventsFunc *EventLogStoreAggregatedCodyEventsFunc
|
||||
// AggregatedRepoMetadataEventsFunc is an instance of a mock function
|
||||
// object controlling the behavior of the method
|
||||
// AggregatedRepoMetadataEvents.
|
||||
AggregatedRepoMetadataEventsFunc *EventLogStoreAggregatedRepoMetadataEventsFunc
|
||||
// AggregatedSearchEventsFunc is an instance of a mock function object
|
||||
// controlling the behavior of the method AggregatedSearchEvents.
|
||||
AggregatedSearchEventsFunc *EventLogStoreAggregatedSearchEventsFunc
|
||||
@ -11976,6 +11980,11 @@ func NewMockEventLogStore() *MockEventLogStore {
|
||||
return
|
||||
},
|
||||
},
|
||||
AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{
|
||||
defaultHook: func(context.Context, time.Time, PeriodType) (r0 *types.RepoMetadataAggregatedEvents, r1 error) {
|
||||
return
|
||||
},
|
||||
},
|
||||
AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{
|
||||
defaultHook: func(context.Context, time.Time) (r0 []types.SearchAggregatedEvent, r1 error) {
|
||||
return
|
||||
@ -12173,6 +12182,11 @@ func NewStrictMockEventLogStore() *MockEventLogStore {
|
||||
panic("unexpected invocation of MockEventLogStore.AggregatedCodyEvents")
|
||||
},
|
||||
},
|
||||
AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{
|
||||
defaultHook: func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
|
||||
panic("unexpected invocation of MockEventLogStore.AggregatedRepoMetadataEvents")
|
||||
},
|
||||
},
|
||||
AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{
|
||||
defaultHook: func(context.Context, time.Time) ([]types.SearchAggregatedEvent, error) {
|
||||
panic("unexpected invocation of MockEventLogStore.AggregatedSearchEvents")
|
||||
@ -12365,6 +12379,9 @@ func NewMockEventLogStoreFrom(i EventLogStore) *MockEventLogStore {
|
||||
AggregatedCodyEventsFunc: &EventLogStoreAggregatedCodyEventsFunc{
|
||||
defaultHook: i.AggregatedCodyEvents,
|
||||
},
|
||||
AggregatedRepoMetadataEventsFunc: &EventLogStoreAggregatedRepoMetadataEventsFunc{
|
||||
defaultHook: i.AggregatedRepoMetadataEvents,
|
||||
},
|
||||
AggregatedSearchEventsFunc: &EventLogStoreAggregatedSearchEventsFunc{
|
||||
defaultHook: i.AggregatedSearchEvents,
|
||||
},
|
||||
@ -12804,6 +12821,121 @@ func (c EventLogStoreAggregatedCodyEventsFuncCall) Results() []interface{} {
|
||||
return []interface{}{c.Result0, c.Result1}
|
||||
}
|
||||
|
||||
// EventLogStoreAggregatedRepoMetadataEventsFunc describes the behavior when
|
||||
// the AggregatedRepoMetadataEvents method of the parent MockEventLogStore
|
||||
// instance is invoked.
|
||||
type EventLogStoreAggregatedRepoMetadataEventsFunc struct {
|
||||
defaultHook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)
|
||||
hooks []func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)
|
||||
history []EventLogStoreAggregatedRepoMetadataEventsFuncCall
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// AggregatedRepoMetadataEvents delegates to the next hook function in the
|
||||
// queue and stores the parameter and result values of this invocation.
|
||||
func (m *MockEventLogStore) AggregatedRepoMetadataEvents(v0 context.Context, v1 time.Time, v2 PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
|
||||
r0, r1 := m.AggregatedRepoMetadataEventsFunc.nextHook()(v0, v1, v2)
|
||||
m.AggregatedRepoMetadataEventsFunc.appendCall(EventLogStoreAggregatedRepoMetadataEventsFuncCall{v0, v1, v2, r0, r1})
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// SetDefaultHook sets function that is called when the
|
||||
// AggregatedRepoMetadataEvents method of the parent MockEventLogStore
|
||||
// instance is invoked and the hook queue is empty.
|
||||
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) SetDefaultHook(hook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)) {
|
||||
f.defaultHook = hook
|
||||
}
|
||||
|
||||
// PushHook adds a function to the end of hook queue. Each invocation of the
|
||||
// AggregatedRepoMetadataEvents method of the parent MockEventLogStore
|
||||
// instance invokes the hook at the front of the queue and discards it.
|
||||
// After the queue is empty, the default hook function is invoked for any
|
||||
// future action.
|
||||
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) PushHook(hook func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error)) {
|
||||
f.mutex.Lock()
|
||||
f.hooks = append(f.hooks, hook)
|
||||
f.mutex.Unlock()
|
||||
}
|
||||
|
||||
// SetDefaultReturn calls SetDefaultHook with a function that returns the
|
||||
// given values.
|
||||
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) SetDefaultReturn(r0 *types.RepoMetadataAggregatedEvents, r1 error) {
|
||||
f.SetDefaultHook(func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
|
||||
return r0, r1
|
||||
})
|
||||
}
|
||||
|
||||
// PushReturn calls PushHook with a function that returns the given values.
|
||||
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) PushReturn(r0 *types.RepoMetadataAggregatedEvents, r1 error) {
|
||||
f.PushHook(func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
|
||||
return r0, r1
|
||||
})
|
||||
}
|
||||
|
||||
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) nextHook() func(context.Context, time.Time, PeriodType) (*types.RepoMetadataAggregatedEvents, error) {
|
||||
f.mutex.Lock()
|
||||
defer f.mutex.Unlock()
|
||||
|
||||
if len(f.hooks) == 0 {
|
||||
return f.defaultHook
|
||||
}
|
||||
|
||||
hook := f.hooks[0]
|
||||
f.hooks = f.hooks[1:]
|
||||
return hook
|
||||
}
|
||||
|
||||
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) appendCall(r0 EventLogStoreAggregatedRepoMetadataEventsFuncCall) {
|
||||
f.mutex.Lock()
|
||||
f.history = append(f.history, r0)
|
||||
f.mutex.Unlock()
|
||||
}
|
||||
|
||||
// History returns a sequence of
|
||||
// EventLogStoreAggregatedRepoMetadataEventsFuncCall objects describing the
|
||||
// invocations of this function.
|
||||
func (f *EventLogStoreAggregatedRepoMetadataEventsFunc) History() []EventLogStoreAggregatedRepoMetadataEventsFuncCall {
|
||||
f.mutex.Lock()
|
||||
history := make([]EventLogStoreAggregatedRepoMetadataEventsFuncCall, len(f.history))
|
||||
copy(history, f.history)
|
||||
f.mutex.Unlock()
|
||||
|
||||
return history
|
||||
}
|
||||
|
||||
// EventLogStoreAggregatedRepoMetadataEventsFuncCall is an object that
|
||||
// describes an invocation of method AggregatedRepoMetadataEvents on an
|
||||
// instance of MockEventLogStore.
|
||||
type EventLogStoreAggregatedRepoMetadataEventsFuncCall struct {
|
||||
// Arg0 is the value of the 1st argument passed to this method
|
||||
// invocation.
|
||||
Arg0 context.Context
|
||||
// Arg1 is the value of the 2nd argument passed to this method
|
||||
// invocation.
|
||||
Arg1 time.Time
|
||||
// Arg2 is the value of the 3rd argument passed to this method
|
||||
// invocation.
|
||||
Arg2 PeriodType
|
||||
// Result0 is the value of the 1st result returned from this method
|
||||
// invocation.
|
||||
Result0 *types.RepoMetadataAggregatedEvents
|
||||
// Result1 is the value of the 2nd result returned from this method
|
||||
// invocation.
|
||||
Result1 error
|
||||
}
|
||||
|
||||
// Args returns an interface slice containing the arguments of this
|
||||
// invocation.
|
||||
func (c EventLogStoreAggregatedRepoMetadataEventsFuncCall) Args() []interface{} {
|
||||
return []interface{}{c.Arg0, c.Arg1, c.Arg2}
|
||||
}
|
||||
|
||||
// Results returns an interface slice containing the results of this
|
||||
// invocation.
|
||||
func (c EventLogStoreAggregatedRepoMetadataEventsFuncCall) Results() []interface{} {
|
||||
return []interface{}{c.Result0, c.Result1}
|
||||
}
|
||||
|
||||
// EventLogStoreAggregatedSearchEventsFunc describes the behavior when the
|
||||
// AggregatedSearchEvents method of the parent MockEventLogStore instance is
|
||||
// invoked.
|
||||
|
||||
@ -1021,6 +1021,37 @@ type CodyAggregatedEvent struct {
|
||||
InvalidDay int32
|
||||
}
|
||||
|
||||
// NOTE: DO NOT alter this struct without making a symmetric change
|
||||
// to the updatecheck handler.
|
||||
// RepoMetadataAggregatedStats represents the total number of repo metadata,
|
||||
// number of repositories with any metadata, total and unique number of
|
||||
// events for repo metadata usage related events over the current day, week, month.
|
||||
type RepoMetadataAggregatedStats struct {
|
||||
Summary *RepoMetadataAggregatedSummary
|
||||
Daily *RepoMetadataAggregatedEvents
|
||||
Weekly *RepoMetadataAggregatedEvents
|
||||
Monthly *RepoMetadataAggregatedEvents
|
||||
}
|
||||
|
||||
type RepoMetadataAggregatedSummary struct {
|
||||
IsEnabled bool
|
||||
RepoMetadataCount *int32
|
||||
ReposWithMetadataCount *int32
|
||||
}
|
||||
|
||||
type RepoMetadataAggregatedEvents struct {
|
||||
StartTime time.Time
|
||||
CreateRepoMetadata *EventStats
|
||||
UpdateRepoMetadata *EventStats
|
||||
DeleteRepoMetadata *EventStats
|
||||
SearchFilterUsage *EventStats
|
||||
}
|
||||
|
||||
type EventStats struct {
|
||||
UsersCount *int32
|
||||
EventsCount *int32
|
||||
}
|
||||
|
||||
// NOTE: DO NOT alter this struct without making a symmetric change
|
||||
// to the updatecheck handler. This struct is marshalled and sent to
|
||||
// BigQuery, which requires the input match its schema exactly.
|
||||
|
||||
1
internal/usagestats/BUILD.bazel
generated
1
internal/usagestats/BUILD.bazel
generated
@ -5,6 +5,7 @@ go_library(
|
||||
srcs = [
|
||||
"aggregated.go",
|
||||
"aggregated_codeintel.go",
|
||||
"aggregated_repo_metadata.go",
|
||||
"aggregated_search.go",
|
||||
"all_time_stats.go",
|
||||
"batches.go",
|
||||
|
||||
60
internal/usagestats/aggregated_repo_metadata.go
Normal file
60
internal/usagestats/aggregated_repo_metadata.go
Normal file
@ -0,0 +1,60 @@
|
||||
package usagestats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/sourcegraph/sourcegraph/internal/database"
|
||||
"github.com/sourcegraph/sourcegraph/internal/types"
|
||||
)
|
||||
|
||||
func GetAggregatedRepoMetadataStats(ctx context.Context, db database.DB) (*types.RepoMetadataAggregatedStats, error) {
|
||||
now := time.Now().UTC()
|
||||
daily, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Daily)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
weekly, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Weekly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
monthly, err := db.EventLogs().AggregatedRepoMetadataEvents(ctx, now, database.Monthly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
summary, err := getAggregatedRepoMetadataSummary(ctx, db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &types.RepoMetadataAggregatedStats{
|
||||
Summary: summary,
|
||||
Daily: daily,
|
||||
Weekly: weekly,
|
||||
Monthly: monthly,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getAggregatedRepoMetadataSummary(ctx context.Context, db database.DB) (*types.RepoMetadataAggregatedSummary, error) {
|
||||
q := `
|
||||
SELECT
|
||||
COUNT(*) AS total_count,
|
||||
COUNT(DISTINCT repo_id) AS total_repos_count
|
||||
FROM repo_kvps
|
||||
`
|
||||
var summary types.RepoMetadataAggregatedSummary
|
||||
err := db.QueryRowContext(ctx, q).Scan(&summary.RepoMetadataCount, &summary.ReposWithMetadataCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
flag, err := db.FeatureFlags().GetFeatureFlag(ctx, "repository-metadata")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
summary.IsEnabled = flag != nil && flag.Bool.Value
|
||||
|
||||
return &summary, nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user