diff --git a/enterprise/internal/insights/store/insight_store.go b/enterprise/internal/insights/store/insight_store.go index 43ad9a16f8a..b909c253c22 100644 --- a/enterprise/internal/insights/store/insight_store.go +++ b/enterprise/internal/insights/store/insight_store.go @@ -400,6 +400,7 @@ func scanDataSeries(rows *sql.Rows, queryErr error) (_ []types.InsightSeries, er &temp.GroupBy, &temp.BackfillAttempts, &temp.SupportsAugmentation, + &temp.RepositoryCriteria, ); err != nil { return []types.InsightSeries{}, err } @@ -472,6 +473,7 @@ func scanInsightViewSeries(rows *sql.Rows, queryErr error) (_ []types.InsightVie &temp.GroupBy, &temp.BackfillAttempts, &temp.SupportsAugmentation, + &temp.RepositoryCriteria, ); err != nil { return []types.InsightViewSeries{}, err } @@ -716,6 +718,7 @@ func (s *InsightStore) CreateSeries(ctx context.Context, series types.InsightSer series.JustInTime, series.GenerationMethod, series.GroupBy, + series.RepositoryCriteria, )) var id int err := row.Scan(&id) @@ -951,8 +954,8 @@ const createInsightSeriesSql = ` INSERT INTO insight_series (series_id, query, created_at, oldest_historical_at, last_recorded_at, next_recording_after, last_snapshot_at, next_snapshot_after, repositories, sample_interval_unit, sample_interval_value, generated_from_capture_groups, - just_in_time, generation_method, group_by, needs_migration) -VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, false) + just_in_time, generation_method, group_by, needs_migration, repository_criteria) +VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, false, %s) RETURNING id;` const getInsightByViewSql = ` @@ -961,8 +964,8 @@ i.id, i.series_id, i.query, i.created_at, i.oldest_historical_at, i.last_recorde i.next_recording_after, i.backfill_queued_at, i.last_snapshot_at, i.next_snapshot_after, i.repositories, i.sample_interval_unit, i.sample_interval_value, iv.default_filter_include_repo_regex, iv.default_filter_exclude_repo_regex, iv.other_threshold, iv.presentation_type, i.generated_from_capture_groups, i.just_in_time, i.generation_method, iv.is_frozen, -default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts, -i.supports_augmentation +default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts, +i.supports_augmentation, i.repository_criteria FROM (%s) iv JOIN insight_view_series ivs ON iv.id = ivs.insight_view_id JOIN insight_series i ON ivs.insight_series_id = i.id @@ -977,7 +980,7 @@ i.next_recording_after, i.backfill_queued_at, i.last_snapshot_at, i.next_snapsho i.sample_interval_unit, i.sample_interval_value, iv.default_filter_include_repo_regex, iv.default_filter_exclude_repo_regex, iv.other_threshold, iv.presentation_type, i.generated_from_capture_groups, i.just_in_time, i.generation_method, iv.is_frozen, default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts, -i.supports_augmentation +i.supports_augmentation, i.repository_criteria FROM dashboard_insight_view as dbiv JOIN insight_view iv ON iv.id = dbiv.insight_view_id JOIN insight_view_series ivs ON iv.id = ivs.insight_view_id @@ -991,7 +994,7 @@ const getInsightDataSeriesSql = ` SELECT id, series_id, query, created_at, oldest_historical_at, last_recorded_at, next_recording_after, last_snapshot_at, next_snapshot_after, (CASE WHEN deleted_at IS NULL THEN TRUE ELSE FALSE END) AS enabled, sample_interval_unit, sample_interval_value, generated_from_capture_groups, -just_in_time, generation_method, repositories, group_by, backfill_attempts, supports_augmentation +just_in_time, generation_method, repositories, group_by, backfill_attempts, supports_augmentation, repository_criteria FROM insight_series WHERE %s ` @@ -1017,7 +1020,9 @@ SELECT iv.id, 0 as dashboard_insight_id, iv.unique_id, iv.title, iv.description, i.next_recording_after, i.backfill_queued_at, i.last_snapshot_at, i.next_snapshot_after, i.repositories, i.sample_interval_unit, i.sample_interval_value, iv.default_filter_include_repo_regex, iv.default_filter_exclude_repo_regex, iv.other_threshold, iv.presentation_type, i.generated_from_capture_groups, i.just_in_time, i.generation_method, iv.is_frozen, -default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts, i.supports_augmentation + default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts, + i.supports_augmentation, i.repository_criteria + FROM insight_view iv JOIN insight_view_series ivs ON iv.id = ivs.insight_view_id JOIN insight_series i ON ivs.insight_series_id = i.id diff --git a/enterprise/internal/insights/store/insight_store_test.go b/enterprise/internal/insights/store/insight_store_test.go index cbe1a0a53e0..7854d22ef52 100644 --- a/enterprise/internal/insights/store/insight_store_test.go +++ b/enterprise/internal/insights/store/insight_store_test.go @@ -42,10 +42,10 @@ func TestGet(t *testing.T) { } _, err = insightsDB.ExecContext(context.Background(), `INSERT INTO insight_series (series_id, query, created_at, oldest_historical_at, last_recorded_at, - next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by) - VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null), - ('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo'), - ('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null);`, now) + next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by, repository_criteria) + VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null,'repo:a'), + ('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo', null), + ('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null, 'repo:*');`, now) if err != nil { t.Fatal(err) } @@ -69,6 +69,7 @@ func TestGet(t *testing.T) { t.Fatal(err) } sampleIntervalUnit := "MONTH" + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 1, @@ -92,6 +93,7 @@ func TestGet(t *testing.T) { GenerationMethod: types.Search, IsFrozen: false, SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 1, @@ -156,6 +158,7 @@ func TestGet(t *testing.T) { t.Fatal(err) } sampleIntervalUnit := "MONTH" + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 1, @@ -179,6 +182,7 @@ func TestGet(t *testing.T) { GenerationMethod: types.Search, IsFrozen: false, SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 1, @@ -218,6 +222,7 @@ func TestGet(t *testing.T) { t.Fatal(err) } sampleIntervalUnit := "MONTH" + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 1, @@ -241,6 +246,7 @@ func TestGet(t *testing.T) { GenerationMethod: types.Search, IsFrozen: false, SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 1, @@ -304,9 +310,9 @@ func TestGetAll(t *testing.T) { t.Fatal(err) } _, err = insightsDB.ExecContext(context.Background(), `INSERT INTO insight_series (id, series_id, query, created_at, oldest_historical_at, last_recorded_at, - next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by) - VALUES (1, 'series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null), - (2, 'series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo')`, now) + next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by, repository_criteria) + VALUES (1, 'series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null, 'repo:a'), + (2, 'series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo', null)`, now) if err != nil { t.Fatal(err) } @@ -348,7 +354,7 @@ func TestGetAll(t *testing.T) { if err != nil { t.Fatal(err) } - + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 5, @@ -371,6 +377,7 @@ func TestGetAll(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 5, @@ -416,6 +423,7 @@ func TestGetAll(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 2, @@ -461,6 +469,7 @@ func TestGetAll(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, } if diff := cmp.Diff(want, got); diff != "" { @@ -473,7 +482,7 @@ func TestGetAll(t *testing.T) { if err != nil { t.Fatal(err) } - + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 5, @@ -496,6 +505,7 @@ func TestGetAll(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 5, @@ -531,7 +541,7 @@ func TestGetAll(t *testing.T) { if err != nil { t.Fatal(err) } - + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 2, @@ -554,6 +564,7 @@ func TestGetAll(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 2, @@ -589,7 +600,7 @@ func TestGetAll(t *testing.T) { if err != nil { t.Fatal(err) } - + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 2, @@ -612,6 +623,7 @@ func TestGetAll(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 2, @@ -657,6 +669,7 @@ func TestGetAll(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, } if diff := cmp.Diff(want, got); diff != "" { @@ -681,10 +694,10 @@ func TestGetAllOnDashboard(t *testing.T) { } _, err = insightsDB.ExecContext(context.Background(), `INSERT INTO insight_series (series_id, query, created_at, oldest_historical_at, last_recorded_at, - next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by) - VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null), - ('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo'), - ('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null);`, now) + next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by, repository_criteria) + VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null, 'repo:a'), + ('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo', null), + ('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null, null);`, now) if err != nil { t.Fatal(err) } @@ -720,7 +733,7 @@ func TestGetAllOnDashboard(t *testing.T) { if err != nil { t.Fatal(err) } - + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 2, @@ -768,6 +781,7 @@ func TestGetAllOnDashboard(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, { ViewID: 4, @@ -815,6 +829,7 @@ func TestGetAllOnDashboard(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, } if diff := cmp.Diff(want, got); diff != "" { @@ -827,7 +842,7 @@ func TestGetAllOnDashboard(t *testing.T) { if err != nil { t.Fatal(err) } - + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 2, @@ -875,6 +890,7 @@ func TestGetAllOnDashboard(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, } if diff := cmp.Diff(want, got); diff != "" { @@ -887,7 +903,7 @@ func TestGetAllOnDashboard(t *testing.T) { if err != nil { t.Fatal(err) } - + series1RepoCriteria := "repo:a" want := []types.InsightViewSeries{ { ViewID: 4, @@ -935,6 +951,7 @@ func TestGetAllOnDashboard(t *testing.T) { PresentationType: types.PresentationType("LINE"), GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: &series1RepoCriteria, }, } if diff := cmp.Diff(want, got); diff != "" { @@ -957,6 +974,7 @@ func TestCreateSeries(t *testing.T) { ctx := context.Background() t.Run("test create series", func(t *testing.T) { + repoCriteria := "repo:a" series := types.InsightSeries{ SeriesID: "unique-1", Query: "query-1", @@ -969,6 +987,7 @@ func TestCreateSeries(t *testing.T) { SampleIntervalUnit: string(types.Month), GenerationMethod: types.Search, GroupBy: &groupByRepo, + RepositoryCriteria: &repoCriteria, } got, err := store.CreateSeries(ctx, series) @@ -991,6 +1010,7 @@ func TestCreateSeries(t *testing.T) { GenerationMethod: types.Search, GroupBy: &groupByRepo, SupportsAugmentation: true, + RepositoryCriteria: &repoCriteria, } if diff := cmp.Diff(want, got); diff != "" { @@ -1000,6 +1020,7 @@ func TestCreateSeries(t *testing.T) { t.Run("test create and get capture groups series", func(t *testing.T) { store := NewInsightStore(insightsDB) sampleIntervalUnit := "MONTH" + repoCriteria := "repo:a" _, err := store.CreateSeries(ctx, types.InsightSeries{ SeriesID: "capture-group-1", Query: "well hello there", @@ -1014,6 +1035,7 @@ func TestCreateSeries(t *testing.T) { CreatedAt: now, GeneratedFromCaptureGroups: true, GenerationMethod: types.Search, + RepositoryCriteria: &repoCriteria, }) if err != nil { return diff --git a/enterprise/internal/insights/store/testdata/TestCreateSeries/test_create_and_get_capture_groups_series.golden b/enterprise/internal/insights/store/testdata/TestCreateSeries/test_create_and_get_capture_groups_series.golden index b4d89a12770..ee26c7f4d79 100644 --- a/enterprise/internal/insights/store/testdata/TestCreateSeries/test_create_and_get_capture_groups_series.golden +++ b/enterprise/internal/insights/store/testdata/TestCreateSeries/test_create_and_get_capture_groups_series.golden @@ -13,4 +13,5 @@ GeneratedFromCaptureGroups: true, GenerationMethod: types.GenerationMethod("search"), SupportsAugmentation: true, + RepositoryCriteria: valast.Addr("repo:a").(*string), }} diff --git a/enterprise/internal/insights/types/types.go b/enterprise/internal/insights/types/types.go index 1f2e56ece66..ff3cfb3e483 100644 --- a/enterprise/internal/insights/types/types.go +++ b/enterprise/internal/insights/types/types.go @@ -41,6 +41,7 @@ type InsightViewSeries struct { GroupBy *string BackfillAttempts int32 SupportsAugmentation bool + RepositoryCriteria *string } type Insight struct { @@ -107,6 +108,7 @@ type InsightSeries struct { GroupBy *string BackfillAttempts int32 SupportsAugmentation bool + RepositoryCriteria *string } type IntervalUnit string diff --git a/internal/database/schema.codeinsights.json b/internal/database/schema.codeinsights.json index ad41b9f4e1a..b78fe5da450 100755 --- a/internal/database/schema.codeinsights.json +++ b/internal/database/schema.codeinsights.json @@ -1045,6 +1045,19 @@ "GenerationExpression": "", "Comment": "" }, + { + "Name": "repository_criteria", + "Index": 23, + "TypeName": "text", + "IsNullable": true, + "Default": "", + "CharacterMaximumLength": 0, + "IsIdentity": false, + "IdentityGeneration": "", + "IsGenerated": "NEVER", + "GenerationExpression": "", + "Comment": "The search criteria used to determine the repositories that are included in this series." + }, { "Name": "sample_interval_unit", "Index": 13, diff --git a/internal/database/schema.codeinsights.md b/internal/database/schema.codeinsights.md index d9ac4b179b4..694cbbef80f 100755 --- a/internal/database/schema.codeinsights.md +++ b/internal/database/schema.codeinsights.md @@ -159,6 +159,7 @@ Stores queries that were unsuccessful or otherwise flagged as incomplete or inco needs_migration | boolean | | | backfill_completed_at | timestamp without time zone | | | supports_augmentation | boolean | | not null | true + repository_criteria | text | | | Indexes: "insight_series_pkey" PRIMARY KEY, btree (id) "insight_series_series_id_unique_idx" UNIQUE, btree (series_id) @@ -193,6 +194,8 @@ Data series that comprise code insights. **query**: Query string that generates this series +**repository_criteria**: The search criteria used to determine the repositories that are included in this series. + **series_id**: Timestamp that this series completed a full repository iteration for backfill. This flag has limited semantic value, and only means it tried to queue up queries for each repository. It does not guarantee success on those queries. # Table "public.insight_series_backfill" diff --git a/migrations/codeinsights/1670253074_insight_series_repo_criteria/down.sql b/migrations/codeinsights/1670253074_insight_series_repo_criteria/down.sql new file mode 100644 index 00000000000..161dbbbdd69 --- /dev/null +++ b/migrations/codeinsights/1670253074_insight_series_repo_criteria/down.sql @@ -0,0 +1 @@ +ALTER TABLE IF EXISTS insight_series DROP COLUMN IF EXISTS repository_criteria; diff --git a/migrations/codeinsights/1670253074_insight_series_repo_criteria/metadata.yaml b/migrations/codeinsights/1670253074_insight_series_repo_criteria/metadata.yaml new file mode 100644 index 00000000000..eea25507b06 --- /dev/null +++ b/migrations/codeinsights/1670253074_insight_series_repo_criteria/metadata.yaml @@ -0,0 +1,2 @@ +name: insight series repo criteria +parents: [1666729025, 1667309737] diff --git a/migrations/codeinsights/1670253074_insight_series_repo_criteria/up.sql b/migrations/codeinsights/1670253074_insight_series_repo_criteria/up.sql new file mode 100644 index 00000000000..608636df76c --- /dev/null +++ b/migrations/codeinsights/1670253074_insight_series_repo_criteria/up.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS insight_series + ADD COLUMN IF NOT EXISTS repository_criteria text; + +COMMENT ON COLUMN insight_series.repository_criteria IS 'The search criteria used to determine the repositories that are included in this series.'; + diff --git a/migrations/codeinsights/squashed.sql b/migrations/codeinsights/squashed.sql index 7fe8ce4d294..1d5337bde06 100755 --- a/migrations/codeinsights/squashed.sql +++ b/migrations/codeinsights/squashed.sql @@ -172,7 +172,8 @@ CREATE TABLE insight_series ( backfill_attempts integer DEFAULT 0 NOT NULL, needs_migration boolean, backfill_completed_at timestamp without time zone, - supports_augmentation boolean DEFAULT true NOT NULL + supports_augmentation boolean DEFAULT true NOT NULL, + repository_criteria text ); COMMENT ON TABLE insight_series IS 'Data series that comprise code insights.'; @@ -197,6 +198,8 @@ COMMENT ON COLUMN insight_series.generation_method IS 'Specifies the execution m COMMENT ON COLUMN insight_series.just_in_time IS 'Specifies if the series should be resolved just in time at query time, or recorded in background processing.'; +COMMENT ON COLUMN insight_series.repository_criteria IS 'The search criteria used to determine the repositories that are included in this series.'; + CREATE TABLE insight_series_backfill ( id integer NOT NULL, series_id integer NOT NULL,