artin/sourcegraph
1
0
Fork 0
mirror of https://github.com/sourcegraph/sourcegraph.git synced 2026-02-06 18:31:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

insights: db and store changes for strat scoped insights (#45624)

Browse Source
This commit is contained in:
chwarwick 2022-12-13 12:45:27 -05:00 committed by GitHub
parent 64210de962
commit 294ccc44fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 83 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
enterprise/internal/insights/store/insight_store.go
View File

@ -400,6 +400,7 @@ func scanDataSeries(rows *sql.Rows, queryErr error) (_ []types.InsightSeries, er
&temp.GroupBy,
&temp.BackfillAttempts,
&temp.SupportsAugmentation,
&temp.RepositoryCriteria,
); err != nil {
return []types.InsightSeries{}, err
}
@ -472,6 +473,7 @@ func scanInsightViewSeries(rows *sql.Rows, queryErr error) (_ []types.InsightVie
&temp.GroupBy,
&temp.BackfillAttempts,
&temp.SupportsAugmentation,
&temp.RepositoryCriteria,
); err != nil {
return []types.InsightViewSeries{}, err
}
@ -716,6 +718,7 @@ func (s *InsightStore) CreateSeries(ctx context.Context, series types.InsightSer
series.JustInTime,
series.GenerationMethod,
series.GroupBy,
series.RepositoryCriteria,
))
var id int
err := row.Scan(&id)
@ -951,8 +954,8 @@ const createInsightSeriesSql = `
INSERT INTO insight_series (series_id, query, created_at, oldest_historical_at, last_recorded_at,
next_recording_after, last_snapshot_at, next_snapshot_after, repositories,
sample_interval_unit, sample_interval_value, generated_from_capture_groups,
just_in_time, generation_method, group_by, needs_migration)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, false)
just_in_time, generation_method, group_by, needs_migration, repository_criteria)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, false, %s)
RETURNING id;`
const getInsightByViewSql = `
@ -961,8 +964,8 @@ i.id, i.series_id, i.query, i.created_at, i.oldest_historical_at, i.last_recorde
i.next_recording_after, i.backfill_queued_at, i.last_snapshot_at, i.next_snapshot_after, i.repositories,
i.sample_interval_unit, i.sample_interval_value, iv.default_filter_include_repo_regex, iv.default_filter_exclude_repo_regex,
iv.other_threshold, iv.presentation_type, i.generated_from_capture_groups, i.just_in_time, i.generation_method, iv.is_frozen,
default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts,
i.supports_augmentation
default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts,
i.supports_augmentation, i.repository_criteria
FROM (%s) iv
JOIN insight_view_series ivs ON iv.id = ivs.insight_view_id
JOIN insight_series i ON ivs.insight_series_id = i.id
@ -977,7 +980,7 @@ i.next_recording_after, i.backfill_queued_at, i.last_snapshot_at, i.next_snapsho
i.sample_interval_unit, i.sample_interval_value, iv.default_filter_include_repo_regex, iv.default_filter_exclude_repo_regex,
iv.other_threshold, iv.presentation_type, i.generated_from_capture_groups, i.just_in_time, i.generation_method, iv.is_frozen,
default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts,
i.supports_augmentation
i.supports_augmentation, i.repository_criteria
FROM dashboard_insight_view as dbiv
JOIN insight_view iv ON iv.id = dbiv.insight_view_id
JOIN insight_view_series ivs ON iv.id = ivs.insight_view_id
@ -991,7 +994,7 @@ const getInsightDataSeriesSql = `
SELECT id, series_id, query, created_at, oldest_historical_at, last_recorded_at, next_recording_after,
last_snapshot_at, next_snapshot_after, (CASE WHEN deleted_at IS NULL THEN TRUE ELSE FALSE END) AS enabled,
sample_interval_unit, sample_interval_value, generated_from_capture_groups,
just_in_time, generation_method, repositories, group_by, backfill_attempts, supports_augmentation
just_in_time, generation_method, repositories, group_by, backfill_attempts, supports_augmentation, repository_criteria
FROM insight_series
WHERE %s
`
@ -1017,7 +1020,9 @@ SELECT iv.id, 0 as dashboard_insight_id, iv.unique_id, iv.title, iv.description,
i.next_recording_after, i.backfill_queued_at, i.last_snapshot_at, i.next_snapshot_after, i.repositories,
i.sample_interval_unit, i.sample_interval_value, iv.default_filter_include_repo_regex, iv.default_filter_exclude_repo_regex,
iv.other_threshold, iv.presentation_type, i.generated_from_capture_groups, i.just_in_time, i.generation_method, iv.is_frozen,
default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts, i.supports_augmentation
default_filter_search_contexts, iv.series_sort_mode, iv.series_sort_direction, iv.series_limit, i.group_by, i.backfill_attempts,
i.supports_augmentation, i.repository_criteria
FROM insight_view iv
JOIN insight_view_series ivs ON iv.id = ivs.insight_view_id
JOIN insight_series i ON ivs.insight_series_id = i.id

58
enterprise/internal/insights/store/insight_store_test.go
View File

@ -42,10 +42,10 @@ func TestGet(t *testing.T) {
}
_, err = insightsDB.ExecContext(context.Background(), `INSERT INTO insight_series (series_id, query, created_at, oldest_historical_at, last_recorded_at,
next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by)
VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null),
('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo'),
('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null);`, now)
next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by, repository_criteria)
VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null,'repo:a'),
('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo', null),
('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null, 'repo:*');`, now)
if err != nil {
t.Fatal(err)
}
@ -69,6 +69,7 @@ func TestGet(t *testing.T) {
t.Fatal(err)
}
sampleIntervalUnit := "MONTH"
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 1,
@ -92,6 +93,7 @@ func TestGet(t *testing.T) {
GenerationMethod: types.Search,
IsFrozen: false,
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 1,
@ -156,6 +158,7 @@ func TestGet(t *testing.T) {
t.Fatal(err)
}
sampleIntervalUnit := "MONTH"
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 1,
@ -179,6 +182,7 @@ func TestGet(t *testing.T) {
GenerationMethod: types.Search,
IsFrozen: false,
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 1,
@ -218,6 +222,7 @@ func TestGet(t *testing.T) {
t.Fatal(err)
}
sampleIntervalUnit := "MONTH"
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 1,
@ -241,6 +246,7 @@ func TestGet(t *testing.T) {
GenerationMethod: types.Search,
IsFrozen: false,
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 1,
@ -304,9 +310,9 @@ func TestGetAll(t *testing.T) {
t.Fatal(err)
}
_, err = insightsDB.ExecContext(context.Background(), `INSERT INTO insight_series (id, series_id, query, created_at, oldest_historical_at, last_recorded_at,
next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by)
VALUES (1, 'series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null),
(2, 'series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo')`, now)
next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by, repository_criteria)
VALUES (1, 'series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null, 'repo:a'),
(2, 'series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo', null)`, now)
if err != nil {
t.Fatal(err)
}
@ -348,7 +354,7 @@ func TestGetAll(t *testing.T) {
if err != nil {
t.Fatal(err)
}
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 5,
@ -371,6 +377,7 @@ func TestGetAll(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 5,
@ -416,6 +423,7 @@ func TestGetAll(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 2,
@ -461,6 +469,7 @@ func TestGetAll(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
}
if diff := cmp.Diff(want, got); diff != "" {
@ -473,7 +482,7 @@ func TestGetAll(t *testing.T) {
if err != nil {
t.Fatal(err)
}
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 5,
@ -496,6 +505,7 @@ func TestGetAll(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 5,
@ -531,7 +541,7 @@ func TestGetAll(t *testing.T) {
if err != nil {
t.Fatal(err)
}
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 2,
@ -554,6 +564,7 @@ func TestGetAll(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 2,
@ -589,7 +600,7 @@ func TestGetAll(t *testing.T) {
if err != nil {
t.Fatal(err)
}
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 2,
@ -612,6 +623,7 @@ func TestGetAll(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 2,
@ -657,6 +669,7 @@ func TestGetAll(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
}
if diff := cmp.Diff(want, got); diff != "" {
@ -681,10 +694,10 @@ func TestGetAllOnDashboard(t *testing.T) {
}
_, err = insightsDB.ExecContext(context.Background(), `INSERT INTO insight_series (series_id, query, created_at, oldest_historical_at, last_recorded_at,
next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by)
VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null),
('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo'),
('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null);`, now)
next_recording_after, last_snapshot_at, next_snapshot_after, deleted_at, generation_method, group_by, repository_criteria)
VALUES ('series-id-1', 'query-1', $1, $1, $1, $1, $1, $1, null, 'search', null, 'repo:a'),
('series-id-2', 'query-2', $1, $1, $1, $1, $1, $1, null, 'search', 'repo', null),
('series-id-3-deleted', 'query-3', $1, $1, $1, $1, $1, $1, $1, 'search', null, null);`, now)
if err != nil {
t.Fatal(err)
}
@ -720,7 +733,7 @@ func TestGetAllOnDashboard(t *testing.T) {
if err != nil {
t.Fatal(err)
}
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 2,
@ -768,6 +781,7 @@ func TestGetAllOnDashboard(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
{
ViewID: 4,
@ -815,6 +829,7 @@ func TestGetAllOnDashboard(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
}
if diff := cmp.Diff(want, got); diff != "" {
@ -827,7 +842,7 @@ func TestGetAllOnDashboard(t *testing.T) {
if err != nil {
t.Fatal(err)
}
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 2,
@ -875,6 +890,7 @@ func TestGetAllOnDashboard(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
}
if diff := cmp.Diff(want, got); diff != "" {
@ -887,7 +903,7 @@ func TestGetAllOnDashboard(t *testing.T) {
if err != nil {
t.Fatal(err)
}
series1RepoCriteria := "repo:a"
want := []types.InsightViewSeries{
{
ViewID: 4,
@ -935,6 +951,7 @@ func TestGetAllOnDashboard(t *testing.T) {
PresentationType: types.PresentationType("LINE"),
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: &series1RepoCriteria,
},
}
if diff := cmp.Diff(want, got); diff != "" {
@ -957,6 +974,7 @@ func TestCreateSeries(t *testing.T) {
ctx := context.Background()
t.Run("test create series", func(t *testing.T) {
repoCriteria := "repo:a"
series := types.InsightSeries{
SeriesID: "unique-1",
Query: "query-1",
@ -969,6 +987,7 @@ func TestCreateSeries(t *testing.T) {
SampleIntervalUnit: string(types.Month),
GenerationMethod: types.Search,
GroupBy: &groupByRepo,
RepositoryCriteria: &repoCriteria,
}
got, err := store.CreateSeries(ctx, series)
@ -991,6 +1010,7 @@ func TestCreateSeries(t *testing.T) {
GenerationMethod: types.Search,
GroupBy: &groupByRepo,
SupportsAugmentation: true,
RepositoryCriteria: &repoCriteria,
}
if diff := cmp.Diff(want, got); diff != "" {
@ -1000,6 +1020,7 @@ func TestCreateSeries(t *testing.T) {
t.Run("test create and get capture groups series", func(t *testing.T) {
store := NewInsightStore(insightsDB)
sampleIntervalUnit := "MONTH"
repoCriteria := "repo:a"
_, err := store.CreateSeries(ctx, types.InsightSeries{
SeriesID: "capture-group-1",
Query: "well hello there",
@ -1014,6 +1035,7 @@ func TestCreateSeries(t *testing.T) {
CreatedAt: now,
GeneratedFromCaptureGroups: true,
GenerationMethod: types.Search,
RepositoryCriteria: &repoCriteria,
})
if err != nil {
return

1
enterprise/internal/insights/store/testdata/TestCreateSeries/test_create_and_get_capture_groups_series.golden vendored
View File

@ -13,4 +13,5 @@
GeneratedFromCaptureGroups: true,
GenerationMethod: types.GenerationMethod("search"),
SupportsAugmentation: true,
RepositoryCriteria: valast.Addr("repo:a").(*string),
}}

2
enterprise/internal/insights/types/types.go
View File

@ -41,6 +41,7 @@ type InsightViewSeries struct {
GroupBy *string
BackfillAttempts int32
SupportsAugmentation bool
RepositoryCriteria *string
}
type Insight struct {
@ -107,6 +108,7 @@ type InsightSeries struct {
GroupBy *string
BackfillAttempts int32
SupportsAugmentation bool
RepositoryCriteria *string
}
type IntervalUnit string

13
internal/database/schema.codeinsights.json
View File

@ -1045,6 +1045,19 @@
"GenerationExpression": "",
"Comment": ""
},
{
"Name": "repository_criteria",
"Index": 23,
"TypeName": "text",
"IsNullable": true,
"Default": "",
"CharacterMaximumLength": 0,
"IsIdentity": false,
"IdentityGeneration": "",
"IsGenerated": "NEVER",
"GenerationExpression": "",
"Comment": "The search criteria used to determine the repositories that are included in this series."
},
{
"Name": "sample_interval_unit",
"Index": 13,

3
internal/database/schema.codeinsights.md
View File

@ -159,6 +159,7 @@ Stores queries that were unsuccessful or otherwise flagged as incomplete or inco
needs_migration | boolean | | |
backfill_completed_at | timestamp without time zone | | |
supports_augmentation | boolean | | not null | true
repository_criteria | text | | |
Indexes:
"insight_series_pkey" PRIMARY KEY, btree (id)
"insight_series_series_id_unique_idx" UNIQUE, btree (series_id)
@ -193,6 +194,8 @@ Data series that comprise code insights.
**query**: Query string that generates this series
**repository_criteria**: The search criteria used to determine the repositories that are included in this series.
**series_id**: Timestamp that this series completed a full repository iteration for backfill. This flag has limited semantic value, and only means it tried to queue up queries for each repository. It does not guarantee success on those queries.
# Table "public.insight_series_backfill"

1
migrations/codeinsights/1670253074_insight_series_repo_criteria/down.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE IF EXISTS insight_series DROP COLUMN IF EXISTS repository_criteria;

2
migrations/codeinsights/1670253074_insight_series_repo_criteria/metadata.yaml Normal file
View File

@ -0,0 +1,2 @@
name: insight series repo criteria
parents: [1666729025, 1667309737]

5
migrations/codeinsights/1670253074_insight_series_repo_criteria/up.sql Normal file
View File

@ -0,0 +1,5 @@
ALTER TABLE IF EXISTS insight_series
ADD COLUMN IF NOT EXISTS repository_criteria text;
COMMENT ON COLUMN insight_series.repository_criteria IS 'The search criteria used to determine the repositories that are included in this series.';

5
migrations/codeinsights/squashed.sql
View File

@ -172,7 +172,8 @@ CREATE TABLE insight_series (
backfill_attempts integer DEFAULT 0 NOT NULL,
needs_migration boolean,
backfill_completed_at timestamp without time zone,
supports_augmentation boolean DEFAULT true NOT NULL
supports_augmentation boolean DEFAULT true NOT NULL,
repository_criteria text
);
COMMENT ON TABLE insight_series IS 'Data series that comprise code insights.';
@ -197,6 +198,8 @@ COMMENT ON COLUMN insight_series.generation_method IS 'Specifies the execution m
COMMENT ON COLUMN insight_series.just_in_time IS 'Specifies if the series should be resolved just in time at query time, or recorded in background processing.';
COMMENT ON COLUMN insight_series.repository_criteria IS 'The search criteria used to determine the repositories that are included in this series.';
CREATE TABLE insight_series_backfill (
id integer NOT NULL,
series_id integer NOT NULL,