From 85bbc9357f86b37d1da5cc843881866b8b07ef70 Mon Sep 17 00:00:00 2001 From: Eric Fritz Date: Tue, 6 Jun 2023 18:05:05 -0500 Subject: [PATCH] ranking: Make counts per definition unique (#52949) --- .../ranking/internal/store/mapper.go | 12 +++++++---- .../ranking/internal/store/mapper_test.go | 2 +- internal/database/migration/runner/run.go | 2 +- internal/database/schema.json | 20 +++++++++---------- internal/database/schema.md | 2 +- migrations/BUILD.bazel | 6 ++++++ .../down.sql | 1 + .../metadata.yaml | 2 ++ .../up.sql | 3 +++ .../1685984018_drop_duplicate_index/down.sql | 1 + .../metadata.yaml | 2 ++ .../1685984018_drop_duplicate_index/up.sql | 1 + migrations/frontend/squashed.sql | 4 ++-- 13 files changed, 39 insertions(+), 19 deletions(-) create mode 100644 migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/down.sql create mode 100644 migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/metadata.yaml create mode 100644 migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/up.sql create mode 100644 migrations/frontend/1685984018_drop_duplicate_index/down.sql create mode 100644 migrations/frontend/1685984018_drop_duplicate_index/metadata.yaml create mode 100644 migrations/frontend/1685984018_drop_duplicate_index/up.sql diff --git a/enterprise/internal/codeintel/ranking/internal/store/mapper.go b/enterprise/internal/codeintel/ranking/internal/store/mapper.go index 283d781bc15..a548132622f 100644 --- a/enterprise/internal/codeintel/ranking/internal/store/mapper.go +++ b/enterprise/internal/codeintel/ranking/internal/store/mapper.go @@ -195,12 +195,14 @@ referenced_definitions AS ( GROUP BY s.definition_id ), ins AS ( - INSERT INTO codeintel_ranking_path_counts_inputs (definition_id, count, graph_key) + INSERT INTO codeintel_ranking_path_counts_inputs AS target (graph_key, definition_id, count, processed) SELECT + %s, rx.definition_id, rx.count, - %s + false FROM referenced_definitions rx + ON CONFLICT (graph_key, definition_id) WHERE NOT processed DO UPDATE SET count = target.count + EXCLUDED.count RETURNING 1 ), set_progress AS ( @@ -336,11 +338,12 @@ expanded_unprocessed_path_counts AS ( FROM unprocessed_path_counts upc ), ins AS ( - INSERT INTO codeintel_ranking_path_counts_inputs (definition_id, count, graph_key) + INSERT INTO codeintel_ranking_path_counts_inputs (graph_key, definition_id, count, processed) SELECT + %s, rd.id, 0, - %s + false FROM locked_path_counts lpc JOIN expanded_unprocessed_path_counts eupc ON eupc.id = lpc.codeintel_initial_path_ranks_id JOIN codeintel_ranking_definitions rd ON @@ -349,6 +352,7 @@ ins AS ( WHERE rd.graph_key = %s AND rd.symbol_name = '$' + ON CONFLICT DO NOTHING RETURNING 1 ), set_progress AS ( diff --git a/enterprise/internal/codeintel/ranking/internal/store/mapper_test.go b/enterprise/internal/codeintel/ranking/internal/store/mapper_test.go index 1ee9c8ed4e9..53c35596887 100644 --- a/enterprise/internal/codeintel/ranking/internal/store/mapper_test.go +++ b/enterprise/internal/codeintel/ranking/internal/store/mapper_test.go @@ -442,7 +442,7 @@ func TestVacuumStaleGraphs(t *testing.T) { } if _, err := db.ExecContext(ctx, ` INSERT INTO codeintel_ranking_path_counts_inputs (definition_id, count, graph_key) - SELECT 0, 100, $1 FROM generate_series(1, 30) + SELECT v, 100, $1 FROM generate_series(1, 30) AS v `, graphKey); err != nil { t.Fatalf("failed to insert ranking path count inputs: %s", err) } diff --git a/internal/database/migration/runner/run.go b/internal/database/migration/runner/run.go index f904a6ae353..c377c717278 100644 --- a/internal/database/migration/runner/run.go +++ b/internal/database/migration/runner/run.go @@ -491,7 +491,7 @@ pollIndexStatusLoop: // of the index. var ( - pgErr pgconn.PgError + pgErr *pgconn.PgError raceDetected bool errorFilter = func(err error) error { diff --git a/internal/database/schema.json b/internal/database/schema.json index e9ef5c0f72c..68bee0aa059 100755 --- a/internal/database/schema.json +++ b/internal/database/schema.json @@ -8245,6 +8245,16 @@ } ], "Indexes": [ + { + "Name": "codeintel_ranking_path_counts_inputs_graph_key_unique_definitio", + "IsPrimaryKey": false, + "IsUnique": true, + "IsExclusion": false, + "IsDeferrable": false, + "IndexDefinition": "CREATE UNIQUE INDEX codeintel_ranking_path_counts_inputs_graph_key_unique_definitio ON codeintel_ranking_path_counts_inputs USING btree (graph_key, definition_id) WHERE NOT processed", + "ConstraintType": "", + "ConstraintDefinition": "" + }, { "Name": "codeintel_ranking_path_counts_inputs_pkey", "IsPrimaryKey": true, @@ -8255,16 +8265,6 @@ "ConstraintType": "p", "ConstraintDefinition": "PRIMARY KEY (id)" }, - { - "Name": "codeintel_ranking_path_counts_inputs_graph_key_definition_id", - "IsPrimaryKey": false, - "IsUnique": false, - "IsExclusion": false, - "IsDeferrable": false, - "IndexDefinition": "CREATE INDEX codeintel_ranking_path_counts_inputs_graph_key_definition_id ON codeintel_ranking_path_counts_inputs USING btree (graph_key, definition_id, id) WHERE NOT processed", - "ConstraintType": "", - "ConstraintDefinition": "" - }, { "Name": "codeintel_ranking_path_counts_inputs_graph_key_id", "IsPrimaryKey": false, diff --git a/internal/database/schema.md b/internal/database/schema.md index f260faf59da..59f9921dc39 100755 --- a/internal/database/schema.md +++ b/internal/database/schema.md @@ -1021,7 +1021,7 @@ Indexes: definition_id | bigint | | | Indexes: "codeintel_ranking_path_counts_inputs_pkey" PRIMARY KEY, btree (id) - "codeintel_ranking_path_counts_inputs_graph_key_definition_id" btree (graph_key, definition_id, id) WHERE NOT processed + "codeintel_ranking_path_counts_inputs_graph_key_unique_definitio" UNIQUE, btree (graph_key, definition_id) WHERE NOT processed "codeintel_ranking_path_counts_inputs_graph_key_id" btree (graph_key, id) ``` diff --git a/migrations/BUILD.bazel b/migrations/BUILD.bazel index 1e81a20b3ba..ab3d6a1d161 100644 --- a/migrations/BUILD.bazel +++ b/migrations/BUILD.bazel @@ -1026,6 +1026,12 @@ go_library( "frontend/1685727930_junk_cleanup/down.sql", "frontend/1685727930_junk_cleanup/metadata.yaml", "frontend/1685727930_junk_cleanup/up.sql", + "frontend/1685983690_make_path_count_inputs_unique_by_definition_id/down.sql", + "frontend/1685983690_make_path_count_inputs_unique_by_definition_id/metadata.yaml", + "frontend/1685983690_make_path_count_inputs_unique_by_definition_id/up.sql", + "frontend/1685984018_drop_duplicate_index/down.sql", + "frontend/1685984018_drop_duplicate_index/metadata.yaml", + "frontend/1685984018_drop_duplicate_index/up.sql", "frontend/1686042710_add_assigned_teams_table/down.sql", "frontend/1686042710_add_assigned_teams_table/metadata.yaml", "frontend/1686042710_add_assigned_teams_table/up.sql", diff --git a/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/down.sql b/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/down.sql new file mode 100644 index 00000000000..8c0a8357826 --- /dev/null +++ b/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS codeintel_ranking_path_counts_inputs_graph_key_unique_definition_id; diff --git a/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/metadata.yaml b/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/metadata.yaml new file mode 100644 index 00000000000..87ba060fc6f --- /dev/null +++ b/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/metadata.yaml @@ -0,0 +1,2 @@ +name: Make path count inputs unique by definition_id. +parents: [1685695443, 1685697346, 1685712730, 1684858266] diff --git a/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/up.sql b/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/up.sql new file mode 100644 index 00000000000..7af3fae890b --- /dev/null +++ b/migrations/frontend/1685983690_make_path_count_inputs_unique_by_definition_id/up.sql @@ -0,0 +1,3 @@ +UPDATE codeintel_ranking_path_counts_inputs SET processed = true WHERE NOT processed; + +CREATE UNIQUE INDEX IF NOT EXISTS codeintel_ranking_path_counts_inputs_graph_key_unique_definition_id ON codeintel_ranking_path_counts_inputs(graph_key, definition_id) WHERE NOT processed; diff --git a/migrations/frontend/1685984018_drop_duplicate_index/down.sql b/migrations/frontend/1685984018_drop_duplicate_index/down.sql new file mode 100644 index 00000000000..7d7d30fe2e5 --- /dev/null +++ b/migrations/frontend/1685984018_drop_duplicate_index/down.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS codeintel_ranking_path_counts_inputs_graph_key_definition_id ON codeintel_ranking_path_counts_inputs(graph_key, definition_id, id) WHERE NOT processed; diff --git a/migrations/frontend/1685984018_drop_duplicate_index/metadata.yaml b/migrations/frontend/1685984018_drop_duplicate_index/metadata.yaml new file mode 100644 index 00000000000..531998c24ef --- /dev/null +++ b/migrations/frontend/1685984018_drop_duplicate_index/metadata.yaml @@ -0,0 +1,2 @@ +name: Drop duplicate index. +parents: [1685983690] diff --git a/migrations/frontend/1685984018_drop_duplicate_index/up.sql b/migrations/frontend/1685984018_drop_duplicate_index/up.sql new file mode 100644 index 00000000000..ddd923afea1 --- /dev/null +++ b/migrations/frontend/1685984018_drop_duplicate_index/up.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS codeintel_ranking_path_counts_inputs_graph_key_definition_id; diff --git a/migrations/frontend/squashed.sql b/migrations/frontend/squashed.sql index 846e92779d3..63777275764 100755 --- a/migrations/frontend/squashed.sql +++ b/migrations/frontend/squashed.sql @@ -5706,10 +5706,10 @@ CREATE INDEX codeintel_ranking_exports_graph_key_last_scanned_at ON codeintel_ra CREATE UNIQUE INDEX codeintel_ranking_exports_graph_key_upload_id ON codeintel_ranking_exports USING btree (graph_key, upload_id); -CREATE INDEX codeintel_ranking_path_counts_inputs_graph_key_definition_id ON codeintel_ranking_path_counts_inputs USING btree (graph_key, definition_id, id) WHERE (NOT processed); - CREATE INDEX codeintel_ranking_path_counts_inputs_graph_key_id ON codeintel_ranking_path_counts_inputs USING btree (graph_key, id); +CREATE UNIQUE INDEX codeintel_ranking_path_counts_inputs_graph_key_unique_definitio ON codeintel_ranking_path_counts_inputs USING btree (graph_key, definition_id) WHERE (NOT processed); + CREATE INDEX codeintel_ranking_references_exported_upload_id ON codeintel_ranking_references USING btree (exported_upload_id); CREATE INDEX codeintel_ranking_references_graph_key_id ON codeintel_ranking_references USING btree (graph_key, id);