diff --git a/enterprise/internal/codeintel/stores/lsifstore/data_write_documentation.go b/enterprise/internal/codeintel/stores/lsifstore/data_write_documentation.go index 033d0a7cde5..ff93c449af8 100644 --- a/enterprise/internal/codeintel/stores/lsifstore/data_write_documentation.go +++ b/enterprise/internal/codeintel/stores/lsifstore/data_write_documentation.go @@ -508,11 +508,9 @@ func (s *Store) truncateDocumentationSearchIndexSize(ctx context.Context, tableS return nil } -// TODO(apidocs): future: introduce materialized count for this table and for other interesting API -// docs data points in general. https://github.com/sourcegraph/sourcegraph/pull/25206#discussion_r714270738 const countDocumentationSearchRowsQuery = ` -- source: enterprise/internal/codeintel/stores/lsifstore/data_write_documentation.go:truncateDocumentationSearchIndexSize -SELECT count(*)::bigint FROM lsif_data_docs_search_$SUFFIX +SELECT count::bigint FROM lsif_data_apidocs_num_search_results_$SUFFIX ` const truncateDocumentationSearchRowsQuery = ` diff --git a/enterprise/internal/codeintel/stores/lsifstore/migration/apidocs_search.go b/enterprise/internal/codeintel/stores/lsifstore/migration/apidocs_search.go index 50f5a7af0a3..5a7a17efd20 100644 --- a/enterprise/internal/codeintel/stores/lsifstore/migration/apidocs_search.go +++ b/enterprise/internal/codeintel/stores/lsifstore/migration/apidocs_search.go @@ -69,9 +69,10 @@ func (m *apiDocsSearchMigrator) Progress(ctx context.Context) (float64, error) { const apiDocsSearchMigratorProgressQuery = ` -- source: enterprise/internal/codeintel/stores/lsifstore/migration/apidocs_search.go:Progress -SELECT CASE c2.count WHEN 0 THEN 1 ELSE cast(c1.count as float) / cast(c2.count as float) END FROM - (SELECT count(DISTINCT dump_id) FROM lsif_data_documentation_pages WHERE search_indexed='true') c1, - (SELECT count(DISTINCT dump_id) FROM lsif_data_documentation_pages) c2 +SELECT CASE c2.count WHEN 0 THEN 1 ELSE cast(c1.count as float) / cast(c2.count as float) END +FROM + (SELECT * FROM lsif_data_apidocs_num_dumps_indexed) c1, + (SELECT * FROM lsif_data_apidocs_num_dumps) c2 ` // Up runs a batch of the migration. This method is called repeatedly until the Progress diff --git a/internal/conf/computed.go b/internal/conf/computed.go index 970817dddaf..6e34a9db26f 100644 --- a/internal/conf/computed.go +++ b/internal/conf/computed.go @@ -301,7 +301,7 @@ func EventLoggingEnabled() bool { func APIDocsSearchIndexingEnabled() bool { val := ExperimentalFeatures().ApidocsSearchIndexing if val == "" { - return false // default to off for now until we've fixed https://github.com/sourcegraph/sourcegraph/issues/25856 + return true } return val == "enabled" } diff --git a/internal/database/schema.codeintel.md b/internal/database/schema.codeintel.md index 83fd49e3be2..57b491462d3 100644 --- a/internal/database/schema.codeintel.md +++ b/internal/database/schema.codeintel.md @@ -9,6 +9,46 @@ Indexes: ``` +# Table "public.lsif_data_apidocs_num_dumps" +``` + Column | Type | Collation | Nullable | Default +--------+--------+-----------+----------+--------- + count | bigint | | | + +``` + +# Table "public.lsif_data_apidocs_num_dumps_indexed" +``` + Column | Type | Collation | Nullable | Default +--------+--------+-----------+----------+--------- + count | bigint | | | + +``` + +# Table "public.lsif_data_apidocs_num_pages" +``` + Column | Type | Collation | Nullable | Default +--------+--------+-----------+----------+--------- + count | bigint | | | + +``` + +# Table "public.lsif_data_apidocs_num_search_results_private" +``` + Column | Type | Collation | Nullable | Default +--------+--------+-----------+----------+--------- + count | bigint | | | + +``` + +# Table "public.lsif_data_apidocs_num_search_results_public" +``` + Column | Type | Collation | Nullable | Default +--------+--------+-----------+----------+--------- + count | bigint | | | + +``` + # Table "public.lsif_data_definitions" ``` Column | Type | Collation | Nullable | Default @@ -142,6 +182,9 @@ Foreign-key constraints: "lsif_data_docs_search_private_lang_name_id_fk" FOREIGN KEY (lang_name_id) REFERENCES lsif_data_docs_search_lang_names_private(id) "lsif_data_docs_search_private_repo_name_id_fk" FOREIGN KEY (repo_name_id) REFERENCES lsif_data_docs_search_repo_names_private(id) "lsif_data_docs_search_private_tags_id_fk" FOREIGN KEY (tags_id) REFERENCES lsif_data_docs_search_tags_private(id) +Triggers: + lsif_data_docs_search_private_delete AFTER DELETE ON lsif_data_docs_search_private REFERENCING OLD TABLE AS oldtbl FOR EACH STATEMENT EXECUTE FUNCTION lsif_data_docs_search_private_delete() + lsif_data_docs_search_private_insert AFTER INSERT ON lsif_data_docs_search_private REFERENCING NEW TABLE AS newtbl FOR EACH STATEMENT EXECUTE FUNCTION lsif_data_docs_search_private_insert() ``` @@ -209,6 +252,9 @@ Foreign-key constraints: "lsif_data_docs_search_public_lang_name_id_fk" FOREIGN KEY (lang_name_id) REFERENCES lsif_data_docs_search_lang_names_public(id) "lsif_data_docs_search_public_repo_name_id_fk" FOREIGN KEY (repo_name_id) REFERENCES lsif_data_docs_search_repo_names_public(id) "lsif_data_docs_search_public_tags_id_fk" FOREIGN KEY (tags_id) REFERENCES lsif_data_docs_search_tags_public(id) +Triggers: + lsif_data_docs_search_public_delete AFTER DELETE ON lsif_data_docs_search_public REFERENCING OLD TABLE AS oldtbl FOR EACH STATEMENT EXECUTE FUNCTION lsif_data_docs_search_public_delete() + lsif_data_docs_search_public_insert AFTER INSERT ON lsif_data_docs_search_public REFERENCING NEW TABLE AS newtbl FOR EACH STATEMENT EXECUTE FUNCTION lsif_data_docs_search_public_insert() ``` @@ -382,6 +428,10 @@ Maps documentation path IDs to their corresponding integral documentationResult search_indexed | boolean | | | false Indexes: "lsif_data_documentation_pages_pkey" PRIMARY KEY, btree (dump_id, path_id) +Triggers: + lsif_data_documentation_pages_delete AFTER DELETE ON lsif_data_documentation_pages REFERENCING OLD TABLE AS oldtbl FOR EACH STATEMENT EXECUTE FUNCTION lsif_data_documentation_pages_delete() + lsif_data_documentation_pages_insert AFTER INSERT ON lsif_data_documentation_pages REFERENCING NEW TABLE AS newtbl FOR EACH STATEMENT EXECUTE FUNCTION lsif_data_documentation_pages_insert() + lsif_data_documentation_pages_update AFTER UPDATE ON lsif_data_documentation_pages REFERENCING OLD TABLE AS oldtbl NEW TABLE AS newtbl FOR EACH STATEMENT EXECUTE FUNCTION lsif_data_documentation_pages_update() ``` diff --git a/migrations/codeintel/1000000025_apidocs_materialized_stats.down.sql b/migrations/codeintel/1000000025_apidocs_materialized_stats.down.sql new file mode 100644 index 00000000000..f2869f37cb5 --- /dev/null +++ b/migrations/codeintel/1000000025_apidocs_materialized_stats.down.sql @@ -0,0 +1,25 @@ +BEGIN; + +DROP TABLE lsif_data_apidocs_num_pages; +DROP TABLE lsif_data_apidocs_num_dumps; +DROP TABLE lsif_data_apidocs_num_dumps_indexed; +DROP TRIGGER lsif_data_documentation_pages_delete ON lsif_data_documentation_pages; +DROP TRIGGER lsif_data_documentation_pages_insert ON lsif_data_documentation_pages; +DROP TRIGGER lsif_data_documentation_pages_update ON lsif_data_documentation_pages; +DROP FUNCTION lsif_data_documentation_pages_delete; +DROP FUNCTION lsif_data_documentation_pages_insert; +DROP FUNCTION lsif_data_documentation_pages_update; + +DROP TABLE lsif_data_apidocs_num_search_results_public; +DROP TRIGGER lsif_data_docs_search_public_delete ON lsif_data_docs_search_public; +DROP TRIGGER lsif_data_docs_search_public_insert ON lsif_data_docs_search_public; +DROP FUNCTION lsif_data_docs_search_public_delete; +DROP FUNCTION lsif_data_docs_search_public_insert; + +DROP TABLE lsif_data_apidocs_num_search_results_private; +DROP TRIGGER lsif_data_docs_search_private_delete ON lsif_data_docs_search_private; +DROP TRIGGER lsif_data_docs_search_private_insert ON lsif_data_docs_search_private; +DROP FUNCTION lsif_data_docs_search_private_delete; +DROP FUNCTION lsif_data_docs_search_private_insert; + +COMMIT; diff --git a/migrations/codeintel/1000000025_apidocs_materialized_stats.up.sql b/migrations/codeintel/1000000025_apidocs_materialized_stats.up.sql new file mode 100644 index 00000000000..02689aaaea4 --- /dev/null +++ b/migrations/codeintel/1000000025_apidocs_materialized_stats.up.sql @@ -0,0 +1,128 @@ +BEGIN; + +-------------------------------------------------------- +-- Stats for the lsif_data_documentation_pages table. -- +-------------------------------------------------------- +CREATE TABLE lsif_data_apidocs_num_pages AS SELECT count(*) FROM lsif_data_documentation_pages; +CREATE TABLE lsif_data_apidocs_num_dumps AS SELECT count(DISTINCT dump_id) FROM lsif_data_documentation_pages; +CREATE TABLE lsif_data_apidocs_num_dumps_indexed AS SELECT count(DISTINCT dump_id) FROM lsif_data_documentation_pages WHERE search_indexed='true'; + +CREATE OR REPLACE FUNCTION lsif_data_documentation_pages_delete() +RETURNS TRIGGER LANGUAGE plpgsql +AS $$ +BEGIN +-- Decrement tally counting tables. +UPDATE lsif_data_apidocs_num_pages SET count = count - (select count(*) from oldtbl); +UPDATE lsif_data_apidocs_num_dumps SET count = count - (select count(DISTINCT dump_id) from oldtbl); +UPDATE lsif_data_apidocs_num_dumps_indexed SET count = count - (select count(DISTINCT dump_id) from oldtbl WHERE search_indexed='true'); +RETURN NULL; +END $$; + +CREATE TRIGGER lsif_data_documentation_pages_delete +AFTER DELETE ON lsif_data_documentation_pages +REFERENCING OLD TABLE AS oldtbl +FOR EACH STATEMENT EXECUTE PROCEDURE lsif_data_documentation_pages_delete(); + +CREATE OR REPLACE FUNCTION lsif_data_documentation_pages_insert() +RETURNS TRIGGER LANGUAGE plpgsql +AS $$ +BEGIN +-- Increment tally counting tables. +UPDATE lsif_data_apidocs_num_pages SET count = count + (select count(*) from newtbl); +UPDATE lsif_data_apidocs_num_dumps SET count = count + (select count(DISTINCT dump_id) from newtbl); +UPDATE lsif_data_apidocs_num_dumps_indexed SET count = count + (select count(DISTINCT dump_id) from newtbl WHERE search_indexed='true'); +RETURN NULL; +END $$; + +CREATE TRIGGER lsif_data_documentation_pages_insert +AFTER INSERT ON lsif_data_documentation_pages +REFERENCING NEW TABLE AS newtbl +FOR EACH STATEMENT EXECUTE PROCEDURE lsif_data_documentation_pages_insert(); + +CREATE OR REPLACE FUNCTION lsif_data_documentation_pages_update() +RETURNS TRIGGER LANGUAGE plpgsql +AS $$ +BEGIN +WITH + beforeIndexed AS (SELECT count(DISTINCT dump_id) FROM oldtbl WHERE search_indexed='true'), + afterIndexed AS (SELECT count(DISTINCT dump_id) FROM newtbl WHERE search_indexed='true') +UPDATE lsif_data_apidocs_num_dumps_indexed SET count=count + ((select * from afterIndexed) - (select * from beforeIndexed)); +RETURN NULL; +END $$; + +CREATE TRIGGER lsif_data_documentation_pages_update +AFTER UPDATE ON lsif_data_documentation_pages +REFERENCING OLD TABLE AS oldtbl NEW TABLE AS newtbl +FOR EACH STATEMENT EXECUTE PROCEDURE lsif_data_documentation_pages_update(); + +------------------------------------------------------- +-- Stats for the lsif_data_docs_search_public table. -- +------------------------------------------------------- +CREATE TABLE lsif_data_apidocs_num_search_results_public AS SELECT count(*) FROM lsif_data_docs_search_public; + +CREATE OR REPLACE FUNCTION lsif_data_docs_search_public_delete() +RETURNS TRIGGER LANGUAGE plpgsql +AS $$ +BEGIN +-- Decrement tally counting tables. +UPDATE lsif_data_apidocs_num_search_results_public SET count = count - (select count(*) from oldtbl); +RETURN NULL; +END $$; + +CREATE TRIGGER lsif_data_docs_search_public_delete +AFTER DELETE +ON lsif_data_docs_search_public +REFERENCING OLD TABLE AS oldtbl +FOR EACH STATEMENT EXECUTE PROCEDURE lsif_data_docs_search_public_delete(); + +CREATE OR REPLACE FUNCTION lsif_data_docs_search_public_insert() +RETURNS TRIGGER LANGUAGE plpgsql +AS $$ +BEGIN +-- Increment tally counting tables. +UPDATE lsif_data_apidocs_num_search_results_public SET count = count + (select count(*) from newtbl); +RETURN NULL; +END $$; + +CREATE TRIGGER lsif_data_docs_search_public_insert +AFTER INSERT +ON lsif_data_docs_search_public +REFERENCING NEW TABLE AS newtbl +FOR EACH STATEMENT EXECUTE PROCEDURE lsif_data_docs_search_public_insert(); + +------------------------------------------------------- +-- Stats for the lsif_data_docs_search_private table. -- +------------------------------------------------------- +CREATE TABLE lsif_data_apidocs_num_search_results_private AS SELECT count(*) FROM lsif_data_docs_search_private; + +CREATE OR REPLACE FUNCTION lsif_data_docs_search_private_delete() +RETURNS TRIGGER LANGUAGE plpgsql +AS $$ +BEGIN +-- Decrement tally counting tables. +UPDATE lsif_data_apidocs_num_search_results_private SET count = count - (select count(*) from oldtbl); +RETURN NULL; +END $$; + +CREATE TRIGGER lsif_data_docs_search_private_delete +AFTER DELETE +ON lsif_data_docs_search_private +REFERENCING OLD TABLE AS oldtbl +FOR EACH STATEMENT EXECUTE PROCEDURE lsif_data_docs_search_private_delete(); + +CREATE OR REPLACE FUNCTION lsif_data_docs_search_private_insert() +RETURNS TRIGGER LANGUAGE plpgsql +AS $$ +BEGIN +-- Increment tally counting tables. +UPDATE lsif_data_apidocs_num_search_results_private SET count = count + (select count(*) from newtbl); +RETURN NULL; +END $$; + +CREATE TRIGGER lsif_data_docs_search_private_insert +AFTER INSERT +ON lsif_data_docs_search_private +REFERENCING NEW TABLE AS newtbl +FOR EACH STATEMENT EXECUTE PROCEDURE lsif_data_docs_search_private_insert(); + +COMMIT;