All specs using :elastic should clear redis shared state
Proposal
All specs that utilize :elastic
or :elastic_delete_by_query
should also be clearing the redis shared state. Currently the Sidekiq queues are being cleared manually in ee/spec/support/elastic.rb
. Every instance in a spec where :elastic
or :elastic_delete_by_query
is used should have :clean_gitlab_redis_shared_state
added to it. Once all of the instances are fixed in the list below, the clear_tracking!
method should be removed from ee/spec/support/elastic.rb
as it will no longer be needed.
Example:
# Before
RSpec.describe 'Group elastic search', :js, :elastic, :sidekiq_might_not_need_inline do
# After
RSpec.describe 'Group elastic search', :js, :elastic, :clean_gitlab_redis_shared_state, :sidekiq_might_not_need_inline do
List of remaining specs to fix
A new MR should be opened for each spec file and items checked off as they are completed.
-
./ee/spec/tasks/gitlab/elastic_rake_spec.rb:5
: RSpec.describe 'gitlab:elastic namespace rake tasks', :elastic do -
./ee/spec/elastic/migrate/20210421140400_add_new_data_to_merge_requests_documents_spec.rb:6
: RSpec.describe AddNewDataToMergeRequestsDocuments, :elastic, :sidekiq_inline do -
./ee/spec/elastic/migrate/202101281636000_add_permissions_data_to_notes_documents_spec.rb:6
: RSpec.describe AddPermissionsDataToNotesDocuments, :elastic, :sidekiq_inline do -
./ee/spec/elastic/migrate/20210112165500_delete_issues_from_original_index_spec.rb:6
: RSpec.describe DeleteIssuesFromOriginalIndex, :elastic, :sidekiq_inline do -
./ee/spec/elastic/migrate/20201123123400_migrate_issues_to_separate_index_spec.rb:6
: RSpec.describe MigrateIssuesToSeparateIndex, :elastic, :sidekiq_inline do -
./ee/spec/elastic/migrate/20210510143200_delete_notes_from_original_index_spec.rb:6
: RSpec.describe DeleteNotesFromOriginalIndex, :elastic, :sidekiq_inline do -
./ee/spec/elastic/migrate/20210127154600_remove_permissions_data_from_notes_documents_spec.rb:6
: RSpec.describe RemovePermissionsDataFromNotesDocuments, :elastic, :sidekiq_inline do -
./ee/spec/elastic/migrate/20210201104800_migrate_notes_to_separate_index_spec.rb:26
: describe '.migrate', :elastic, :clean_gitlab_redis_shared_state do -
./ee/spec/elastic/migrate/20210429154500_migrate_merge_requests_to_separate_index_spec.rb:26
: describe '.migrate', :elastic, :clean_gitlab_redis_shared_state do -
./ee/spec/elastic/migrate/20201116142400_add_new_data_to_issues_documents_spec.rb:6
: RSpec.describe AddNewDataToIssuesDocuments, :elastic, :sidekiq_inline do -
./ee/spec/features/search/elastic/group_search_spec.rb:5
: RSpec.describe 'Group elastic search', :js, :elastic, :sidekiq_might_not_need_inline do -
./ee/spec/features/search/elastic/group_search_spec.rb:98
: RSpec.describe 'Group elastic search redactions', :elastic do -
./ee/spec/features/search/elastic/project_search_spec.rb:5
: RSpec.describe 'Project elastic search', :js, :elastic do -
./ee/spec/features/search/elastic/project_search_spec.rb:129
: RSpec.describe 'Project elastic search redactions', :elastic do -
./ee/spec/features/search/elastic/snippet_search_spec.rb:5
: RSpec.describe 'Snippet elastic search', :js, :elastic, :aggregate_failures, :sidekiq_might_not_need_inline do -
./ee/spec/features/search/elastic/global_search_spec.rb:5
: RSpec.describe 'Global elastic search', :elastic, :sidekiq_inline do -
./ee/spec/features/search/elastic/global_search_spec.rb:278
: RSpec.describe 'Global elastic search redactions', :elastic do -
./ee/spec/models/project_spec.rb:2868
: context 'indexing updates in Elasticsearch', :elastic do -
./ee/spec/models/elastic/migration_record_spec.rb:5
: RSpec.describe Elastic::MigrationRecord, :elastic do -
./ee/spec/models/issue_spec.rb:496
: context 'ES related specs', :elastic do -
./ee/spec/models/repository_spec.rb:81
: describe "Elastic search", :elastic do -
./ee/spec/models/concerns/elastic/project_spec.rb:5
: RSpec.describe Project, :elastic do -
./ee/spec/models/concerns/elastic/project_wiki_spec.rb:5
: RSpec.describe ProjectWiki, :elastic do -
./ee/spec/models/concerns/elastic/issue_spec.rb:5
: RSpec.describe Issue, :elastic do -
./ee/spec/models/concerns/elastic/repository_spec.rb:5
: RSpec.describe Repository, :elastic do -
./ee/spec/models/concerns/elastic/snippet_spec.rb:5
: RSpec.describe Snippet, :elastic do -
./ee/spec/models/concerns/elastic/milestone_spec.rb:5
: RSpec.describe Milestone, :elastic do -
./ee/spec/models/concerns/elastic/merge_request_spec.rb:5
: RSpec.describe MergeRequest, :elastic do -
./ee/spec/models/concerns/elastic/note_spec.rb:5
: RSpec.describe Note, :elastic do -
./ee/spec/requests/search_controller_spec.rb:34
: context 'when elasticsearch is enabled', :elastic, :sidekiq_inline do -
./ee/spec/requests/api/search_spec.rb:358
: context 'when elasticsearch is enabled', :elastic do -
./ee/spec/requests/api/search_spec.rb:396
: context 'when elasticsearch is enabled', :elastic do -
./ee/spec/requests/api/search_spec.rb:499
: context 'when elasticsearch is enabled', :elastic do -
./ee/spec/elastic_integration/global_search_spec.rb:5
: RSpec.describe 'GlobalSearch', :elastic do -
./ee/spec/elastic_integration/repository_index_spec.rb:5
: RSpec.describe 'Repository index', :elastic do -
./ee/spec/support/elastic.rb:59
: config.around(:each, :elastic) do |example| -
./ee/spec/support/elastic.rb:68
: config.before(:context, :elastic_delete_by_query) do -
./ee/spec/support/elastic.rb:72
: config.after(:context, :elastic_delete_by_query) do -
./ee/spec/support/elastic.rb:76
: config.around(:each, :elastic_delete_by_query) do |example| -
./ee/spec/support/elastic.rb:85
: config.include ElasticsearchHelpers, :elastic -
./ee/spec/support/elastic.rb:86
: config.include ElasticsearchHelpers, :elastic_delete_by_query -
./ee/spec/support/shared_examples/lib/gitlab/elastic/search_results_shared_examples.rb:5
: context "for scope #{scope}", :elastic, :request_store do -
./ee/spec/support/shared_examples/lib/gitlab/elastic/search_results_shared_examples.rb:28
: context "for scope #{scope}", :elastic, :request_store do -
./ee/spec/support/shared_examples/services/search_notes_shared_examples.rb:4
: context 'notes confidentiality', :elastic, :sidekiq_inline do -
./ee/spec/support/shared_examples/services/search_query_applies_joins_based_on_migrations_shared_examples.rb:4
: context 'using joins for global permission checks', :elastic do -
./ee/spec/support/shared_examples/services/search_service_shared_examples.rb:49
: context 'advanced syntax queries for all scopes', :elastic, :sidekiq_inline do -
./ee/spec/lib/gitlab/elastic/bulk_indexer_spec.rb:5
: RSpec.describe Gitlab::Elastic::BulkIndexer, :elastic do -
./ee/spec/lib/gitlab/elastic/snippet_search_results_spec.rb:5
: RSpec.describe Gitlab::Elastic::SnippetSearchResults, :elastic, :sidekiq_might_not_need_inline do -
./ee/spec/lib/gitlab/elastic/search_results_spec.rb:5
: RSpec.describe Gitlab::Elastic::SearchResults, :elastic, :sidekiq_might_not_need_inline do -
./ee/spec/lib/gitlab/elastic/project_search_results_spec.rb:5
: RSpec.describe Gitlab::Elastic::ProjectSearchResults, :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:24
: context 'empty project', :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:35
: context 'when indexing an unborn head', :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:58
: context 'with an indexed project', :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:83
: context 'when indexing a HEAD commit', :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:132
: context 'when indexing a non-HEAD commit', :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:203
: context "when indexing a project's wiki", :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:359
: context 'when a file is larger than elasticsearch_indexed_file_size_limit_kb', :elastic do -
./ee/spec/lib/gitlab/elastic/indexer_spec.rb:378
: context 'when a file path is larger than elasticsearch max size of 512 bytes', :elastic do -
./ee/spec/lib/gitlab/elastic/group_search_results_spec.rb:5
: RSpec.describe Gitlab::Elastic::GroupSearchResults, :elastic do -
./ee/spec/lib/gitlab/instrumentation/elasticsearch_transport_spec.rb:8
: RSpec.describe ::Gitlab::Instrumentation::ElasticsearchTransport, :elastic, :request_store do -
./ee/spec/lib/gitlab/instrumentation/elasticsearch_transport_spec.rb:55
: RSpec.describe ::Gitlab::Instrumentation::ElasticsearchTransportInterceptor, :elastic, :request_store do -
./ee/spec/lib/gitlab/instrumentation_helper_spec.rb:49
: context 'when Elasticsearch calls are made', :elastic do -
./ee/spec/lib/elastic/migration_spec.rb:5
: RSpec.describe Elastic::Migration, :elastic do -
./ee/spec/lib/elastic/latest/project_wiki_class_proxy_spec.rb:5
: RSpec.describe Elastic::Latest::ProjectWikiClassProxy, :elastic do -
./ee/spec/lib/elastic/latest/git_class_proxy_spec.rb:5
: RSpec.describe Elastic::Latest::GitClassProxy, :elastic do -
./ee/spec/lib/ee/gitlab/elastic/helper_spec.rb:335
: describe '#delete_migration_record', :elastic do -
./ee/spec/lib/gem_extensions/elasticsearch/model/adapter/active_record/records_spec.rb:5
: RSpec.describe Elasticsearch::Model::Adapter::ActiveRecord::Records, :elastic do -
./ee/spec/lib/peek/views/elasticsearch_spec.rb:8
: RSpec.describe Peek::Views::Elasticsearch, :elastic, :request_store do -
./ee/spec/workers/elastic/migration_worker_spec.rb:5
: RSpec.describe Elastic::MigrationWorker, :elastic do -
./ee/spec/workers/elastic_delete_project_worker_spec.rb:5
: RSpec.describe ElasticDeleteProjectWorker, :elastic, :sidekiq_inline do -
./ee/spec/workers/elastic_namespace_indexer_worker_spec.rb:21
: context 'when ES is enabled', :elastic do -
./ee/spec/controllers/ee/search_controller_spec.rb:5
: RSpec.describe SearchController, :elastic do -
./ee/spec/services/search_service_spec.rb:7
: context 'redacting search results (repository)', :elastic, :sidekiq_inline do -
./ee/spec/services/elastic/cluster_reindexing_service_spec.rb
: 5:RSpec.describe Elastic::ClusterReindexingService, :elastic do -
./ee/spec/services/elastic/process_bookkeeping_service_spec.rb
: 5:RSpec.describe Elastic::ProcessBookkeepingService, :clean_gitlab_redis_shared_state, :elastic do -
./ee/spec/services/elastic/data_migration_service_spec.rb:5
: RSpec.describe Elastic::DataMigrationService, :elastic do -
./ee/spec/services/elastic/legacy_reindexing_service_spec.rb:5
: RSpec.describe Elastic::LegacyReindexingService, :elastic do -
./ee/spec/services/ee/git/branch_push_service_spec.rb:41
: context 'ElasticSearch indexing', :elastic do -
./ee/spec/services/projects/transfer_service_spec.rb:68
: describe 'elasticsearch indexing', :elastic, :aggregate_failures do -
./ee/spec/services/search/snippet_service_spec.rb:24
: context 'visibility', :elastic_delete_by_query, :sidekiq_inline do -
./ee/spec/services/search/project_service_spec.rb:45
: context 'visibility', :elastic_delete_by_query, :sidekiq_inline do -
./ee/spec/services/search/project_service_spec.rb:275
: context 'sorting', :elastic, :sidekiq_inline do -
./ee/spec/services/search/group_service_spec.rb:53
: context 'finding projects by name', :elastic, :sidekiq_might_not_need_inline do -
./ee/spec/services/search/group_service_spec.rb:70
: context 'notes search', :elastic do -
./ee/spec/services/search/group_service_spec.rb:79
: context 'visibility', :elastic_delete_by_query, :sidekiq_inline do -
./ee/spec/services/search/group_service_spec.rb:334
: context 'sorting', :elastic do -
./ee/spec/services/search/global_service_spec.rb:39
: context 'visibility', :elastic_delete_by_query, :sidekiq_inline do -
./ee/spec/services/search/global_service_spec.rb:325
: context 'sorting', :elastic, :sidekiq_inline do
The following discussion from !60651 (merged) was the inspiration for creating this issue:
-
@DylanGriffith started a discussion: (+2 comments) I think we also need to clear the initial queue:
Elastic::ProcessInitialBookkeepingService.clear_tracking! Elastic::ProcessBookkeepingService.clear_tracking!