ElasticDeleteProjectWorker failed with version conflict message
Summary
There are a lot of ElasticDeleteProjectWorker
job failures in the log. It may be caused by "version conflict" from the error mesage
Steps to reproduce
Check the log at https://log.gprd.gitlab.net/app/r/s/AFwib
What is the current bug behavior?
Exceptions related to ElasticDeleteProjectWorker
are seen in the logs
What is the expected correct behavior?
We shouldn't see these errors
Relevant logs and/or screenshots
exception backtrace
"lib/gitlab/instrumentation/elasticsearch_transport.rb:12:in `perform_request'",
"ee/app/workers/elastic_delete_project_worker.rb:81:in `remove_children_documents'",
"ee/app/workers/elastic_delete_project_worker.rb:18:in `perform'",
"ee/app/workers/concerns/elastic/indexing_control.rb:34:in `perform'",
"ee/app/workers/concerns/geo/skip_secondary.rb:14:in `perform'",
"lib/gitlab/sidekiq_middleware/skip_jobs.rb:49:in `call'",
"lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:29:in `call'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/concurrency_limit/middleware.rb:32:in `perform'",
"lib/gitlab/sidekiq_middleware/concurrency_limit/server.rb:8:in `call'",
"lib/click_house/migration_support/sidekiq_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:in `perform'",
"lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
"lib/gitlab/application_context.rb:130:in `block in use'",
"lib/gitlab/application_context.rb:130:in `use'",
"lib/gitlab/application_context.rb:64:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:40:in `within'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:in `block in call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:105:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:133:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:33:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:133:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:104:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
exception message
{
"took": 352,
"timed_out": false,
"total": 2060,
"deleted": 993,
"batches": 1,
"version_conflicts": 7,
"noops": 0,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"requests_per_second": -1,
"throttled_until_millis": 0,
"failures": [
{
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0",
"id": "merge_request_284008041",
"cause": {
"type": "version_conflict_engine_exception",
"reason": "[merge_request_284008041]: version conflict, required seqNo [9212391], primary term [1]. but no document was found",
"index_uuid": "jbmf1OqlS8mPtikf5Fh0Fw",
"shard": "6",
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0"
},
"status": 409
},
{
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0",
"id": "merge_request_284007983",
"cause": {
"type": "version_conflict_engine_exception",
"reason": "[merge_request_284007983]: version conflict, required seqNo [9212304], primary term [1]. but no document was found",
"index_uuid": "jbmf1OqlS8mPtikf5Fh0Fw",
"shard": "6",
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0"
},
"status": 409
},
{
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0",
"id": "merge_request_284007965",
"cause": {
"type": "version_conflict_engine_exception",
"reason": "[merge_request_284007965]: version conflict, required seqNo [9212308], primary term [1]. but no document was found",
"index_uuid": "jbmf1OqlS8mPtikf5Fh0Fw",
"shard": "6",
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0"
},
"status": 409
},
{
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0",
"id": "merge_request_284007970",
"cause": {
"type": "version_conflict_engine_exception",
"reason": "[merge_request_284007970]: version conflict, required seqNo [9212310], primary term [1]. but no document was found",
"index_uuid": "jbmf1OqlS8mPtikf5Fh0Fw",
"shard": "6",
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0"
},
"status": 409
},
{
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0",
"id": "merge_request_284007989",
"cause": {
"type": "version_conflict_engine_exception",
"reason": "[merge_request_284007989]: version conflict, required seqNo [9212341], primary term [1]. but no document was found",
"index_uuid": "jbmf1OqlS8mPtikf5Fh0Fw",
"shard": "6",
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0"
},
"status": 409
},
{
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0",
"id": "merge_request_284008016",
"cause": {
"type": "version_conflict_engine_exception",
"reason": "[merge_request_284008016]: version conflict, required seqNo [9212353], primary term [1]. but no document was found",
"index_uuid": "jbmf1OqlS8mPtikf5Fh0Fw",
"shard": "6",
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0"
},
"status": 409
},
{
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0",
"id": "merge_request_284008019",
"cause": {
"type": "version_conflict_engine_exception",
"reason": "[merge_request_284008019]: version conflict, required seqNo [9212354], primary term [1]. but no document was found",
"index_uuid": "jbmf1OqlS8mPtikf5Fh0Fw",
"shard": "6",
"index": "gitlab-production-merge_requests-20240205-1030-reindex-1000079-0"
},
"status": 409
}
]
}
Possible fixes
TBD