I went looking for logs in https://gitlab.com/gitlab-org/gitlab/-/issues/213620#note_318833488, and while I found this JSON entry in the sidekiq-cluster logs directly, it seems that any entry with `error_backtrace` and `MergeWorker` is coming up empty in https://log.gprd.gitlab.net/goto/37b8d01efeee0fc997858ca6857b05d1. It doesn't help that we've turned off FluentD logging because of the error spew (https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9530#note_312626014): ``` 2020-03-20 20:23:41 +0000 [warn]: #0 Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory 2020-03-20 20:23:41 +0000 [warn]: #0 Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory ``` This was the missing entry: ```json { "severity": "WARN", "time": "2020-04-06T23:46:45.928Z", "error_class": "Gitlab::Git::CommandError", "error_message": "9:Could not update normalize-import-error-messages. Please refresh and try again.", "context": "Job raised exception", "jobstr": "{\"class\":\"MergeWorker\",\"args\":[54646713,33100,{\"force_remove_source_branch\":\"1\",\"auto_merge_strategy\":\"merge_when_pipeline_succeeds\",\"should_remove_source_branch\":true,\"commit_message\":\"Merge branch 'normalize-import-error-messages' into 'master'\\n\\nNormalize error message between Gitea and Fogbugz importers\\n\\nSee merge request gitlab-org/gitlab!28802\",\"sha\":\"5342d1a7f5f3eb5edb1de40d560b5c3bdce42224\"}],\"retry\":3,\"queue\":\"merge\",\"jid\":\"94ec160f911e3c63b34939d1\",\"created_at\":1586216690.995178,\"meta.user\":\"mayra-cabrera\",\"meta.project\":\"gitlab-org/gitlab\",\"meta.root_namespace\":\"gitlab-org\",\"meta.subscription_plan\":\"gold\",\"meta.caller_id\":\"AutoMergeProcessWorker\",\"correlation_id\":\"67899bd2eb7a00d4986728e2156ca42c\",\"enqueued_at\":1586216784.5158522,\"error_message\":\"9:Could not update normalize-import-error-messages. Please refresh and try again.\",\"error_class\":\"Gitlab::Git::CommandError\",\"failed_at\":1586216705.6987324,\"retry_count\":1,\"retried_at\":1586216737.6654246}", "class": "MergeWorker", "args": [ 54646713, 33100, { "force_remove_source_branch": "1", "auto_merge_strategy": "merge_when_pipeline_succeeds", "should_remove_source_branch": true, "commit_message": "Merge branch 'normalize-import-error-messages' into 'master'\n\nNormalize error message between Gitea and Fogbugz importers\n\nSee merge request gitlab-org/gitlab!28802", "sha": "5342d1a7f5f3eb5edb1de40d560b5c3bdce42224" } ], "retry": 3, "queue": "merge", "jid": "94ec160f911e3c63b34939d1", "created_at": "2020-04-06T23:44:50.995Z", "meta.user": "mayra-cabrera", "meta.project": "gitlab-org/gitlab", "meta.root_namespace": "gitlab-org", "meta.subscription_plan": "gold", "meta.caller_id": "AutoMergeProcessWorker", "correlation_id": "67899bd2eb7a00d4986728e2156ca42c", "enqueued_at": "2020-04-06T23:46:24.515Z", "failed_at": "2020-04-06T23:45:05.698Z", "retry_count": 1, "retried_at": "2020-04-06T23:45:37.665Z", "error_backtrace": [ "lib/gitlab/git/wraps_gitaly_errors.rb:13:in `rescue in wrapped_gitaly_errors'", "lib/gitlab/git/wraps_gitaly_errors.rb:6:in `wrapped_gitaly_errors'", "lib/gitlab/git/repository.rb:568:in `rm_branch'", "app/models/repository.rb:225:in `rm_branch'", "app/services/branches/delete_service.rb:21:in `execute'", "app/services/merge_requests/merge_service.rb:104:in `after_merge'", "app/services/merge_requests/merge_service.rb:25:in `block in execute'", "app/models/merge_request.rb:1234:in `in_locked_state'", "app/services/merge_requests/merge_service.rb:23:in `execute'", "app/workers/merge_worker.rb:16:in `perform'", "lib/gitlab/with_request_store.rb:7:in `with_request_store'", "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'" ] } ```