Missing Sidekiq logs for MergeWorker and error_backtrace

I went looking for logs in gitlab-org/gitlab#213620 (comment 318833488), and while I found this JSON entry in the sidekiq-cluster logs directly, it seems that any entry with error_backtrace and MergeWorker is coming up empty in https://log.gprd.gitlab.net/goto/37b8d01efeee0fc997858ca6857b05d1.

It doesn't help that we've turned off FluentD logging because of the error spew (https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9530#note_312626014):

2020-03-20 20:23:41 +0000 [warn]: #0 Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory
2020-03-20 20:23:41 +0000 [warn]: #0 Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory

This was the missing entry:

{
  "severity": "WARN",
  "time": "2020-04-06T23:46:45.928Z",
  "error_class": "Gitlab::Git::CommandError",
  "error_message": "9:Could not update normalize-import-error-messages. Please refresh and try again.",
  "context": "Job raised exception",
  "jobstr": "{\"class\":\"MergeWorker\",\"args\":[54646713,33100,{\"force_remove_source_branch\":\"1\",\"auto_merge_strategy\":\"merge_when_pipeline_succeeds\",\"should_remove_source_branch\":true,\"commit_message\":\"Merge branch 'normalize-import-error-messages' into 'master'\\n\\nNormalize error message between Gitea and Fogbugz importers\\n\\nSee merge request gitlab-org/gitlab!28802\",\"sha\":\"5342d1a7f5f3eb5edb1de40d560b5c3bdce42224\"}],\"retry\":3,\"queue\":\"merge\",\"jid\":\"94ec160f911e3c63b34939d1\",\"created_at\":1586216690.995178,\"meta.user\":\"mayra-cabrera\",\"meta.project\":\"gitlab-org/gitlab\",\"meta.root_namespace\":\"gitlab-org\",\"meta.subscription_plan\":\"gold\",\"meta.caller_id\":\"AutoMergeProcessWorker\",\"correlation_id\":\"67899bd2eb7a00d4986728e2156ca42c\",\"enqueued_at\":1586216784.5158522,\"error_message\":\"9:Could not update normalize-import-error-messages. Please refresh and try again.\",\"error_class\":\"Gitlab::Git::CommandError\",\"failed_at\":1586216705.6987324,\"retry_count\":1,\"retried_at\":1586216737.6654246}",
  "class": "MergeWorker",
  "args": [
    54646713,
    33100,
    {
      "force_remove_source_branch": "1",
      "auto_merge_strategy": "merge_when_pipeline_succeeds",
      "should_remove_source_branch": true,
      "commit_message": "Merge branch 'normalize-import-error-messages' into 'master'\n\nNormalize error message between Gitea and Fogbugz importers\n\nSee merge request gitlab-org/gitlab!28802",
      "sha": "5342d1a7f5f3eb5edb1de40d560b5c3bdce42224"
    }
  ],
  "retry": 3,
  "queue": "merge",
  "jid": "94ec160f911e3c63b34939d1",
  "created_at": "2020-04-06T23:44:50.995Z",
  "meta.user": "mayra-cabrera",
  "meta.project": "gitlab-org/gitlab",
  "meta.root_namespace": "gitlab-org",
  "meta.subscription_plan": "gold",
  "meta.caller_id": "AutoMergeProcessWorker",
  "correlation_id": "67899bd2eb7a00d4986728e2156ca42c",
  "enqueued_at": "2020-04-06T23:46:24.515Z",
  "failed_at": "2020-04-06T23:45:05.698Z",
  "retry_count": 1,
  "retried_at": "2020-04-06T23:45:37.665Z",
  "error_backtrace": [
    "lib/gitlab/git/wraps_gitaly_errors.rb:13:in `rescue in wrapped_gitaly_errors'",
    "lib/gitlab/git/wraps_gitaly_errors.rb:6:in `wrapped_gitaly_errors'",
    "lib/gitlab/git/repository.rb:568:in `rm_branch'",
    "app/models/repository.rb:225:in `rm_branch'",
    "app/services/branches/delete_service.rb:21:in `execute'",
    "app/services/merge_requests/merge_service.rb:104:in `after_merge'",
    "app/services/merge_requests/merge_service.rb:25:in `block in execute'",
    "app/models/merge_request.rb:1234:in `in_locked_state'",
    "app/services/merge_requests/merge_service.rb:23:in `execute'",
    "app/workers/merge_worker.rb:16:in `perform'",
    "lib/gitlab/with_request_store.rb:7:in `with_request_store'",
    "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'"
  ]
}
Edited by Stan Hu