Missing Sidekiq logs for MergeWorker and error_backtrace
I went looking for logs in #213620 (comment 318833488), and while I found this JSON entry in the sidekiq-cluster logs directly, it seems that any entry with error_backtrace
and MergeWorker
is coming up empty in https://log.gprd.gitlab.net/goto/37b8d01efeee0fc997858ca6857b05d1.
It doesn't help that we've turned off FluentD logging because of the error spew (https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/9530#note_312626014):
2020-03-20 20:23:41 +0000 [warn]: #0 Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory
2020-03-20 20:23:41 +0000 [warn]: #0 Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory
This was the missing entry:
{
"severity": "WARN",
"time": "2020-04-06T23:46:45.928Z",
"error_class": "Gitlab::Git::CommandError",
"error_message": "9:Could not update normalize-import-error-messages. Please refresh and try again.",
"context": "Job raised exception",
"jobstr": "{\"class\":\"MergeWorker\",\"args\":[54646713,33100,{\"force_remove_source_branch\":\"1\",\"auto_merge_strategy\":\"merge_when_pipeline_succeeds\",\"should_remove_source_branch\":true,\"commit_message\":\"Merge branch 'normalize-import-error-messages' into 'master'\\n\\nNormalize error message between Gitea and Fogbugz importers\\n\\nSee merge request gitlab-org/gitlab!28802\",\"sha\":\"5342d1a7f5f3eb5edb1de40d560b5c3bdce42224\"}],\"retry\":3,\"queue\":\"merge\",\"jid\":\"94ec160f911e3c63b34939d1\",\"created_at\":1586216690.995178,\"meta.user\":\"mayra-cabrera\",\"meta.project\":\"gitlab-org/gitlab\",\"meta.root_namespace\":\"gitlab-org\",\"meta.subscription_plan\":\"gold\",\"meta.caller_id\":\"AutoMergeProcessWorker\",\"correlation_id\":\"67899bd2eb7a00d4986728e2156ca42c\",\"enqueued_at\":1586216784.5158522,\"error_message\":\"9:Could not update normalize-import-error-messages. Please refresh and try again.\",\"error_class\":\"Gitlab::Git::CommandError\",\"failed_at\":1586216705.6987324,\"retry_count\":1,\"retried_at\":1586216737.6654246}",
"class": "MergeWorker",
"args": [
54646713,
33100,
{
"force_remove_source_branch": "1",
"auto_merge_strategy": "merge_when_pipeline_succeeds",
"should_remove_source_branch": true,
"commit_message": "Merge branch 'normalize-import-error-messages' into 'master'\n\nNormalize error message between Gitea and Fogbugz importers\n\nSee merge request gitlab-org/gitlab!28802",
"sha": "5342d1a7f5f3eb5edb1de40d560b5c3bdce42224"
}
],
"retry": 3,
"queue": "merge",
"jid": "94ec160f911e3c63b34939d1",
"created_at": "2020-04-06T23:44:50.995Z",
"meta.user": "mayra-cabrera",
"meta.project": "gitlab-org/gitlab",
"meta.root_namespace": "gitlab-org",
"meta.subscription_plan": "gold",
"meta.caller_id": "AutoMergeProcessWorker",
"correlation_id": "67899bd2eb7a00d4986728e2156ca42c",
"enqueued_at": "2020-04-06T23:46:24.515Z",
"failed_at": "2020-04-06T23:45:05.698Z",
"retry_count": 1,
"retried_at": "2020-04-06T23:45:37.665Z",
"error_backtrace": [
"lib/gitlab/git/wraps_gitaly_errors.rb:13:in `rescue in wrapped_gitaly_errors'",
"lib/gitlab/git/wraps_gitaly_errors.rb:6:in `wrapped_gitaly_errors'",
"lib/gitlab/git/repository.rb:568:in `rm_branch'",
"app/models/repository.rb:225:in `rm_branch'",
"app/services/branches/delete_service.rb:21:in `execute'",
"app/services/merge_requests/merge_service.rb:104:in `after_merge'",
"app/services/merge_requests/merge_service.rb:25:in `block in execute'",
"app/models/merge_request.rb:1234:in `in_locked_state'",
"app/services/merge_requests/merge_service.rb:23:in `execute'",
"app/workers/merge_worker.rb:16:in `perform'",
"lib/gitlab/with_request_store.rb:7:in `with_request_store'",
"lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'"
]
}
Edited by Stan Hu