Error 500 while submitting review with 192 comments on a merge request touching 210 files
Gitlab version: self-hosted GitLab Community Edition v16.4.1
I have worked for the past 3 months on an epic feature. The merge request is touching 210 files. I just spent 4 hours making a first code review which resulted in 192 comments.
When I hit the Submit review button:
- The
Submit reviewbutton is greyed-out - A POST request is made on
/group/project/-/merge_requests/939/drafts/publishwith following data:
{
"noteable_type": "MergeRequest",
"noteable_id": 942,
"note": "",
"approve": false,
"approval_password": ""
}- The
Overviewcounter gradually increments - After a while I get a 500 HTTP status code response on the POST request
- The
Submit reviewbutton comes back to clickable again - ¯\_(ツ)_/¯
If I hit the Submit review again, of course it does the same thing which is create new comments on the PR but not actually removing the pending comments. Which resulted in this:
Here are some logs connected to this issue:
{
"method": "POST",
"path": "/group/project/-/merge_requests/939/drafts/publish",
"format": "json",
"controller": "Projects::MergeRequests::DraftsController",
"action": "publish",
"status": 500,
"time": "2023-10-06T12:14:50.065Z",
"params": [
{
"key": "noteable_type",
"value": "MergeRequest"
},
{
"key": "noteable_id",
"value": 942
},
{
"key": "note",
"value": "[FILTERED]"
},
{
"key": "approve",
"value": false
},
{
"key": "approval_password",
"value": "[FILTERED]"
},
{
"key": "namespace_id",
"value": "group"
},
{
"key": "project_id",
"value": "project"
},
{
"key": "merge_request_id",
"value": "939"
},
{
"key": "draft",
"value": {
"noteable_type": "MergeRequest",
"noteable_id": 942,
"note": "[FILTERED]",
"approve": false,
"approval_password": "[FILTERED]"
}
}
],
"correlation_id": "01HC2EH1PKXJ90Y85TP01WXEZE",
"meta.caller_id": "Projects::MergeRequests::DraftsController#publish",
"meta.remote_ip": "XXX",
"meta.feature_category": "code_review_workflow",
"meta.client_id": "ip/XXX",
"remote_ip": "XXX",
"user_id": 2,
"username": "XXX",
"ua": "XXX",
"request_urgency": "low",
"target_duration_s": 5,
"gitaly_calls": 34,
"gitaly_duration_s": 1.141965,
"redis_calls": 862,
"redis_allowed_cross_slot_calls": 1,
"redis_duration_s": 5.790807,
"redis_read_bytes": 5803,
"redis_write_bytes": 100073,
"redis_cache_calls": 199,
"redis_cache_duration_s": 1.972065,
"redis_cache_read_bytes": 248,
"redis_cache_write_bytes": 32432,
"redis_feature_flag_calls": 18,
"redis_feature_flag_duration_s": 0.003248,
"redis_feature_flag_read_bytes": 3142,
"redis_feature_flag_write_bytes": 1378,
"redis_queues_calls": 147,
"redis_queues_duration_s": 1.797569,
"redis_queues_read_bytes": 196,
"redis_queues_write_bytes": 45641,
"redis_repository_cache_calls": 1,
"redis_repository_cache_duration_s": 0.000207,
"redis_repository_cache_read_bytes": 100,
"redis_repository_cache_write_bytes": 41,
"redis_sessions_calls": 3,
"redis_sessions_allowed_cross_slot_calls": 1,
"redis_sessions_duration_s": 0.00062,
"redis_sessions_read_bytes": 300,
"redis_sessions_write_bytes": 592,
"redis_shared_state_calls": 444,
"redis_shared_state_duration_s": 1.166734,
"redis_shared_state_read_bytes": 1766,
"redis_shared_state_write_bytes": 15956,
"redis_action_cable_calls": 50,
"redis_action_cable_duration_s": 0.850364,
"redis_action_cable_read_bytes": 51,
"redis_action_cable_write_bytes": 4033,
"db_count": 1079,
"db_write_count": 203,
"db_cached_count": 102,
"db_replica_count": 0,
"db_primary_count": 1079,
"db_main_count": 1079,
"db_ci_count": 0,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 102,
"db_main_cached_count": 102,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_replica_duration_s": 0,
"db_primary_duration_s": 40.202,
"db_main_duration_s": 40.202,
"db_ci_duration_s": 0,
"db_main_replica_duration_s": 0,
"db_ci_replica_duration_s": 0,
"cpu_s": 3.651099,
"mem_objects": 1606166,
"mem_bytes": 291611440,
"mem_mallocs": 781483,
"mem_total_bytes": 355858080,
"pid": 1222,
"worker_id": "puma_0",
"rate_limiting_gates": [],
"net_ldap_count": 3,
"net_ldap_duration_s": 0.019758995999931358,
"exception.class": "Rack::Timeout::RequestTimeoutException",
"exception.message": "Request ran for longer than 60000ms ",
"exception.backtrace": [
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:137:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:224:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:126:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:61:in `block (2 levels) in <class:ConnectionProxy>'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:137:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:224:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:126:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"app/models/concerns/cross_database_modification.rb:92:in `block in transaction'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:137:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:224:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:126:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"lib/gitlab/database.rb:359:in `block in transaction'",
"lib/gitlab/database.rb:358:in `transaction'",
"app/models/concerns/cross_database_modification.rb:83:in `transaction'",
"app/models/diff_note.rb:57:in `create_diff_file'",
"config/initializers/forbid_sidekiq_in_transactions.rb:106:in `block in committed!'",
"config/initializers/forbid_sidekiq_in_transactions.rb:10:in `skipping_transaction_check'",
"config/initializers/forbid_sidekiq_in_transactions.rb:106:in `committed!'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:127:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:137:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:224:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:126:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:78:in `transaction'",
"app/services/notes/create_service.rb:31:in `block in execute'",
"app/services/notes/create_service.rb:62:in `execute_quick_actions'",
"app/services/notes/create_service.rb:23:in `execute'",
"app/services/draft_notes/publish_service.rb:61:in `create_note_from_draft'",
"app/services/draft_notes/publish_service.rb:37:in `block in publish_draft_notes'",
"app/services/draft_notes/publish_service.rb:35:in `map'",
"app/services/draft_notes/publish_service.rb:35:in `publish_draft_notes'",
"app/services/draft_notes/publish_service.rb:11:in `execute'",
"app/controllers/projects/merge_requests/drafts_controller.rb:56:in `publish'",
"app/controllers/application_controller.rb:520:in `set_current_admin'",
"lib/gitlab/session.rb:11:in `with_session'",
"app/controllers/application_controller.rb:511:in `set_session_storage'",
"lib/gitlab/i18n.rb:107:in `with_locale'",
"lib/gitlab/i18n.rb:113:in `with_user_locale'",
"app/controllers/application_controller.rb:502:in `set_locale'",
"app/controllers/application_controller.rb:495:in `set_current_context'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/jira/middleware.rb:19:in `call'",
"lib/gitlab/middleware/go.rb:20:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:15:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:44:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call'",
"lib/gitlab/middleware/release_env.rb:13:in `call'"
],
"db_duration_s": 48.17405,
"view_duration_s": 0,
"duration_s": 60.38823
}Has anyone any idea how to make it work?
Edited by tomcodes