pipeline_hooks:build_hooks is pending and sidekiq queue is pilling up
Hi,
My Gitlab instance sidekiq queue is getting enqueued and which is not reducing/executing. Gitlab version: 12.7.6 CE
sudo gitlab-ctl tail sidekiq
==> /var/log/gitlab/sidekiq/current <==
{"severity":"INFO","time":"2020-09-15T08:41:13.085Z","class":"BuildFinishedWorker","retry":3,"queue":"pipeline_processing:build_finished","queue_namespace":"pipeline_processing","jid":"88b5a051c3a11a3f38ac710a","created_at":"2020-09-15T08:36:54.390Z","correlation_id":"s3AHlCDkCz9","enqueued_at":"2020-09-15T08:36:54.391Z","pid":46357,"message":"BuildFinishedWorker JID-88b5a051c3a11a3f38ac710a: done: 28.70172 sec","job_status":"done","scheduling_latency_s":229.992888,"duration":28.70172,"cpu_s":0.171344,"completed_at":"2020-09-15T08:41:13.085Z","db_duration":3136.717263609171,"db_duration_s":3.136717263609171}
{"severity":"INFO","time":"2020-09-15T08:41:12.978Z","class":"PipelineProcessWorker","retry":3,"queue":"pipeline_processing:pipeline_process","queue_namespace":"pipeline_processing","jid":"65352c661c8a08be27bb7f6b","created_at":"2020-09-15T08:25:07.279Z","meta.user":"tabiszew","meta.project":"next/next","meta.root_namespace":"Robots","correlation_id":"01405174-e165-4ae2-8b27-e90205b31779","enqueued_at":"2020-09-15T08:25:07.279Z","pid":46357,"message":"PipelineProcessWorker JID-65352c661c8a08be27bb7f6b: start","job_status":"start","scheduling_latency_s":965.699039}
{"severity":"INFO","time":"2020-09-15T08:41:13.074Z","class":"AutoMergeProcessWorker","retry":3,"queue":"auto_merge:auto_merge_process","queue_namespace":"auto_merge","jid":"b9aa6eb7921c8820030d59a3","created_at":"2020-09-15T08:34:26.385Z","correlation_id":"pP8xuG7Bfr1","enqueued_at":"2020-09-15T08:34:26.385Z","pid":46357,"message":"AutoMergeProcessWorker JID-b9aa6eb7921c8820030d59a3: done: 1.563886 sec","job_status":"done","scheduling_latency_s":405.124901,"duration":1.563886,"cpu_s":0.022702,"completed_at":"2020-09-15T08:41:13.074Z","db_duration":393.03520135581493,"db_duration_s":0.39303520135581493}
{"severity":"INFO","time":"2020-09-15T08:41:13.560Z","class":"RebaseWorker","retry":3,"queue":"rebase","jid":"c82e34ba4ffb42b40f0b3930","created_at":"2020-09-15T08:41:09.377Z","meta.user":"mevijaya","meta.project":"test/ouife","meta.root_namespace":"FonTech","correlation_id":"Plf0EBPsjX3","enqueued_at":"2020-09-15T08:41:09.378Z","pid":46357,"message":"RebaseWorker JID-c82e34ba4ffb42b40f0b3930: start","job_status":"start","scheduling_latency_s":4.182343}
{"severity":"INFO","time":"2020-09-15T08:41:13.622Z","class":"BuildFinishedWorker","retry":3,"queue":"pipeline_processing:build_finished","queue_namespace":"pipeline_processing","jid":"ed9c1aedce7f91cd04294ab7","created_at":"2020-09-15T08:38:06.755Z","correlation_id":"Ra7xP8cOqk3","enqueued_at":"2020-09-15T08:38:06.756Z","pid":46357,"message":"BuildFinishedWorker JID-ed9c1aedce7f91cd04294ab7: start","job_status":"start","scheduling_latency_s":186.866067}
{"severity":"INFO","time":"2020-09-15T08:41:13.622Z","class":"Namespaces::ScheduleAggregationWorker","retry":3,"queue":"update_namespace_statistics:namespaces_schedule_aggregation","queue_namespace":"update_namespace_statistics","jid":"c397705d858f4b5ea7b2eb34","created_at":"2020-09-15T07:45:38.219Z","correlation_id":"jakOUsUqR61","enqueued_at":"2020-09-15T07:45:38.231Z","pid":46357,"message":"Namespaces::ScheduleAggregationWorker JID-c397705d858f4b5ea7b2eb34: start","job_status":"start","scheduling_latency_s":3335.391747}
{"severity":"INFO","time":"2020-09-15T08:41:13.624Z","class":"BuildSuccessWorker","retry":3,"queue":"pipeline_processing:build_success","queue_namespace":"pipeline_processing","jid":"b3e07751e18599f0c298dee0","created_at":"2020-09-15T08:41:09.721Z","correlation_id":"HFm74tfqmh9","enqueued_at":"2020-09-15T08:41:09.730Z","pid":46357,"message":"BuildSuccessWorker JID-b3e07751e18599f0c298dee0: start","job_status":"start","scheduling_latency_s":3.894513}
{"severity":"INFO","time":"2020-09-15T08:41:13.775Z","class":"PipelineUpdateWorker","retry":3,"queue":"pipeline_processing:pipeline_update","queue_namespace":"pipeline_processing","jid":"b624921b79f7b20d3c9bfc83","created_at":"2020-09-15T08:04:21.792Z","correlation_id":"pbvPjNYUvN7","enqueued_at":"2020-09-15T08:04:21.792Z","pid":46357,"message":"PipelineUpdateWorker JID-b624921b79f7b20d3c9bfc83: done: 1.316529 sec","job_status":"done","scheduling_latency_s":2210.666713,"duration":1.316529,"cpu_s":0.020624,"completed_at":"2020-09-15T08:41:13.775Z","db_duration":527.294447645545,"db_duration_s":0.527294447645545}
{"severity":"INFO","time":"2020-09-15T08:41:13.598Z","class":"ExpireJobCacheWorker","retry":3,"queue":"pipeline_cache:expire_job_cache","queue_namespace":"pipeline_cache","jid":"f93e454056e8a3fee175b308","created_at":"2020-09-15T07:41:36.790Z","correlation_id":"K2Vfjs3PTc5","enqueued_at":"2020-09-15T07:41:36.790Z","pid":46357,"message":"ExpireJobCacheWorker JID-f93e454056e8a3fee175b308: start","job_status":"start","scheduling_latency_s":3576.807595}
{"severity":"INFO","time":"2020-09-15T08:41:13.852Z","class":"BuildHooksWorker","retry":3,"queue":"pipeline_hooks:build_hooks","queue_namespace":"pipeline_hooks","jid":"7e8f231a11d2308eccdcc376","created_at":"2020-09-15T07:27:21.713Z","correlation_id":"4so3Enu1jA","enqueued_at":"2020-09-15T07:27:21.713Z","pid":46357,"message":"BuildHooksWorker JID-7e8f231a11d2308eccdcc376: start","job_status":"start","scheduling_latency_s":4432.138321}
Edited by Grzegorz Bizon