Skip to content

StuckCiJobsWorker always times out

StuckCiJobsWorker runs every hour to look for stuck jobs, but it is no longer working as the search always times out (https://log.gprd.gitlab.net/goto/4560a2ec7ace2b31f12b13ac36760eec):

image

    "json.db_count": 1,
    "json.db_write_count": 0,
    "json.db_cached_count": 0,
    "json.error_backtrace": [
      "app/workers/stuck_ci_jobs_worker.rb:61:in `block in search'",
      "app/workers/stuck_ci_jobs_worker.rb:56:in `loop'",
      "app/workers/stuck_ci_jobs_worker.rb:56:in `search'",
      "app/workers/stuck_ci_jobs_worker.rb:41:in `drop'",
      "app/workers/stuck_ci_jobs_worker.rb:22:in `perform'",
      "lib/gitlab/metrics/sidekiq_middleware.rb:18:in `block in call'",
      "lib/gitlab/metrics/transaction.rb:56:in `run'",
      "lib/gitlab/metrics/sidekiq_middleware.rb:18:in `call'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:40:in `perform'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
      "lib/gitlab/application_context.rb:54:in `block in use'",
      "lib/gitlab/application_context.rb:54:in `use'",
      "lib/gitlab/sidekiq_middleware/worker_context.rb:11:in `wrap_in_optional_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `block in call'",
      "lib/gitlab/application_context.rb:54:in `block in use'",
      "lib/gitlab/application_context.rb:54:in `use'",
      "lib/gitlab/application_context.rb:21:in `with_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in `call'",
      "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
      "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:8:in `call'",
      "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
      "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
      "lib/gitlab/with_request_store.rb:10:in `with_request_store'",
      "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:37:in `call'",
      "lib/gitlab/sidekiq_middleware/monitor.rb:8:in `block in call'",
      "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
      "lib/gitlab/sidekiq_middleware/monitor.rb:7:in `call'",
      "lib/gitlab/sidekiq_logging/structured_logger.rb:18:in `call'"
    ],