All Jobs stuck. Error: CI job token signing key is not set
Hello Together
I've updated Gitlab and Gitlab Runner this night on the latest version 17.7.0. Since then, all pipelines remain in "running" state until they break due to a timeout.
I've tried to re regsiter the runners but without success. I have looked into the changes and suspect this change to cause the issue: !167938 (diffs)
Unfortuantely, i'm unable to find a solutoion.
In the runner logs i see the following entries:
WARNING: Appending trace to coordinator... failed code=500 job=28557 job-log= job-status= runner=t1_Kpc4Xo sent-log=0-1437 status=500 Internal Server Error update-interval=0s
On the gitlab server i see the log entry:
/var/log/gitlab/gitlab-rails/api_json.log <==
{
"time": "2024-12-20T13:52:41.664Z",
"severity": "INFO",
"duration_s": 0.01189,
"db_duration_s": 0.00221,
"view_duration_s": 0.00968,
"status": 500,
"method": "PATCH",
"path": "/api/v4/jobs/28558/trace",
"params": [{ "key": "debug_trace", "value": "[FILTERED]" }],
"host": "**REDACTED**",
"remote_ip": "**REDACTED**, 127.0.0.1",
"ua": "gitlab-runner 17.7.0 (17-7-stable; go1.23.2; linux/amd64)",
"route": "/api/:version/jobs/:id/trace",
"exception.class": "RuntimeError",
"exception.message": "CI job token signing key is not set",
"exception.backtrace": [
"lib/ci/job_token/jwt.rb:66:in `key'",
"lib/ci/job_token/jwt.rb:41:in `decode'",
"app/models/ci/build.rb:776:in `valid_token?'",
"lib/api/ci/helpers/runner.rb:81:in `authenticate_job!'",
"lib/api/ci/runner.rb:278:in `block (2 levels) in <class:Runner>'",
"grape (2.0.0) lib/grape/endpoint.rb:58:in `call'",
"grape (2.0.0) lib/grape/endpoint.rb:58:in `block (2 levels) in generate_api_method'",
"activesupport (7.0.8.6) lib/active_support/notifications.rb:208:in `instrument'",
"grape (2.0.0) lib/grape/endpoint.rb:57:in `block in generate_api_method'",
"grape (2.0.0) lib/grape/endpoint.rb:328:in `execute'",
"grape (2.0.0) lib/grape/endpoint.rb:260:in `block in run'",
"activesupport (7.0.8.6) lib/active_support/notifications.rb:208:in `instrument'",
"grape (2.0.0) lib/grape/endpoint.rb:240:in `run'",
"grape (2.0.0) lib/grape/endpoint.rb:316:in `block in build_stack'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"ee/lib/gitlab/middleware/ip_restrictor.rb:14:in `block in call'",
"lib/gitlab/ip_address_state.rb:11:in `with'",
"ee/lib/gitlab/middleware/ip_restrictor.rb:13:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"lib/api/api_guard.rb:219:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/resource.rb:20:in `_call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/resource/bearer.rb:8:in `_call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/abstract/handler.rb:17:in `call'",
"grape (2.0.0) lib/grape/middleware/error.rb:39:in `block in call!'",
"grape (2.0.0) lib/grape/middleware/error.rb:38:in `catch'",
"grape (2.0.0) lib/grape/middleware/error.rb:38:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"grape_logging (1.8.4) lib/grape_logging/middleware/request_logger.rb:60:in `block in call!'",
"grape_logging (1.8.4) lib/grape_logging/middleware/request_logger.rb:58:in `catch'",
"grape_logging (1.8.4) lib/grape_logging/middleware/request_logger.rb:58:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"rack (2.2.10) lib/rack/head.rb:12:in `call'",
"grape (2.0.0) lib/grape/endpoint.rb:224:in `call!'",
"grape (2.0.0) lib/grape/endpoint.rb:218:in `call'",
"grape (2.0.0) lib/grape/router/route.rb:58:in `exec'",
"grape (2.0.0) lib/grape/router.rb:120:in `process_route'",
"grape (2.0.0) lib/grape/router.rb:74:in `block in identity'",
"grape (2.0.0) lib/grape/router.rb:94:in `transaction'",
"grape (2.0.0) lib/grape/router.rb:72:in `identity'",
"grape (2.0.0) lib/grape/router.rb:56:in `block in call'",
"grape (2.0.0) lib/grape/router.rb:136:in `with_optimization'",
"grape (2.0.0) lib/grape/router.rb:55:in `call'",
"grape (2.0.0) lib/grape/api/instance.rb:165:in `call'",
"grape (2.0.0) lib/grape/api/instance.rb:70:in `call!'",
"grape (2.0.0) lib/grape/api/instance.rb:65:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/routing/mapper.rb:19:in `block in <class:Constraints>'",
"actionpack (7.0.8.6) lib/action_dispatch/routing/mapper.rb:48:in `serve'",
"actionpack (7.0.8.6) lib/action_dispatch/journey/router.rb:50:in `block in serve'",
"actionpack (7.0.8.6) lib/action_dispatch/journey/router.rb:32:in `each'",
"actionpack (7.0.8.6) lib/action_dispatch/journey/router.rb:32:in `serve'",
"actionpack (7.0.8.6) lib/action_dispatch/routing/route_set.rb:852:in `call'",
"gitlab-experiment (0.9.1) lib/gitlab/experiment/middleware.rb:19:in `call'",
"omniauth (2.1.0) lib/omniauth/strategy.rb:202:in `call!'",
"omniauth (2.1.0) lib/omniauth/strategy.rb:169:in `call'",
"flipper (0.26.2) lib/flipper/middleware/memoizer.rb:72:in `memoized_call'",
"flipper (0.26.2) lib/flipper/middleware/memoizer.rb:37:in `call'",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `block in call'",
"lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
"lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/go.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:83:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/ci/job_token/middleware.rb:11:in `call'",
"batch-loader (2.0.5) lib/batch_loader/middleware.rb:11:in `call'",
"rack-attack (6.7.0) lib/rack/attack.rb:103:in `call'",
"apollo_upload_server (2.1.6) lib/apollo_upload_server/middleware.rb:19:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"rack-attack (6.7.0) lib/rack/attack.rb:127:in `call'",
"warden (1.2.9) lib/warden/manager.rb:36:in `block in call'",
"warden (1.2.9) lib/warden/manager.rb:34:in `catch'",
"warden (1.2.9) lib/warden/manager.rb:34:in `call'",
"rack-cors (2.0.2) lib/rack/cors.rb:102:in `call'",
"rack (2.2.10) lib/rack/tempfile_reaper.rb:15:in `call'",
"rack (2.2.10) lib/rack/etag.rb:27:in `call'",
"rack (2.2.10) lib/rack/conditional_get.rb:40:in `call'",
"rack (2.2.10) lib/rack/head.rb:12:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/http/permissions_policy.rb:38:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/http/content_security_policy.rb:36:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/unauthenticated_session_expiry.rb:18:in `call'",
"rack (2.2.10) lib/rack/session/abstract/id.rb:266:in `context'",
"rack (2.2.10) lib/rack/session/abstract/id.rb:260:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/cookies.rb:704:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/callbacks.rb:27:in `block in call'",
"activesupport (7.0.8.6) lib/active_support/callbacks.rb:99:in `run_callbacks'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/callbacks.rb:26:in `call'",
"sentry-rails (5.21.0) lib/sentry/rails/rescued_exception_interceptor.rb:14:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/debug_exceptions.rb:28:in `call'",
"lib/gitlab/middleware/path_traversal_check.rb:35:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"sentry-ruby (5.21.0) lib/sentry/rack/capture_exceptions.rb:30:in `block (2 levels) in call'",
"sentry-ruby (5.21.0) lib/sentry/hub.rb:265:in `with_session_tracking'",
"sentry-ruby (5.21.0) lib/sentry-ruby.rb:412:in `with_session_tracking'",
"sentry-ruby (5.21.0) lib/sentry/rack/capture_exceptions.rb:21:in `block in call'",
"sentry-ruby (5.21.0) lib/sentry/hub.rb:59:in `with_scope'",
"sentry-ruby (5.21.0) lib/sentry-ruby.rb:392:in `with_scope'",
"sentry-ruby (5.21.0) lib/sentry/rack/capture_exceptions.rb:20:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/show_exceptions.rb:29:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lograge (0.11.2) lib/lograge/rails_ext/rack/logger.rb:15:in `call_app'",
"railties (7.0.8.6) lib/rails/rack/logger.rb:25:in `block in call'",
"activesupport (7.0.8.6) lib/active_support/tagged_logging.rb:99:in `block in tagged'",
"activesupport (7.0.8.6) lib/active_support/tagged_logging.rb:37:in `tagged'",
"activesupport (7.0.8.6) lib/active_support/tagged_logging.rb:99:in `tagged'",
"railties (7.0.8.6) lib/rails/rack/logger.rb:25:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/remote_ip.rb:93:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:15:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"request_store (1.5.1) lib/request_store/middleware.rb:19:in `call'",
"rack (2.2.10) lib/rack/method_override.rb:24:in `call'",
"rack (2.2.10) lib/rack/runtime.rb:22:in `call'",
"rack-timeout (0.7.0) lib/rack/timeout/core.rb:154:in `block in call'",
"rack-timeout (0.7.0) lib/rack/timeout/support/timeout.rb:19:in `timeout'",
"rack-timeout (0.7.0) lib/rack/timeout/core.rb:153:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:44:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/executor.rb:14:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"rack (2.2.10) lib/rack/sendfile.rb:110:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call'",
"gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:22:in `block in call'",
"gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
"gitlab-labkit (0.37.0) lib/labkit/middleware/rack.rb:21:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/request_id.rb:26:in `call'",
"actionpack (7.0.8.6) lib/action_dispatch/middleware/host_authorization.rb:131:in `call'",
"railties (7.0.8.6) lib/rails/engine.rb:530:in `call'",
"railties (7.0.8.6) lib/rails/railtie.rb:226:in `public_send'",
"railties (7.0.8.6) lib/rails/railtie.rb:226:in `method_missing'",
"lib/gitlab/middleware/release_env.rb:12:in `call'",
"rack (2.2.10) lib/rack/urlmap.rb:74:in `block in call'",
"rack (2.2.10) lib/rack/urlmap.rb:58:in `each'",
"rack (2.2.10) lib/rack/urlmap.rb:58:in `call'",
"puma (6.5.0) lib/puma/configuration.rb:279:in `call'",
"puma (6.5.0) lib/puma/request.rb:99:in `block in handle_request'",
"puma (6.5.0) lib/puma/thread_pool.rb:389:in `with_force_shutdown'",
"puma (6.5.0) lib/puma/request.rb:98:in `handle_request'",
"puma (6.5.0) lib/puma/server.rb:468:in `process_client'",
"puma (6.5.0) lib/puma/server.rb:249:in `block in run'",
"puma (6.5.0) lib/puma/thread_pool.rb:166:in `block in spawn_thread'"
],
"queue_duration_s": 0.005503,
"redis_calls": 3,
"redis_duration_s": 0.000276,
"redis_write_bytes": 168,
"redis_db_load_balancing_calls": 3,
"redis_db_load_balancing_duration_s": 0.000276,
"redis_db_load_balancing_write_bytes": 168,
"db_count": 4,
"db_write_count": 0,
"db_cached_count": 0,
"db_txn_count": 0,
"db_replica_txn_count": 0,
"db_primary_txn_count": 0,
"db_replica_count": 0,
"db_primary_count": 4,
"db_replica_write_count": 0,
"db_primary_write_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_replica_txn_max_duration_s": 0.0,
"db_primary_txn_max_duration_s": 0.0,
"db_replica_txn_duration_s": 0.0,
"db_primary_txn_duration_s": 0.0,
"db_replica_duration_s": 0.0,
"db_primary_duration_s": 0.002,
"db_main_txn_count": 0,
"db_ci_txn_count": 0,
"db_main_replica_txn_count": 0,
"db_ci_replica_txn_count": 0,
"db_main_count": 3,
"db_ci_count": 1,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_main_write_count": 0,
"db_ci_write_count": 0,
"db_main_replica_write_count": 0,
"db_ci_replica_write_count": 0,
"db_main_cached_count": 0,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_main_txn_max_duration_s": 0.0,
"db_ci_txn_max_duration_s": 0.0,
"db_main_replica_txn_max_duration_s": 0.0,
"db_ci_replica_txn_max_duration_s": 0.0,
"db_main_txn_duration_s": 0.0,
"db_ci_txn_duration_s": 0.0,
"db_main_replica_txn_duration_s": 0.0,
"db_ci_replica_txn_duration_s": 0.0,
"db_main_duration_s": 0.001,
"db_ci_duration_s": 0.001,
"db_main_replica_duration_s": 0.0,
"db_ci_replica_duration_s": 0.0,
"cpu_s": 0.016128,
"mem_objects": 11037,
"mem_bytes": 932952,
"mem_mallocs": 2995,
"mem_total_bytes": 1374432,
"pid": 301851,
"worker_id": "puma_1",
"rate_limiting_gates": [],
"correlation_id": "01JFJ5F1NCPFV7NPCAM0QY7FW5",
"meta.caller_id": "PATCH /api/:version/jobs/:id/trace",
"meta.remote_ip": "**REDACTED**",
"meta.feature_category": "continuous_integration",
"meta.project": "internal-tools/gitlab-landing-page",
"meta.root_namespace": "internal-tools",
"meta.client_id": "user/16",
"meta.pipeline_id": 10236,
"meta.job_id": 28558,
"content_length": "1438",
"content_range": "0-1437",
"request_urgency": "low",
"target_duration_s": 5
}System:
- Alma Linux 8.10
- All latest updates installed
- Gitlab version 17.7.0
- Gitlab Runner version 17.7.0
Thanks a lot for having a look.
Best regard and happy holiday
Flurin Feuerstein