CI: Backup and restore spec fails
Summary
Our backup and restore spec fails during project setup:
Failures:
1) Backup and restore Create the backup Creates the backup
Failure/Error: response = RestClient::Request.execute(default_args)
RestClient::InternalServerError:
500 Internal Server Error
# ./gems/ruby/3.3.0/gems/rest-client-2.1.0/lib/restclient/abstract_response.rb:249:in `exception_with_response'
# ./gems/ruby/3.3.0/gems/rest-client-2.1.0/lib/restclient/abstract_response.rb:129:in `return!'
# ./gems/ruby/3.3.0/gems/rest-client-2.1.0/lib/restclient/request.rb:836:in `process_result'
# ./gems/ruby/3.3.0/gems/rest-client-2.1.0/lib/restclient/request.rb:743:in `block in transmit'
# ./gems/ruby/3.3.0/gems/rest-client-2.1.0/lib/restclient/request.rb:727:in `transmit'
# ./gems/ruby/3.3.0/gems/rest-client-2.1.0/lib/restclient/request.rb:163:in `execute'
# ./gems/ruby/3.3.0/gems/rest-client-2.1.0/lib/restclient/request.rb:63:in `execute'https://gitlab.com/gitlab-org/charts/gitlab/-/jobs/10769892250
This happens when we attempt to commit a new file. Looking into the puma logs, it seems the creation works fine, but something goes wrong during the event tracking.
Steps to reproduce
- Create a fresh project.
- Commit something via API.
Relevant logs
Log/stacktrace
{
"component": "gitlab",
"subcomponent": "api_json",
"time": "2025-07-22T12:16:51.633Z",
"severity": "INFO",
"duration_s": 1.21292,
"db_duration_s": 0.04233,
"view_duration_s": 1.17059,
"status": 500,
"method": "POST",
"path": "/api/v4/projects/root%2Ftestproject-2025-07-22-12-16-40/repository/commits",
"params": [
{
"key": "branch",
"value": "main"
},
{
"key": "commit_message",
"value": "Test Dockerfile"
},
{
"key": "actions",
"value": [
{
"action": "create",
"file_path": "Dockerfile",
"content": "[FILTERED]",
"encoding": "text"
}
]
}
],
"host": "gitlab-gke133-review-193-q56utf.cloud-native-v133.helm-charts.win",
"remote_ip": "REDACTED",
"ua": "rest-client/2.1.0 (linux x86_64) ruby/3.2.6p234",
"route": "/api/:version/projects/:id/repository/commits",
"user_id": 1,
"username": "root",
"token_type": "PersonalAccessToken",
"token_id": 2,
"exception.class": "Gitlab::Auth::UnauthorizedError",
"exception.message": "Gitlab::Auth::UnauthorizedError",
"exception.backtrace": [
"lib/gitlab/auth/auth_finders.rb:330:in `find_oauth_access_token'",
"ee/lib/ee/gitlab/auth/auth_finders.rb:37:in `find_oauth_access_token'",
"lib/api/commits.rb:38:in `track_commit_events'",
"lib/api/commits.rb:259:in `block (2 levels) in <class:Commits>'",
"grape (2.0.0) lib/grape/endpoint.rb:58:in `call'",
"grape (2.0.0) lib/grape/endpoint.rb:58:in `block (2 levels) in generate_api_method'",
"activesupport (7.1.5.1) lib/active_support/notifications.rb:208:in `instrument'",
"grape (2.0.0) lib/grape/endpoint.rb:57:in `block in generate_api_method'",
"grape (2.0.0) lib/grape/endpoint.rb:328:in `execute'",
"grape (2.0.0) lib/grape/endpoint.rb:260:in `block in run'",
"activesupport (7.1.5.1) lib/active_support/notifications.rb:208:in `instrument'",
"grape (2.0.0) lib/grape/endpoint.rb:240:in `run'",
"grape (2.0.0) lib/grape/endpoint.rb:316:in `block in build_stack'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"lib/api/api_guard.rb:266:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/resource.rb:20:in `_call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/resource/bearer.rb:8:in `_call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/abstract/handler.rb:17:in `call'",
"lib/gitlab/middleware/ip_address.rb:14:in `block in call'",
"lib/gitlab/ip_address_state.rb:11:in `with'",
"lib/gitlab/middleware/ip_address.rb:13:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"lib/api/api_guard.rb:266:in `call'",
"grape (2.0.0) lib/grape/middleware/base.rb:36:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/resource.rb:20:in `_call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/resource/bearer.rb:8:in `_call'",
"rack-oauth2 (2.2.1) lib/rack/oauth2/server/abstract/handler.rb:17:in `call'",
"grape (2.0.0) lib/grape/middleware/error.rb:39:in `block in call!'",
"grape (2.0.0) lib/grape/middleware/error.rb:38:in `catch'",
"grape (2.0.0) lib/grape/middleware/error.rb:38:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"grape_logging (1.8.4) lib/grape_logging/middleware/request_logger.rb:60:in `block in call!'",
"grape_logging (1.8.4) lib/grape_logging/middleware/request_logger.rb:58:in `catch'",
"grape_logging (1.8.4) lib/grape_logging/middleware/request_logger.rb:58:in `call!'",
"grape (2.0.0) lib/grape/middleware/base.rb:29:in `call'",
"rack (2.2.17) lib/rack/head.rb:12:in `call'",
"grape (2.0.0) lib/grape/endpoint.rb:224:in `call!'",
"grape (2.0.0) lib/grape/endpoint.rb:218:in `call'",
"grape (2.0.0) lib/grape/router/route.rb:58:in `exec'",
"grape (2.0.0) lib/grape/router.rb:120:in `process_route'",
"grape (2.0.0) lib/grape/router.rb:74:in `block in identity'",
"grape (2.0.0) lib/grape/router.rb:94:in `transaction'",
"grape (2.0.0) lib/grape/router.rb:72:in `identity'",
"grape (2.0.0) lib/grape/router.rb:56:in `block in call'",
"grape (2.0.0) lib/grape/router.rb:136:in `with_optimization'",
"grape (2.0.0) lib/grape/router.rb:55:in `call'",
"grape (2.0.0) lib/grape/api/instance.rb:165:in `call'",
"grape (2.0.0) lib/grape/api/instance.rb:70:in `call!'",
"grape (2.0.0) lib/grape/api/instance.rb:65:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/routing/mapper.rb:31:in `block in <class:Constraints>'",
"actionpack (7.1.5.1) lib/action_dispatch/routing/mapper.rb:60:in `serve'",
"actionpack (7.1.5.1) lib/action_dispatch/journey/router.rb:51:in `block in serve'",
"config/initializers/action_dispatch_journey_router.rb:52:in `block in find_routes'",
"config/initializers/action_dispatch_journey_router.rb:25:in `map!'",
"config/initializers/action_dispatch_journey_router.rb:25:in `find_routes'",
"actionpack (7.1.5.1) lib/action_dispatch/journey/router.rb:32:in `serve'",
"actionpack (7.1.5.1) lib/action_dispatch/routing/route_set.rb:882:in `call'",
"gitlab-experiment (0.9.1) lib/gitlab/experiment/middleware.rb:19:in `call'",
"flipper (0.28.3) lib/flipper/middleware/memoizer.rb:72:in `memoized_call'",
"flipper (0.28.3) lib/flipper/middleware/memoizer.rb:37:in `call'",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `block in call'",
"lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
"lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/go.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:83:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/ci/job_token/middleware.rb:11:in `call'",
"batch-loader (2.0.5) lib/batch_loader/middleware.rb:11:in `call'",
"rack-attack (6.7.0) lib/rack/attack.rb:103:in `call'",
"apollo_upload_server (2.1.6) lib/apollo_upload_server/middleware.rb:19:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"rack-attack (6.7.0) lib/rack/attack.rb:127:in `call'",
"warden (1.2.9) lib/warden/manager.rb:36:in `block in call'",
"warden (1.2.9) lib/warden/manager.rb:34:in `catch'",
"warden (1.2.9) lib/warden/manager.rb:34:in `call'",
"rack-cors (2.0.2) lib/rack/cors.rb:102:in `call'",
"rack (2.2.17) lib/rack/tempfile_reaper.rb:15:in `call'",
"rack (2.2.17) lib/rack/etag.rb:27:in `call'",
"rack (2.2.17) lib/rack/conditional_get.rb:40:in `call'",
"rack (2.2.17) lib/rack/head.rb:12:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/http/permissions_policy.rb:36:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/http/content_security_policy.rb:36:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/unauthenticated_session_expiry.rb:18:in `call'",
"rack (2.2.17) lib/rack/session/abstract/id.rb:266:in `context'",
"rack (2.2.17) lib/rack/session/abstract/id.rb:260:in `call'",
"lib/gitlab/middleware/secure_headers.rb:11:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/cookies.rb:689:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/callbacks.rb:29:in `block in call'",
"activesupport (7.1.5.1) lib/active_support/callbacks.rb:101:in `run_callbacks'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/callbacks.rb:28:in `call'",
"sentry-rails (5.23.0) lib/sentry/rails/rescued_exception_interceptor.rb:14:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/debug_exceptions.rb:29:in `call'",
"lib/gitlab/middleware/path_traversal_check.rb:40:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"sentry-ruby (5.23.0) lib/sentry/rack/capture_exceptions.rb:30:in `block (2 levels) in call'",
"sentry-ruby (5.23.0) lib/sentry/hub.rb:299:in `with_session_tracking'",
"sentry-ruby (5.23.0) lib/sentry-ruby.rb:428:in `with_session_tracking'",
"sentry-ruby (5.23.0) lib/sentry/rack/capture_exceptions.rb:21:in `block in call'",
"sentry-ruby (5.23.0) lib/sentry/hub.rb:89:in `with_scope'",
"sentry-ruby (5.23.0) lib/sentry-ruby.rb:408:in `with_scope'",
"sentry-ruby (5.23.0) lib/sentry/rack/capture_exceptions.rb:20:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/show_exceptions.rb:31:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lograge (0.11.2) lib/lograge/rails_ext/rack/logger.rb:15:in `call_app'",
"railties (7.1.5.1) lib/rails/rack/logger.rb:24:in `block in call'",
"activesupport (7.1.5.1) lib/active_support/tagged_logging.rb:139:in `block in tagged'",
"activesupport (7.1.5.1) lib/active_support/tagged_logging.rb:39:in `tagged'",
"activesupport (7.1.5.1) lib/active_support/tagged_logging.rb:139:in `tagged'",
"activesupport (7.1.5.1) lib/active_support/broadcast_logger.rb:241:in `method_missing'",
"railties (7.1.5.1) lib/rails/rack/logger.rb:24:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/remote_ip.rb:92:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:15:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"request_store (1.7.0) lib/request_store/middleware.rb:19:in `call'",
"rack (2.2.17) lib/rack/method_override.rb:24:in `call'",
"rack (2.2.17) lib/rack/runtime.rb:22:in `call'",
"rack-timeout (0.7.0) lib/rack/timeout/core.rb:154:in `block in call'",
"rack-timeout (0.7.0) lib/rack/timeout/support/timeout.rb:19:in `timeout'",
"rack-timeout (0.7.0) lib/rack/timeout/core.rb:153:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:44:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/executor.rb:14:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:83:in `call'",
"gitlab-labkit (0.39.0) lib/labkit/middleware/rack.rb:22:in `block in call'",
"gitlab-labkit (0.39.0) lib/labkit/context.rb:35:in `with_context'",
"gitlab-labkit (0.39.0) lib/labkit/middleware/rack.rb:21:in `call'",
"rack (2.2.17) lib/rack/sendfile.rb:110:in `call'",
"actionpack (7.1.5.1) lib/action_dispatch/middleware/request_id.rb:28:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"railties (7.1.5.1) lib/rails/engine.rb:536:in `call'",
"railties (7.1.5.1) lib/rails/railtie.rb:226:in `public_send'",
"railties (7.1.5.1) lib/rails/railtie.rb:226:in `method_missing'",
"lib/gitlab/middleware/release_env.rb:12:in `call'",
"rack (2.2.17) lib/rack/urlmap.rb:74:in `block in call'",
"rack (2.2.17) lib/rack/urlmap.rb:58:in `each'",
"rack (2.2.17) lib/rack/urlmap.rb:58:in `call'",
"puma (6.6.0) lib/puma/configuration.rb:279:in `call'",
"puma (6.6.0) lib/puma/request.rb:99:in `block in handle_request'",
"puma (6.6.0) lib/puma/thread_pool.rb:390:in `with_force_shutdown'",
"puma (6.6.0) lib/puma/request.rb:98:in `handle_request'",
"puma (6.6.0) lib/puma/server.rb:472:in `process_client'",
"puma (6.6.0) lib/puma/server.rb:254:in `block in run'",
"puma (6.6.0) lib/puma/thread_pool.rb:167:in `block in spawn_thread'"
],
"queue_duration_s": 0.035401,
"gitaly_calls": 4,
"gitaly_duration_s": 0.709579,
"redis_calls": 10,
"redis_duration_s": 0.008213,
"redis_read_bytes": 171,
"redis_write_bytes": 580,
"redis_cache_calls": 1,
"redis_cache_duration_s": 0.000201,
"redis_cache_read_bytes": 4,
"redis_cache_write_bytes": 107,
"redis_db_load_balancing_calls": 2,
"redis_db_load_balancing_duration_s": 0.000556,
"redis_db_load_balancing_write_bytes": 106,
"redis_feature_flag_calls": 1,
"redis_feature_flag_duration_s": 0.00339,
"redis_feature_flag_read_bytes": 115,
"redis_feature_flag_write_bytes": 66,
"redis_repository_cache_calls": 6,
"redis_repository_cache_duration_s": 0.004066,
"redis_repository_cache_read_bytes": 52,
"redis_repository_cache_write_bytes": 301,
"db_count": 26,
"db_write_count": 0,
"db_cached_count": 7,
"db_txn_count": 0,
"db_replica_txn_count": 0,
"db_primary_txn_count": 0,
"db_replica_count": 0,
"db_primary_count": 26,
"db_replica_write_count": 0,
"db_primary_write_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 7,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_replica_txn_max_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"db_replica_txn_duration_s": 0,
"db_primary_txn_duration_s": 0,
"db_replica_duration_s": 0,
"db_primary_duration_s": 0.044,
"db_main_txn_count": 0,
"db_ci_txn_count": 0,
"db_main_replica_txn_count": 0,
"db_ci_replica_txn_count": 0,
"db_main_count": 26,
"db_ci_count": 0,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_main_write_count": 0,
"db_ci_write_count": 0,
"db_main_replica_write_count": 0,
"db_ci_replica_write_count": 0,
"db_main_cached_count": 7,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_main_txn_max_duration_s": 0,
"db_ci_txn_max_duration_s": 0,
"db_main_replica_txn_max_duration_s": 0,
"db_ci_replica_txn_max_duration_s": 0,
"db_main_txn_duration_s": 0,
"db_ci_txn_duration_s": 0,
"db_main_replica_txn_duration_s": 0,
"db_ci_replica_txn_duration_s": 0,
"db_main_duration_s": 0.044,
"db_ci_duration_s": 0,
"db_main_replica_duration_s": 0,
"db_ci_replica_duration_s": 0,
"external_http_count": 1,
"external_http_duration_s": 0.08665042899974651,
"path_traversal_check_duration_s": 0.000074,
"cpu_s": 0.312506,
"mem_objects": 40331,
"mem_bytes": 4804256,
"mem_mallocs": 5389,
"mem_total_bytes": 6417496,
"pid": 80,
"worker_id": "puma_1",
"rate_limiting_gates": [],
"correlation_id": "01K0S11B6AZXFAX74A9EJP2C37",
"meta.caller_id": "POST /api/:version/projects/:id/repository/commits",
"meta.remote_ip": "84.146.10.138",
"meta.feature_category": "source_code_management",
"meta.user": "root",
"meta.user_id": 1,
"meta.project": "root/testproject-2025-07-22-12-16-40",
"meta.root_namespace": "root",
"meta.client_id": "user/1",
"content_length": "136",
"request_urgency": "low",
"target_duration_s": 5
}