Advanced Search - x_content_parse_exception error on simple search
Summary
After upgrading to the GitLab 17.1.1 security patch, our advanced search has started giving a 500 error when executing a simple search of code (searching issues/merge requests seems fine). Prior to the patch we were on 17.0.2 and this issue was not present. We are using Elasticsearch 7.7.0.
Steps to reproduce
- Connect GitLab 17.1.1 to ES 7.7.0 and enable advanced search.
- Create two projects with the same file in
- Search for a word that is in both files
- Click to the code search tab and receive a 500 error
What is the current bug behavior?
When searching for a word or term, a 500 error is shown in the UI and an error is thrown in api_json.log
. Clicking to the issues/merge requests tabs the search is as expected, trying to search code or commits gives 500.
What is the expected correct behavior?
Search should not throw an error, instead showing the results of the search.
Relevant logs and/or screenshots
Expand for error log and stack trace
{ "method": "GET", "path": "/search", "format": "html", "controller": "SearchController", "action": "show", "status": 500, "time": "2024-07-01T11:27:46.004Z", "params": [ { "key": "scope", "value": "blobs" }, { "key": "search", "value": "[FILTERED]" } ], "correlation_id": "01J1Q0T1RXRKJV0952JRZWPY14", "meta.caller_id": "SearchController#show", "meta.remote_ip": "130.246.160.211", "meta.feature_category": "global_search", "meta.user": "rose.dickinson", "meta.user_id": 36, "meta.client_id": "user/36", "meta.search.group_id": null, "meta.search.project_id": null, "meta.search.scope": "blobs", "meta.search.page": "1", "meta.search.filters.confidential": null, "meta.search.filters.state": null, "meta.search.force_search_results": null, "meta.search.project_ids": null, "meta.search.filters.language": null, "meta.search.type": "advanced", "meta.search.level": "global", "remote_ip": "130.246.160.211", "user_id": 36, "username": "rose.dickinson", "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0", "queue_duration_s": 0.006694, "request_urgency": "low", "target_duration_s": 5, "redis_calls": 13, "redis_allowed_cross_slot_calls": 1, "redis_duration_s": 0.001383, "redis_read_bytes": 1312, "redis_write_bytes": 1240, "redis_cache_calls": 1, "redis_cache_duration_s": 9.8e-05, "redis_cache_read_bytes": 109, "redis_cache_write_bytes": 100, "redis_db_load_balancing_calls": 2, "redis_db_load_balancing_duration_s": 0.000158, "redis_db_load_balancing_write_bytes": 106, "redis_feature_flag_calls": 5, "redis_feature_flag_duration_s": 0.000569, "redis_feature_flag_read_bytes": 991, "redis_feature_flag_write_bytes": 275, "redis_rate_limiting_calls": 2, "redis_rate_limiting_duration_s": 0.000181, "redis_rate_limiting_read_bytes": 2, "redis_rate_limiting_write_bytes": 142, "redis_sessions_calls": 3, "redis_sessions_allowed_cross_slot_calls": 1, "redis_sessions_duration_s": 0.000377, "redis_sessions_read_bytes": 210, "redis_sessions_write_bytes": 617, "elasticsearch_calls": 1, "elasticsearch_duration_s": 0.01877, "elasticsearch_timed_out_count": 0, "db_count": 3, "db_write_count": 0, "db_cached_count": 0, "db_txn_count": 0, "db_replica_txn_count": 0, "db_primary_txn_count": 0, "db_main_txn_count": 0, "db_ci_txn_count": 0, "db_main_replica_txn_count": 0, "db_ci_replica_txn_count": 0, "db_replica_count": 0, "db_primary_count": 3, "db_main_count": 3, "db_ci_count": 0, "db_main_replica_count": 0, "db_ci_replica_count": 0, "db_replica_cached_count": 0, "db_primary_cached_count": 0, "db_main_cached_count": 0, "db_ci_cached_count": 0, "db_main_replica_cached_count": 0, "db_ci_replica_cached_count": 0, "db_replica_wal_count": 0, "db_primary_wal_count": 0, "db_main_wal_count": 0, "db_ci_wal_count": 0, "db_main_replica_wal_count": 0, "db_ci_replica_wal_count": 0, "db_replica_wal_cached_count": 0, "db_primary_wal_cached_count": 0, "db_main_wal_cached_count": 0, "db_ci_wal_cached_count": 0, "db_main_replica_wal_cached_count": 0, "db_ci_replica_wal_cached_count": 0, "db_replica_txn_max_duration_s": 0.0, "db_primary_txn_max_duration_s": 0.0, "db_main_txn_max_duration_s": 0.0, "db_ci_txn_max_duration_s": 0.0, "db_main_replica_txn_max_duration_s": 0.0, "db_ci_replica_txn_max_duration_s": 0.0, "db_replica_txn_duration_s": 0.0, "db_primary_txn_duration_s": 0.0, "db_main_txn_duration_s": 0.0, "db_ci_txn_duration_s": 0.0, "db_main_replica_txn_duration_s": 0.0, "db_ci_replica_txn_duration_s": 0.0, "db_replica_duration_s": 0.0, "db_primary_duration_s": 0.002, "db_main_duration_s": 0.002, "db_ci_duration_s": 0.0, "db_main_replica_duration_s": 0.0, "db_ci_replica_duration_s": 0.0, "external_http_count": 1, "external_http_duration_s": 0.01445901207625866, "cpu_s": 0.036029, "mem_objects": 22396, "mem_bytes": 2968104, "mem_mallocs": 6325, "mem_total_bytes": 3863944, "pid": 8246, "worker_id": "puma_1", "rate_limiting_gates": [ "search_rate_limit" ], "exception.class": "Elasticsearch::Transport::Transport::Errors::BadRequest", "exception.message": "[400] {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:1371] [bool] failed to parse field [minimum_should_match]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:1371] [bool] failed to parse field [minimum_should_match]\",\"caused_by\":{\"type\":\"illegal_state_exception\",\"reason\":\"Can't get text on a VALUE_NULL at 1:1371\"}},\"status\":400}", "exception.backtrace": [ "elasticsearch-transport (7.13.3) lib/elasticsearch/transport/transport/base.rb:218:in `__raise_transport_error'", "elasticsearch-transport (7.13.3) lib/elasticsearch/transport/transport/base.rb:347:in `perform_request'", "elasticsearch-transport (7.13.3) lib/elasticsearch/transport/transport/http/faraday.rb:37:in `perform_request'", "elasticsearch-transport (7.13.3) lib/elasticsearch/transport/client.rb:192:in `perform_request'", "lib/gitlab/instrumentation/elasticsearch_transport.rb:12:in `perform_request'", "elasticsearch-api (7.13.3) lib/elasticsearch/api/actions/search.rb:104:in `search'", "config/initializers/elastic_client_setup.rb:32:in `execute!'", "elasticsearch-model (7.2.0) lib/elasticsearch/model/response.rb:46:in `response'", "elasticsearch-model (7.2.0) lib/elasticsearch/model/response/results.rb:42:in `results'", "elasticsearch-model (7.2.0) lib/elasticsearch/model/response/results.rb:30:in `size'", "elasticsearch-model (7.2.0) lib/elasticsearch/model/response.rb:34:in `size'", "ee/lib/elastic/latest/git_class_proxy.rb:191:in `search_blob'", "ee/lib/elastic/latest/git_class_proxy.rb:19:in `elastic_search'", "ee/lib/elastic/latest/git_class_proxy.rb:199:in `elastic_search_and_wrap'", "ee/lib/elastic/latest/git_class_proxy.rb:28:in `elastic_search_as_found_blob'", "ee/lib/elastic/multi_version_util.rb:76:in `elastic_search_as_found_blob'", "ee/lib/gitlab/elastic/search_results.rb:374:in `block in blobs'", "gems/gitlab-utils/lib/gitlab/utils/strong_memoize.rb:34:in `strong_memoize'", "ee/lib/gitlab/elastic/search_results.rb:373:in `blobs'", "ee/lib/gitlab/elastic/search_results.rb:48:in `objects'", "app/services/search_service.rb:68:in `search_objects'", "app/presenters/search_service_presenter.rb:25:in `search_objects'", "app/controllers/search_controller.rb:72:in `block in show'", "/opt/gitlab/embedded/lib/ruby/3.1.0/benchmark.rb:311:in `realtime'", "app/controllers/search_controller.rb:69:in `show'", "actionpack (7.0.8.4) lib/action_controller/metal/basic_implicit_render.rb:6:in `send_action'", "actionpack (7.0.8.4) lib/abstract_controller/base.rb:215:in `process_action'", "actionpack (7.0.8.4) lib/action_controller/metal/rendering.rb:165:in `process_action'", "actionpack (7.0.8.4) lib/abstract_controller/callbacks.rb:234:in `block in process_action'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:118:in `block in run_callbacks'", "app/controllers/application_controller.rb:517:in `block in allow_gitaly_ref_name_caching'", "lib/gitlab/gitaly_client.rb:472:in `allow_ref_name_caching'", "app/controllers/application_controller.rb:516:in `allow_gitaly_ref_name_caching'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "lib/gitlab/ip_address_state.rb:11:in `with'", "ee/app/controllers/ee/application_controller.rb:45:in `set_current_ip_address'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "app/controllers/application_controller.rb:468:in `set_current_admin'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "lib/gitlab/session.rb:11:in `with_session'", "app/controllers/application_controller.rb:459:in `set_session_storage'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "lib/gitlab/i18n.rb:114:in `with_locale'", "lib/gitlab/i18n.rb:120:in `with_user_locale'", "app/controllers/application_controller.rb:450:in `set_locale'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "marginalia (1.11.1) lib/marginalia.rb:109:in `record_query_comment'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "app/controllers/application_controller.rb:443:in `set_current_context'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "sentry-rails (5.17.3) lib/sentry/rails/controller_transaction.rb:28:in `block in sentry_around_action'", "sentry-ruby (5.17.3) lib/sentry/hub.rb:102:in `with_child_span'", "sentry-ruby (5.17.3) lib/sentry-ruby.rb:490:in `with_child_span'", "sentry-rails (5.17.3) lib/sentry/rails/controller_transaction.rb:14:in `sentry_around_action'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:127:in `block in run_callbacks'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:138:in `run_callbacks'", "actionpack (7.0.8.4) lib/abstract_controller/callbacks.rb:233:in `process_action'", "actionpack (7.0.8.4) lib/action_controller/metal/rescue.rb:23:in `process_action'", "actionpack (7.0.8.4) lib/action_controller/metal/instrumentation.rb:67:in `block in process_action'", "activesupport (7.0.8.4) lib/active_support/notifications.rb:206:in `block in instrument'", "activesupport (7.0.8.4) lib/active_support/notifications/instrumenter.rb:24:in `instrument'", "activesupport (7.0.8.4) lib/active_support/notifications.rb:206:in `instrument'", "actionpack (7.0.8.4) lib/action_controller/metal/instrumentation.rb:66:in `process_action'", "actionpack (7.0.8.4) lib/action_controller/metal/params_wrapper.rb:259:in `process_action'", "activerecord (7.0.8.4) lib/active_record/railties/controller_runtime.rb:27:in `process_action'", "actionpack (7.0.8.4) lib/abstract_controller/base.rb:151:in `process'", "actionview (7.0.8.4) lib/action_view/rendering.rb:39:in `process'", "actionpack (7.0.8.4) lib/action_controller/metal.rb:188:in `dispatch'", "actionpack (7.0.8.4) lib/action_controller/metal.rb:251:in `dispatch'", "actionpack (7.0.8.4) lib/action_dispatch/routing/route_set.rb:49:in `dispatch'", "actionpack (7.0.8.4) lib/action_dispatch/routing/route_set.rb:32:in `serve'", "actionpack (7.0.8.4) lib/action_dispatch/journey/router.rb:50:in `block in serve'", "actionpack (7.0.8.4) lib/action_dispatch/journey/router.rb:32:in `each'", "actionpack (7.0.8.4) lib/action_dispatch/journey/router.rb:32:in `serve'", "actionpack (7.0.8.4) lib/action_dispatch/routing/route_set.rb:852:in `call'", "gitlab-experiment (0.9.1) lib/gitlab/experiment/middleware.rb:19:in `call'", "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:72:in `memoized_call'", "flipper (0.26.2) lib/flipper/middleware/memoizer.rb:37:in `call'", "lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'", "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `block in call'", "lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'", "lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `call'", "lib/gitlab/middleware/memory_report.rb:13:in `call'", "lib/gitlab/middleware/speedscope.rb:13:in `call'", "lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'", "lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'", "lib/gitlab/etag_caching/middleware.rb:21:in `call'", "lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'", "lib/gitlab/metrics/web_transaction.rb:46:in `run'", "lib/gitlab/metrics/rack_middleware.rb:16:in `call'", "lib/gitlab/middleware/go.rb:20:in `call'", "lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'", "lib/gitlab/database/query_analyzer.rb:40:in `within'", "lib/gitlab/middleware/query_analyzer.rb:11:in `call'", "lib/gitlab/middleware/organizations/current.rb:20:in `call'", "batch-loader (2.0.5) lib/batch_loader/middleware.rb:11:in `call'", "rack-attack (6.7.0) lib/rack/attack.rb:103:in `call'", "apollo_upload_server (2.1.6) lib/apollo_upload_server/middleware.rb:19:in `call'", "lib/gitlab/middleware/multipart.rb:173:in `call'", "rack-attack (6.7.0) lib/rack/attack.rb:127:in `call'", "warden (1.2.9) lib/warden/manager.rb:36:in `block in call'", "warden (1.2.9) lib/warden/manager.rb:34:in `catch'", "warden (1.2.9) lib/warden/manager.rb:34:in `call'", "rack-cors (2.0.1) lib/rack/cors.rb:102:in `call'", "rack (2.2.8.1) lib/rack/tempfile_reaper.rb:15:in `call'", "rack (2.2.8.1) lib/rack/etag.rb:27:in `call'", "rack (2.2.8.1) lib/rack/conditional_get.rb:27:in `call'", "rack (2.2.8.1) lib/rack/head.rb:12:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/http/permissions_policy.rb:38:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/http/content_security_policy.rb:36:in `call'", "lib/gitlab/middleware/read_only/controller.rb:50:in `call'", "lib/gitlab/middleware/read_only.rb:18:in `call'", "lib/gitlab/middleware/unauthenticated_session_expiry.rb:18:in `call'", "rack (2.2.8.1) lib/rack/session/abstract/id.rb:266:in `context'", "rack (2.2.8.1) lib/rack/session/abstract/id.rb:260:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/cookies.rb:704:in `call'", "lib/gitlab/middleware/same_site_cookies.rb:27:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/callbacks.rb:27:in `block in call'", "activesupport (7.0.8.4) lib/active_support/callbacks.rb:99:in `run_callbacks'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/callbacks.rb:26:in `call'", "sentry-rails (5.17.3) lib/sentry/rails/rescued_exception_interceptor.rb:12:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/debug_exceptions.rb:28:in `call'", "lib/gitlab/middleware/path_traversal_check.rb:27:in `call'", "lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'", "sentry-ruby (5.17.3) lib/sentry/rack/capture_exceptions.rb:29:in `block (2 levels) in call'", "sentry-ruby (5.17.3) lib/sentry/hub.rb:251:in `with_session_tracking'", "sentry-ruby (5.17.3) lib/sentry-ruby.rb:403:in `with_session_tracking'", "sentry-ruby (5.17.3) lib/sentry/rack/capture_exceptions.rb:20:in `block in call'", "sentry-ruby (5.17.3) lib/sentry/hub.rb:59:in `with_scope'", "sentry-ruby (5.17.3) lib/sentry-ruby.rb:383:in `with_scope'", "sentry-ruby (5.17.3) lib/sentry/rack/capture_exceptions.rb:19:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/show_exceptions.rb:29:in `call'", "lib/gitlab/middleware/basic_health_check.rb:25:in `call'", "lograge (0.11.2) lib/lograge/rails_ext/rack/logger.rb:15:in `call_app'", "railties (7.0.8.4) lib/rails/rack/logger.rb:25:in `block in call'", "activesupport (7.0.8.4) lib/active_support/tagged_logging.rb:99:in `block in tagged'", "activesupport (7.0.8.4) lib/active_support/tagged_logging.rb:37:in `tagged'", "activesupport (7.0.8.4) lib/active_support/tagged_logging.rb:99:in `tagged'", "railties (7.0.8.4) lib/rails/rack/logger.rb:25:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/remote_ip.rb:93:in `call'", "lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'", "lib/gitlab/middleware/request_context.rb:15:in `call'", "lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'", "request_store (1.5.1) lib/request_store/middleware.rb:19:in `call'", "rack (2.2.8.1) lib/rack/method_override.rb:24:in `call'", "rack (2.2.8.1) lib/rack/runtime.rb:22:in `call'", "rack-timeout (0.6.3) lib/rack/timeout/core.rb:148:in `block in call'", "rack-timeout (0.6.3) lib/rack/timeout/support/timeout.rb:19:in `timeout'", "rack-timeout (0.6.3) lib/rack/timeout/core.rb:147:in `call'", "config/initializers/fix_local_cache_middleware.rb:11:in `call'", "lib/gitlab/middleware/compressed_json.rb:44:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/executor.rb:14:in `call'", "lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'", "rack (2.2.8.1) lib/rack/sendfile.rb:110:in `call'", "lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'", "lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call'", "gitlab-labkit (0.36.0) lib/labkit/middleware/rack.rb:22:in `block in call'", "gitlab-labkit (0.36.0) lib/labkit/context.rb:35:in `with_context'", "gitlab-labkit (0.36.0) lib/labkit/middleware/rack.rb:21:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/request_id.rb:26:in `call'", "actionpack (7.0.8.4) lib/action_dispatch/middleware/host_authorization.rb:131:in `call'", "railties (7.0.8.4) lib/rails/engine.rb:530:in `call'", "railties (7.0.8.4) lib/rails/railtie.rb:226:in `public_send'", "railties (7.0.8.4) lib/rails/railtie.rb:226:in `method_missing'", "lib/gitlab/middleware/release_env.rb:13:in `call'", "rack (2.2.8.1) lib/rack/urlmap.rb:74:in `block in call'", "rack (2.2.8.1) lib/rack/urlmap.rb:58:in `each'", "rack (2.2.8.1) lib/rack/urlmap.rb:58:in `call'", "puma (6.4.0) lib/puma/configuration.rb:272:in `call'", "puma (6.4.0) lib/puma/request.rb:100:in `block in handle_request'", "puma (6.4.0) lib/puma/thread_pool.rb:378:in `with_force_shutdown'", "puma (6.4.0) lib/puma/request.rb:99:in `handle_request'", "puma (6.4.0) lib/puma/server.rb:443:in `process_client'", "puma (6.4.0) lib/puma/server.rb:241:in `block in run'", "puma (6.4.0) lib/puma/thread_pool.rb:155:in `block in spawn_thread'" ], "db_duration_s": 0.00155, "view_duration_s": 0.0, "duration_s": 0.04211 }
Results of GitLab environment info
Expand for output related to GitLab environment info
System information System: Proxy: no Current User: git Using RVM: no Ruby Version: 3.1.5p253 Gem Version: 3.5.11 Bundler Version:2.5.11 Rake Version: 13.0.6 Redis Version: 7.0.15 Sidekiq Version:7.1.6 Go Version: unknown GitLab information Version: 17.1.1-ee Revision: d0ac56e0be2 Directory: /opt/gitlab/embedded/service/gitlab-rails DB Adapter: PostgreSQL DB Version: 14.11 URL: https://gitlab.dafni.rl.ac.uk HTTP Clone URL: https://gitlab.dafni.rl.ac.uk/some-group/some-project.git SSH Clone URL: git@git.dafni.rl.ac.uk:some-group/some-project.git Elasticsearch: yes Geo: no Using LDAP: no Using Omniauth: yes Omniauth Providers: GitLab Shell Version: 14.36.0 Repository storages: - default: unix:/var/opt/gitlab/gitaly/gitaly.socket GitLab Shell path: /opt/gitlab/embedded/service/gitlab-shell Gitaly - default Address: unix:/var/opt/gitlab/gitaly/gitaly.socket - default Version: 17.1.1 - default Git Version: 2.45.1
Results of GitLab application Check
Expand for output related to the GitLab application check
Checking GitLab subtasks ...
Checking GitLab Shell ...
GitLab Shell: ... GitLab Shell version >= 14.36.0 ? ... OK (14.36.0) Running /opt/gitlab/embedded/service/gitlab-shell/bin/check Internal API available: OK Redis available via internal API: OK gitlab-shell self-check successful
Checking GitLab Shell ... Finished
Checking Gitaly ...
Gitaly: ... default ... OK
Checking Gitaly ... Finished
Checking Sidekiq ...
Sidekiq: ... Running? ... yes Number of Sidekiq processes (cluster/worker) ... 1/1
Checking Sidekiq ... Finished
Checking Incoming Email ...
Incoming Email: ... Checking Reply by email ...
IMAP server credentials are correct? ... Checking gitlab@dafni.rl.ac.uk no Try fixing it: An error occurred: Errno::EHOSTUNREACH: No route to host - connect(2) for 130.246.5.8:143 Check that the information in config/gitlab.yml is correct For more information see: doc/administration/reply_by_email.md Please fix the error above and rerun the checks. Mailroom enabled? ... skipped MailRoom running? ... skipped
Checking Reply by email ... Finished
Checking Incoming Email ... Finished
Checking LDAP ...
LDAP: ... LDAP is disabled in config/gitlab.yml
Checking LDAP ... Finished
Checking GitLab App ...
Database config exists? ... yes Tables are truncated? ... skipped All migrations up? ... yes Database contains orphaned GroupMembers? ... no GitLab config exists? ... yes GitLab config up to date? ... yes Cable config exists? ... yes Resque config exists? ... yes Log directory writable? ... yes Tmp directory writable? ... yes Uploads directory exists? ... yes Uploads directory has correct permissions? ... yes Uploads directory tmp has correct permissions? ... yes Systemd unit files or init script exist? ... skipped (omnibus-gitlab has neither init script nor systemd units) Systemd unit files or init script up-to-date? ... skipped (omnibus-gitlab has neither init script nor systemd units) Projects have namespace: ... 3/2 ... yes 2/3 ... yes 2/4 ... yes 3/5 ... yes 3/6 ... yes 6/7 ... yes 6/8 ... yes 2/9 ... yes 3/10 ... yes 7/11 ... yes 7/12 ... yes 2/13 ... yes 6/14 ... yes 2/17 ... yes 3/18 ... yes 2/20 ... yes 2/21 ... yes 10/22 ... yes 4/26 ... yes 3/30 ... yes 3/33 ... yes 3/36 ... yes 3/37 ... yes 3/39 ... yes 3/41 ... yes 14/42 ... yes 14/43 ... yes 2/44 ... yes 2/46 ... yes 2/47 ... yes 2/48 ... yes 14/50 ... yes 4/52 ... yes 18/53 ... yes 17/54 ... yes 17/55 ... yes 17/56 ... yes 17/57 ... yes 17/58 ... yes 17/59 ... yes 17/60 ... yes 17/61 ... yes 3/62 ... yes 17/63 ... yes 17/64 ... yes 17/65 ... yes 17/66 ... yes 2/67 ... yes 3/68 ... yes 13/69 ... yes 2/70 ... yes 3/71 ... yes 10/72 ... yes 3/73 ... yes 2/74 ... yes 3/75 ... yes 10/76 ... yes 13/78 ... yes 2/79 ... yes 14/80 ... yes 3/81 ... yes 3/85 ... yes 2/86 ... yes 3/87 ... yes 10/89 ... yes 3/90 ... yes 3/91 ... yes 3/92 ... yes 23/95 ... yes 2/99 ... yes 25/100 ... yes 26/101 ... yes 26/102 ... yes 26/103 ... yes 26/104 ... yes 26/105 ... yes 25/106 ... yes 26/107 ... yes 2/109 ... yes 2/111 ... yes 26/112 ... yes 2/113 ... yes 2/114 ... yes 2/115 ... yes 2/116 ... yes 10/117 ... yes 25/118 ... yes 25/119 ... yes 3/120 ... yes 3/121 ... yes 25/123 ... yes 25/125 ... yes 28/126 ... yes 25/128 ... yes 28/129 ... yes 28/132 ... yes 2/133 ... yes 26/134 ... yes 2/136 ... yes 26/137 ... yes 2/138 ... yes 28/139 ... yes 28/141 ... yes 28/142 ... yes 32/143 ... yes 28/144 ... yes 45/145 ... yes 45/146 ... yes 45/147 ... yes 45/148 ... yes 24/149 ... yes 45/150 ... yes 2/151 ... yes 45/152 ... yes 45/153 ... yes 45/155 ... yes 45/156 ... yes 45/157 ... yes 45/158 ... yes 45/159 ... yes 45/160 ... yes 32/161 ... yes 48/162 ... yes 28/163 ... yes 32/164 ... yes 49/165 ... yes 49/166 ... yes 48/167 ... yes 32/168 ... yes 25/169 ... yes 51/170 ... yes 32/171 ... yes 48/172 ... yes 25/173 ... yes 48/174 ... yes 23/176 ... yes 25/177 ... yes 53/178 ... yes 53/179 ... yes 53/180 ... yes 53/181 ... yes 54/188 ... yes 54/189 ... yes 25/190 ... yes 53/191 ... yes 54/192 ... yes 32/193 ... yes 32/195 ... yes 48/196 ... yes 53/197 ... yes 32/198 ... yes 25/200 ... yes 32/201 ... yes 54/202 ... yes 53/203 ... yes 28/204 ... yes 2/205 ... yes 53/206 ... yes 53/207 ... yes 25/208 ... yes 14/209 ... yes 59/210 ... yes 54/211 ... yes 53/212 ... yes 2/213 ... yes 25/214 ... yes 14/215 ... yes 55/216 ... yes 23/217 ... yes 3/218 ... yes 14/221 ... yes 54/222 ... yes 3/223 ... yes 54/224 ... yes 54/225 ... yes 54/226 ... yes 32/227 ... yes 50/229 ... yes 23/230 ... yes 25/231 ... yes 14/232 ... yes 54/233 ... yes 54/234 ... yes 54/235 ... yes 54/236 ... yes 53/237 ... yes 50/238 ... yes 54/239 ... yes 23/240 ... yes 54/241 ... yes 2/242 ... yes 23/243 ... yes 54/244 ... yes 2/245 ... yes 14/246 ... yes 14/247 ... yes 2/248 ... yes 64/249 ... yes 14/250 ... yes 69/251 ... yes 14/252 ... yes 23/253 ... yes 26/254 ... yes 2/256 ... yes 28/257 ... yes 71/258 ... yes 14/259 ... yes 64/261 ... yes 65/262 ... yes 62/263 ... yes 62/264 ... yes 23/265 ... yes 14/266 ... yes 28/267 ... yes 14/268 ... yes 14/269 ... yes 23/270 ... yes 23/271 ... yes 10/272 ... yes 10/273 ... yes 14/274 ... yes 10/275 ... yes 55/276 ... yes 72/277 ... yes 72/278 ... yes 25/279 ... yes 28/280 ... yes 72/281 ... yes 14/283 ... yes 72/284 ... yes 72/285 ... yes 23/286 ... yes 72/287 ... yes 72/288 ... yes 28/289 ... yes 26/290 ... yes 72/291 ... yes 3/292 ... yes 14/293 ... yes 14/294 ... yes 54/296 ... yes 23/297 ... yes 63/300 ... yes 54/301 ... yes 72/302 ... yes 14/303 ... yes 26/304 ... yes 75/305 ... yes 76/306 ... yes 10/307 ... yes 78/308 ... yes 14/309 ... yes 82/312 ... yes 72/313 ... yes 78/314 ... yes 28/317 ... yes 85/318 ... yes 85/319 ... yes 85/320 ... yes 59/321 ... yes 48/322 ... yes 24/323 ... yes 3/325 ... yes 78/327 ... yes 24/328 ... yes 14/329 ... yes 72/330 ... yes 59/332 ... yes 85/334 ... yes 89/335 ... yes 32/336 ... yes 3/337 ... yes 3/338 ... yes 14/339 ... yes 14/340 ... yes 24/341 ... yes 92/342 ... yes 54/343 ... yes 14/344 ... yes 32/345 ... yes 92/347 ... yes 59/348 ... yes 59/349 ... yes 24/350 ... yes 92/351 ... yes 63/352 ... yes 2/353 ... yes 2/354 ... yes 23/355 ... yes 3/356 ... yes 59/357 ... yes 14/358 ... yes 14/359 ... yes 14/360 ... yes 14/361 ... yes 85/362 ... yes 3/363 ... yes 3/364 ... yes 98/365 ... yes 98/366 ... yes 28/367 ... yes 14/368 ... yes 2/369 ... yes 14/370 ... yes 2/371 ... yes 98/372 ... yes 14/373 ... yes 14/374 ... yes 14/376 ... yes 68/377 ... yes 108/378 ... yes 10/379 ... yes 109/381 ... yes 14/383 ... yes 85/384 ... yes 24/385 ... yes 109/386 ... yes 14/387 ... yes 78/388 ... yes 14/389 ... yes 75/391 ... yes 78/392 ... yes 78/393 ... yes 14/394 ... yes 78/395 ... yes 78/396 ... yes 78/397 ... yes 2/398 ... yes 50/399 ... yes 14/400 ... yes 25/401 ... yes 25/402 ... yes 491/403 ... yes 25/404 ... yes 75/405 ... yes 50/406 ... yes 503/407 ... yes 504/408 ... yes 504/409 ... yes 509/411 ... yes 14/412 ... yes 2/413 ... yes 76/414 ... yes 10/415 ... yes 75/416 ... yes 10/417 ... yes 2/418 ... yes 504/419 ... yes 527/420 ... yes 2/421 ... yes 75/422 ... yes 14/423 ... yes 491/424 ... yes 108/425 ... yes 491/426 ... yes 50/427 ... yes 14/429 ... yes 2/430 ... yes 2/431 ... yes 10/433 ... yes 10/434 ... yes 23/435 ... yes 14/436 ... yes 561/437 ... yes 563/438 ... yes 565/439 ... yes Redis version >= 6.2.14? ... yes Ruby version >= 3.0.6 ? ... yes (3.1.5) Git user has default SSH configuration? ... no Try fixing it: mkdir ~/gitlab-check-backup-1719842287 sudo mv /var/opt/gitlab/.ssh/xx ~/gitlab-check-backup-1719842287 For more information see: doc/user/ssh.md#overriding-ssh-settings-on-the-gitlab-server Please fix the error above and rerun the checks. Active users: ... 21 Is authorized keys file accessible? ... yes GitLab configured to store new projects in hashed storage? ... yes All projects are in hashed storage? ... yes Elasticsearch version 7.x-8.x or OpenSearch version 1.x ... yes (elasticsearch 7.7.0) All migrations must be finished before doing a major upgrade ... yes
Checking GitLab App ... Finished
Checking GitLab subtasks ... Finished
Possible fixes
We tried a full advanced search index delete and recreation which hasn't done anything. We also tried restarting GitLab which also did nothing. The error in the relevant logs section has a full stack trace.