Server Error 500 when triggering Elasticsearch indexing for the first time
Summary
When setting up Advanced Search on our QA environment, using the AWS OpenSearch Service with IAM credentials via the web UI, we received a 500 Server Error response. According to rake tasks, the indexing got triggered just fine nevertheless and we were able to enable and use Advanced Search on the instance afterwards.
Steps to reproduce
- Prerequisites:
- Instance without ES indexes (if they have been created previously, delete them via
gitlab-rake gitlab:elastic:delete_index
) - AWS OpenSearch Service domain with Security Groups granting access to it from GitLab instance and permissions configured according to https://docs.gitlab.com/ee/integration/advanced_search/elasticsearch.html#aws-opensearch-service-with-fine-grained-access-control and AWS IAM permissions assigned via the instance profile.
- Instance without ES indexes (if they have been created previously, delete them via
- As an admin user, go to
/admin/application_settings/advanced_search
and verify nothing has been configured so far. - Leave configuration values at their default except for the following:
- Elasticsearch indexing: checked
- URL: AWS OpenSearch Service domain endpoint
- Use AWS OpenSearch Service with IAM credentials: checked
- AWS region: region in which the domain resides
- Save changes.
- Click Index all projects.
- Encounter 500 Server Error.
What is the current bug behavior?
After clicking "index all projects" during setting up elastic search configuration, a 500 error occurs on a self-managed instance.
What is the expected correct behavior?
GitLab should start indexing the projects without throwing an error, after clicking "index all projects"
Relevant logs and/or screenshots
The following two log entries have been captured from CloudWatch at the time of the error:
{
"content_type": "text/html; charset=utf-8",
"correlation_id": "[FILTERED]",
"duration_ms": 3526,
"host": "qa-git.hostname.hidden",
"level": "info",
"method": "POST",
"msg": "access",
"proto": "HTTP/1.1",
"referrer": "https://qa-git.hostname.hidden/admin/application_settings/advanced_search",
"remote_addr": "127.0.0.1:0",
"remote_ip": "127.0.0.1",
"route": "",
"status": 500,
"system": "http",
"time": "2023-06-02T15:03:43Z",
"ttfb_ms": 3526,
"uri": "/admin/application_settings/advanced_search",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",
"written_bytes": 3054
}
{
"method": "PATCH",
"path": "/admin/application_settings/advanced_search",
"format": "html",
"controller": "Admin::ApplicationSettingsController",
"action": "advanced_search",
"status": 500,
"time": "2023-06-02T15:03:43.167Z",
"params": [
{
"key": "_method",
"value": "patch"
},
{
"key": "authenticity_token",
"value": "[FILTERED]"
},
{
"key": "application_setting",
"value": {
"elasticsearch_indexing": "[FILTERED]",
"elasticsearch_search": "[FILTERED]",
"elasticsearch_url": "[FILTERED]",
"elasticsearch_username": "[FILTERED]",
"elasticsearch_password": "[FILTERED]",
"elasticsearch_shards": "[FILTERED]",
"elasticsearch_replicas": "[FILTERED]",
"elasticsearch_indexed_file_size_limit_kb": "[FILTERED]",
"elasticsearch_indexed_field_length_limit": "[FILTERED]",
"elasticsearch_max_bulk_size_mb": "[FILTERED]",
"elasticsearch_max_bulk_concurrency": "[FILTERED]",
"elasticsearch_client_request_timeout": "[FILTERED]",
"elasticsearch_limit_indexing": "[FILTERED]",
"elasticsearch_namespace_ids": "[FILTERED]",
"elasticsearch_project_ids": "[FILTERED]",
"elasticsearch_analyzers_smartcn_enabled": "[FILTERED]",
"elasticsearch_analyzers_kuromoji_enabled": "[FILTERED]",
"elasticsearch_aws": "[FILTERED]",
"elasticsearch_aws_region": "[FILTERED]",
"elasticsearch_aws_access_key": "[FILTERED]",
"elasticsearch_aws_secret_access_key": "[FILTERED]"
}
}
],
"correlation_id": "[FILTERED]",
"meta.caller_id": "Admin::ApplicationSettingsController#advanced_search",
"meta.remote_ip": "[FILTERED]",
"meta.feature_category": "global_search",
"meta.user": "[FILTERED]",
"meta.user_id": "[FILTERED]",
"meta.client_id": "[FILTERED]",
"remote_ip": "[FILTERED]",
"user_id": "[FILTERED]",
"username": "[FILTERED]",
"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36",
"queue_duration_s": 0.031953,
"request_urgency": "low",
"target_duration_s": 5,
"redis_calls": 25,
"redis_allowed_cross_slot_calls": 1,
"redis_duration_s": 0.020668,
"redis_read_bytes": 3933,
"redis_write_bytes": 223037,
"redis_cache_calls": 2,
"redis_cache_duration_s": 0.005023,
"redis_cache_read_bytes": 120,
"redis_cache_write_bytes": 221227,
"redis_feature_flag_calls": 18,
"redis_feature_flag_duration_s": 0.012825,
"redis_feature_flag_read_bytes": 3570,
"redis_feature_flag_write_bytes": 1116,
"redis_sessions_calls": 3,
"redis_sessions_allowed_cross_slot_calls": 1,
"redis_sessions_duration_s": 0.001515,
"redis_sessions_read_bytes": 243,
"redis_sessions_write_bytes": 590,
"redis_shared_state_calls": 2,
"redis_shared_state_duration_s": 0.001305,
"redis_shared_state_write_bytes": 104,
"elasticsearch_calls": 34,
"elasticsearch_duration_s": 3.223628,
"elasticsearch_timed_out_count": 0,
"db_count": 38,
"db_write_count": 1,
"db_cached_count": 23,
"db_replica_count": 0,
"db_primary_count": 38,
"db_main_count": 38,
"db_ci_count": 0,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 23,
"db_main_cached_count": 23,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_replica_duration_s": 0,
"db_primary_duration_s": 0.064,
"db_main_duration_s": 0.064,
"db_ci_duration_s": 0,
"db_main_replica_duration_s": 0,
"db_ci_replica_duration_s": 0,
"external_http_count": 30,
"external_http_duration_s": 2.9357465095818043,
"cpu_s": 0.407091,
"mem_objects": 262728,
"mem_bytes": 22772566,
"mem_mallocs": 114353,
"mem_total_bytes": 33281686,
"pid": 9729,
"worker_id": "puma_1",
"rate_limiting_gates": [],
"exception.class": "Elasticsearch::Transport::Transport::Error",
"exception.message": "Cannot get new connection from pool.",
"exception.backtrace": [
"lib/gitlab/instrumentation/elasticsearch_transport.rb:12:in `perform_request'",
"ee/lib/gitlab/elastic/helper.rb:222:in `index_exists?'",
"ee/app/models/elastic/migration_record.rb:19:in `save!'",
"ee/app/services/elastic/data_migration_service.rb:97:in `block in mark_all_as_completed!'",
"ee/app/services/elastic/data_migration_service.rb:96:in `each'",
"ee/app/services/elastic/data_migration_service.rb:96:in `mark_all_as_completed!'",
"ee/app/services/ee/application_settings/update_service.rb:102:in `find_or_create_elasticsearch_index'",
"ee/app/services/ee/application_settings/update_service.rb:26:in `execute'",
"app/controllers/admin/application_settings_controller.rb:203:in `perform_update'",
"ee/app/controllers/ee/admin/application_settings_controller.rb:72:in `block (2 levels) in <module:ApplicationSettingsController>'",
"ee/lib/gitlab/ip_address_state.rb:10:in `with'",
"ee/app/controllers/ee/application_controller.rb:45:in `set_current_ip_address'",
"app/controllers/application_controller.rb:500:in `set_current_admin'",
"lib/gitlab/session.rb:11:in `with_session'",
"app/controllers/application_controller.rb:491:in `set_session_storage'",
"lib/gitlab/i18n.rb:107:in `with_locale'",
"lib/gitlab/i18n.rb:113:in `with_user_locale'",
"app/controllers/application_controller.rb:482:in `set_locale'",
"app/controllers/application_controller.rb:475:in `set_current_context'",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/jira/middleware.rb:19:in `call'",
"lib/gitlab/middleware/go.rb:20:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:15:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:45:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call'",
"lib/gitlab/middleware/release_env.rb:13:in `call'"
],
"db_duration_s": 0.06193,
"view_duration_s": 0,
"duration_s": 3.47569
}