MigrateProjectsToSeparateIndex failing
Summary
MigrateProjectsToSeparateIndex migration is failing
Relevant logs and/or screenshots
Full Logs
[
{
"line": "Elastic::MigrationWorker JID-b8bcc3c0bd6514fbe9c8afa8: done: 0.337116 sec",
"timestamp": "1691071204064108046",
"fields": {
"Line": "Elastic::MigrationWorker JID-b8bcc3c0bd6514fbe9c8afa8: done: 0.337116 sec",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47_gitlab_sidekiq-6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Elastic::MigrationWorker",
"completed_at": "2023-08-03T14:00:04.063Z",
"correlation_id": "a38ec0e357cc80d5f184261928f10f04",
"cpu_s": "0.073051",
"created_at": "2023-08-03T14:00:03.721Z",
"db_cached_count": "0",
"db_ci_cached_count": "0",
"db_ci_count": "0",
"db_ci_duration_s": "0.0",
"db_ci_replica_cached_count": "0",
"db_ci_replica_count": "0",
"db_ci_replica_duration_s": "0.0",
"db_ci_replica_wal_cached_count": "0",
"db_ci_replica_wal_count": "0",
"db_ci_wal_cached_count": "0",
"db_ci_wal_count": "0",
"db_count": "0",
"db_duration_s": "0.0",
"db_main_cached_count": "0",
"db_main_count": "0",
"db_main_duration_s": "0.0",
"db_main_replica_cached_count": "0",
"db_main_replica_count": "0",
"db_main_replica_duration_s": "0.0",
"db_main_replica_wal_cached_count": "0",
"db_main_replica_wal_count": "0",
"db_main_wal_cached_count": "0",
"db_main_wal_count": "0",
"db_primary_cached_count": "0",
"db_primary_count": "0",
"db_primary_duration_s": "0.0",
"db_primary_wal_cached_count": "0",
"db_primary_wal_count": "0",
"db_replica_cached_count": "0",
"db_replica_count": "0",
"db_replica_duration_s": "0.0",
"db_replica_wal_cached_count": "0",
"db_replica_wal_count": "0",
"db_write_count": "0",
"duration_s": "0.337116",
"elasticsearch_calls": "17",
"elasticsearch_duration_s": "0.271274",
"elasticsearch_timed_out_count": "0",
"enqueued_at": "2023-08-03T14:00:03.723Z",
"environment": "aws",
"external_http_count": "17",
"external_http_duration_s": "0.1724706690038147",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:18a1564991a290ba236afceb12ec6c3c0dcffaff06178046cbd7d22b5ef4ab9b",
"jid": "b8bcc3c0bd6514fbe9c8afa8",
"job_size_bytes": "2",
"job_status": "done",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c82ef28c-f814-4a19-9217-1dd68b4c5363",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47",
"load_balancing_strategy": "primary",
"logtag": "F",
"mem_bytes": "2117584",
"mem_mallocs": "13874",
"mem_objects": "45432",
"mem_total_bytes": "3934864",
"message": "Elastic::MigrationWorker JID-b8bcc3c0bd6514fbe9c8afa8: done: 0.337116 sec",
"meta_caller_id": "Cronjob",
"meta_feature_category": "global_search",
"meta_root_caller_id": "Cronjob",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "cronjob",
"queue_pod_name": "all-in-1",
"redis_calls": "6",
"redis_duration_s": "0.00703",
"redis_feature_flag_calls": "2",
"redis_feature_flag_duration_s": "0.001395",
"redis_feature_flag_read_bytes": "398",
"redis_feature_flag_write_bytes": "146",
"redis_queues_calls": "2",
"redis_queues_duration_s": "0.001902",
"redis_queues_read_bytes": "2",
"redis_queues_write_bytes": "186",
"redis_read_bytes": "402",
"redis_shared_state_calls": "2",
"redis_shared_state_duration_s": "0.003733",
"redis_shared_state_read_bytes": "2",
"redis_shared_state_write_bytes": "285",
"redis_write_bytes": "617",
"release": "gitlab",
"retry": "0",
"scheduling_latency_s": "0.002514",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"target_duration_s": "300",
"time": "2023-08-03T14:00:04.063Z",
"urgency": "low",
"version": "0",
"worker_data_consistency": "always",
"worker_id": "sidekiq_0"
}
},
{
"line": "Elastic::MigrationWorker: Elasticsearch::Transport::Transport::Errors::BadRequest [400] {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:350] [reindex] failed to parse field [source]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"max must be greater than 1\"}}},\"status\":400}",
"timestamp": "1691071204059984497",
"fields": {
"Line": "Elastic::MigrationWorker: Elasticsearch::Transport::Transport::Errors::BadRequest [400] {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:350] [reindex] failed to parse field [source]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"max must be greater than 1\"}}},\"status\":400}",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47_gitlab_sidekiq-6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821.log",
"app": "sidekiq",
"backtrace": "/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/elasticsearch-transport-7.13.3/lib/elasticsearch/transport/transport/base.rb:218:in `__raise_transport_error'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/elasticsearch-transport-7.13.3/lib/elasticsearch/transport/transport/base.rb:347:in `perform_request'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/elasticsearch-transport-7.13.3/lib/elasticsearch/transport/transport/http/faraday.rb:37:in `perform_request'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/elasticsearch-transport-7.13.3/lib/elasticsearch/transport/client.rb:192:in `perform_request'\n/srv/gitlab/lib/gitlab/instrumentation/elasticsearch_transport.rb:12:in `perform_request'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/elasticsearch-api-7.13.3/lib/elasticsearch/api/actions/reindex.rb:50:in `reindex'\n/srv/gitlab/ee/app/workers/concerns/elastic/migration_helper.rb:63:in `reindex'\n/srv/gitlab/ee/elastic/migrate/20230530500000_migrate_projects_to_separate_index.rb:50:in `migrate'\n/srv/gitlab/ee/app/models/elastic/migration_record.rb:104:in `public_send'\n/srv/gitlab/ee/app/models/elastic/migration_record.rb:104:in `method_missing'\n/srv/gitlab/ee/app/workers/elastic/migration_worker.rb:100:in `execute_migration'\n/srv/gitlab/ee/app/workers/elastic/migration_worker.rb:59:in `block in perform'\n/srv/gitlab/lib/gitlab/exclusive_lease_helpers.rb:38:in `in_lock'\n/srv/gitlab/ee/app/workers/elastic/migration_worker.rb:26:in `perform'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:202:in `execute_job'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:170:in `block (2 levels) in process'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:177:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/skip_jobs.rb:49:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:29:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/application_context.rb:124:in `block in use'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/gitlab-labkit-0.33.0/lib/labkit/context.rb:35:in `with_context'\n/srv/gitlab/lib/gitlab/application_context.rb:124:in `use'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/worker_context.rb:11:in `wrap_in_optional_context'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'\n/srv/gitlab/lib/gitlab/application_context.rb:124:in `block in use'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/gitlab-labkit-0.33.0/lib/labkit/context.rb:35:in `with_context'\n/srv/gitlab/lib/gitlab/application_context.rb:124:in `use'\n/srv/gitlab/lib/gitlab/application_context.rb:62:in `with_context'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'\n/srv/gitlab/lib/gitlab/database/query_analyzer.rb:37:in `within'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'\n/srv/gitlab/lib/gitlab/with_request_store.rb:17:in `enabling_request_store'\n/srv/gitlab/lib/gitlab/with_request_store.rb:10:in `with_request_store'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/server_metrics.rb:84:in `block in call'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `block in instrument'\n/srv/gitlab/lib/gitlab/metrics/background_transaction.rb:33:in `run'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `instrument'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/server_metrics.rb:83:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/gitlab-labkit-0.33.0/lib/labkit/middleware/sidekiq/server.rb:21:in `block in call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:177:in `block in invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/gitlab-labkit-0.33.0/lib/labkit/middleware/sidekiq/context/server.rb:16:in `block in call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/gitlab-labkit-0.33.0/lib/labkit/context.rb:35:in `with_context'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/gitlab-labkit-0.33.0/lib/labkit/middleware/sidekiq/context/server.rb:15:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:182:in `invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/gitlab-labkit-0.33.0/lib/labkit/middleware/sidekiq/server.rb:20:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'\n/srv/gitlab/lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/marginalia-1.11.1/lib/marginalia/sidekiq_instrumentation.rb:9:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sentry-sidekiq-5.8.0/lib/sentry/sidekiq/sentry_context_middleware.rb:26:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sentry-raven-3.1.2/lib/raven/integrations/sidekiq/cleanup_middleware.rb:7:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:179:in `block in invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/middleware/chain.rb:182:in `invoke'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:169:in `block in process'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:136:in `block (6 levels) in dispatch'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/job_retry.rb:113:in `local'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:135:in `block (5 levels) in dispatch'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/rails.rb:14:in `block in call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/activesupport-7.0.6/lib/active_support/execution_wrapper.rb:92:in `wrap'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/activesupport-7.0.6/lib/active_support/reloader.rb:72:in `block in wrap'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/activesupport-7.0.6/lib/active_support/execution_wrapper.rb:92:in `wrap'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/activesupport-7.0.6/lib/active_support/reloader.rb:71:in `wrap'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/rails.rb:13:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:131:in `block (4 levels) in dispatch'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:263:in `stats'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:126:in `block (3 levels) in dispatch'\n/srv/gitlab/lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:125:in `block (2 levels) in dispatch'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/job_retry.rb:80:in `global'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:124:in `block in dispatch'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/job_logger.rb:39:in `prepare'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:123:in `dispatch'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:168:in `process'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:78:in `process_one'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/processor.rb:68:in `run'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/component.rb:8:in `watchdog'\n/srv/gitlab/vendor/bundle/ruby/3.0.0/gems/sidekiq-6.5.7/lib/sidekiq/component.rb:17:in `block in safe_thread'",
"chart": "sidekiq-7.2.2",
"class": "Elastic::MigrationWorker",
"component": "gitlab",
"correlation_id": "a38ec0e357cc80d5f184261928f10f04",
"environment": "aws",
"heritage": "Helm",
"jid": "b8bcc3c0bd6514fbe9c8afa8",
"job_status": "running",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c82ef28c-f814-4a19-9217-1dd68b4c5363",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47",
"level": "error",
"logtag": "F",
"message": "Elastic::MigrationWorker: Elasticsearch::Transport::Transport::Errors::BadRequest [400] {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:350] [reindex] failed to parse field [source]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"max must be greater than 1\"}}},\"status\":400}",
"meta_caller_id": "Elastic::MigrationWorker",
"meta_client_id": "ip/",
"meta_feature_category": "global_search",
"meta_root_caller_id": "Cronjob",
"namespace": "gitlab",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"severity": "ERROR",
"stream": "stderr",
"subcomponent": "elasticsearch",
"time": "2023-08-03T14:00:04.059Z"
}
},
{
"line": "[Elastic::Migration: 20230530500000] Setting migration_state to {\"slice\":0,\"task_id\":null,\"retry_attempt\":4,\"max_slices\":1}",
"timestamp": "1691071203967933691",
"fields": {
"Line": "[Elastic::Migration: 20230530500000] Setting migration_state to {\"slice\":0,\"task_id\":null,\"retry_attempt\":4,\"max_slices\":1}",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47_gitlab_sidekiq-6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "MigrateProjectsToSeparateIndex",
"component": "gitlab",
"correlation_id": "a38ec0e357cc80d5f184261928f10f04",
"environment": "aws",
"heritage": "Helm",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c82ef28c-f814-4a19-9217-1dd68b4c5363",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47",
"level": "info",
"logtag": "F",
"message": "[Elastic::Migration: 20230530500000] Setting migration_state to {\"slice\":0,\"task_id\":null,\"retry_attempt\":4,\"max_slices\":1}",
"meta_caller_id": "Elastic::MigrationWorker",
"meta_client_id": "ip/",
"meta_feature_category": "global_search",
"meta_root_caller_id": "Cronjob",
"namespace": "gitlab",
"pod_template_hash": "666cd666cc",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"severity": "INFO",
"stream": "stderr",
"subcomponent": "elasticsearch",
"time": "2023-08-03T14:00:03.967Z"
}
},
{
"line": "[Elastic::Migration: 20230530500000] migrate failed, increasing migration_state for slice: 0 retry_attempt: 3 error: [400] {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:350] [reindex] failed to parse field [source]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"max must be greater than 1\"}}},\"status\":400}",
"timestamp": "1691071203967832200",
"fields": {
"Line": "[Elastic::Migration: 20230530500000] migrate failed, increasing migration_state for slice: 0 retry_attempt: 3 error: [400] {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:350] [reindex] failed to parse field [source]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"max must be greater than 1\"}}},\"status\":400}",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47_gitlab_sidekiq-6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "MigrateProjectsToSeparateIndex",
"component": "gitlab",
"correlation_id": "a38ec0e357cc80d5f184261928f10f04",
"environment": "aws",
"heritage": "Helm",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c82ef28c-f814-4a19-9217-1dd68b4c5363",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47",
"level": "error",
"logtag": "F",
"message": "[Elastic::Migration: 20230530500000] migrate failed, increasing migration_state for slice: 0 retry_attempt: 3 error: [400] {\"error\":{\"root_cause\":[{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\"}],\"type\":\"x_content_parse_exception\",\"reason\":\"[1:350] [reindex] failed to parse field [source]\",\"caused_by\":{\"type\":\"x_content_parse_exception\",\"reason\":\"[1:17] [slice] failed to parse field [max]\",\"caused_by\":{\"type\":\"illegal_argument_exception\",\"reason\":\"max must be greater than 1\"}}},\"status\":400}",
"meta_caller_id": "Elastic::MigrationWorker",
"meta_client_id": "ip/",
"meta_feature_category": "global_search",
"meta_root_caller_id": "Cronjob",
"namespace": "gitlab",
"pod_template_hash": "666cd666cc",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"severity": "INFO",
"stream": "stderr",
"subcomponent": "elasticsearch",
"time": "2023-08-03T14:00:03.967Z"
}
},
{
"line": "[Elastic::Migration: 20230530500000] Launching reindexing for slice: 0 | max_slices: 1",
"timestamp": "1691071203951614950",
"fields": {
"Line": "[Elastic::Migration: 20230530500000] Launching reindexing for slice: 0 | max_slices: 1",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47_gitlab_sidekiq-6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "MigrateProjectsToSeparateIndex",
"component": "gitlab",
"correlation_id": "a38ec0e357cc80d5f184261928f10f04",
"environment": "aws",
"heritage": "Helm",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c82ef28c-f814-4a19-9217-1dd68b4c5363",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47",
"level": "info",
"logtag": "F",
"message": "[Elastic::Migration: 20230530500000] Launching reindexing for slice: 0 | max_slices: 1",
"meta_caller_id": "Elastic::MigrationWorker",
"meta_client_id": "ip/",
"meta_feature_category": "global_search",
"meta_root_caller_id": "Cronjob",
"namespace": "gitlab",
"pod_template_hash": "666cd666cc",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"severity": "INFO",
"stream": "stderr",
"subcomponent": "elasticsearch",
"time": "2023-08-03T14:00:03.951Z"
}
},
{
"line": "MigrationWorker: migration[MigrateProjectsToSeparateIndex] executing migrate method",
"timestamp": "1691071203813119242",
"fields": {
"Line": "MigrationWorker: migration[MigrateProjectsToSeparateIndex] executing migrate method",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47_gitlab_sidekiq-6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Elastic::MigrationWorker",
"component": "gitlab",
"correlation_id": "a38ec0e357cc80d5f184261928f10f04",
"environment": "aws",
"heritage": "Helm",
"jid": "b8bcc3c0bd6514fbe9c8afa8",
"job_status": "running",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c82ef28c-f814-4a19-9217-1dd68b4c5363",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47",
"level": "info",
"logtag": "F",
"message": "MigrationWorker: migration[MigrateProjectsToSeparateIndex] executing migrate method",
"meta_caller_id": "Elastic::MigrationWorker",
"meta_client_id": "ip/",
"meta_feature_category": "global_search",
"meta_root_caller_id": "Cronjob",
"namespace": "gitlab",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"severity": "INFO",
"stream": "stderr",
"subcomponent": "elasticsearch",
"time": "2023-08-03T14:00:03.812Z"
}
},
{
"line": "Elastic::MigrationWorker JID-b8bcc3c0bd6514fbe9c8afa8: start",
"timestamp": "1691071203726940651",
"fields": {
"Line": "Elastic::MigrationWorker JID-b8bcc3c0bd6514fbe9c8afa8: start",
"_tail_path": "/var/log/containers/gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47_gitlab_sidekiq-6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821.log",
"app": "sidekiq",
"chart": "sidekiq-7.2.2",
"class": "Elastic::MigrationWorker",
"correlation_id": "a38ec0e357cc80d5f184261928f10f04",
"created_at": "2023-08-03T14:00:03.721Z",
"enqueued_at": "2023-08-03T14:00:03.723Z",
"environment": "aws",
"heritage": "Helm",
"idempotency_key": "resque:gitlab:duplicate:default:18a1564991a290ba236afceb12ec6c3c0dcffaff06178046cbd7d22b5ef4ab9b",
"jid": "b8bcc3c0bd6514fbe9c8afa8",
"job_size_bytes": "2",
"job_status": "start",
"kubernetes_annotations_checksum_configmap": "df7cde15a80a423b0e50c0af97b1206ba4b27f01472155863d9d1e4470be6198",
"kubernetes_annotations_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
"kubernetes_annotations_gitlab_com_prometheus_path": "/metrics",
"kubernetes_annotations_gitlab_com_prometheus_port": "3807",
"kubernetes_annotations_gitlab_com_prometheus_scrape": "true",
"kubernetes_annotations_prometheus_io_path": "/metrics",
"kubernetes_annotations_prometheus_io_port": "3807",
"kubernetes_annotations_prometheus_io_scrape": "true",
"kubernetes_container_hash": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee@sha256:9f716fec9e05fcf625e41c2941f51ab1cc3f3ba4dd27b21c5514e6710066f6cc",
"kubernetes_container_image": "registry.gitlab.com/gitlab-org/build/cng/gitlab-sidekiq-ee:v16.2.2",
"kubernetes_container_name": "sidekiq",
"kubernetes_docker_id": "6e0383996eea9873484ce14702f98da5ec791934d24172a158480c568d5b5821",
"kubernetes_host": "[FILTERED]",
"kubernetes_labels_app": "sidekiq",
"kubernetes_labels_chart": "sidekiq-7.2.2",
"kubernetes_labels_heritage": "Helm",
"kubernetes_labels_pod_template_hash": "666cd666cc",
"kubernetes_labels_queue_pod_name": "all-in-1",
"kubernetes_labels_release": "gitlab",
"kubernetes_namespace_name": "gitlab",
"kubernetes_pod_id": "c82ef28c-f814-4a19-9217-1dd68b4c5363",
"kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-666cd666cc-cjp47",
"logtag": "F",
"message": "Elastic::MigrationWorker JID-b8bcc3c0bd6514fbe9c8afa8: start",
"meta_caller_id": "Cronjob",
"meta_feature_category": "global_search",
"meta_root_caller_id": "Cronjob",
"namespace": "gitlab",
"pid": "24",
"pod_template_hash": "666cd666cc",
"queue": "default",
"queue_namespace": "cronjob",
"queue_pod_name": "all-in-1",
"release": "gitlab",
"retry": "0",
"scheduling_latency_s": "0.002514",
"severity": "INFO",
"size_limiter": "validated",
"stream": "stdout",
"time": "2023-08-03T14:00:03.726Z",
"version": "0",
"worker_data_consistency": "always"
}
}
]