Gitlab 14.9.3 BackfillProjectNamespaces Migration Fails - Finalize not working
Hi everyone,
in a gitlab 14.9.3 installation running on kubernetes, the ProjectNamespaces:BackfillProjectNamespaces:projects keeps failing.
Checking logs, I can see an ERROR regarding a particolar index_projects_on_id_partial_for_visibility:
{
"severity": "ERROR",
"time": "2022-05-05T17:42:11.663Z",
"correlation_id": "2efa593fe0d072a9fa1c8572d96124d0",
"exception.class": "ActiveRecord::StatementInvalid",
"exception.message": **"**PG::WrongObjectType: ERROR: \"index_projects_on_id_partial_for_visibility\" is not a GIN index**\n",**
"exception.backtrace": [
"lib/gitlab/database/load_balancing/connection_proxy.rb:126:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:112:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:172:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:110:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:125:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:95:in `method_missing'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:63:in `block in cleanup_gin_index'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:62:in `each'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:62:in `cleanup_gin_index'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:41:in `block in backfill_project_namespaces'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:36:in `each'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:36:in `each_slice'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:36:in `backfill_project_namespaces'",
"lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb:21:in `perform'",
"lib/gitlab/database/background_migration/batched_migration_wrapper.rb:45:in `execute_batch'",
"lib/gitlab/database/background_migration/batched_migration_wrapper.rb:23:in `perform'",
"lib/gitlab/database/background_migration/batched_migration_runner.rb:30:in `run_migration_job'",
"app/workers/database/batched_background_migration/single_database_worker.rb:66:in `run_active_migration'",
"app/workers/database/batched_background_migration/single_database_worker.rb:54:in `block (2 levels) in perform'",
"app/workers/database/batched_background_migration/single_database_worker.rb:77:in `with_exclusive_lease'",
"app/workers/database/batched_background_migration/single_database_worker.rb:46:in `block in perform'",
"lib/gitlab/database/shared_model.rb:23:in `using_connection'",
"app/workers/database/batched_background_migration/single_database_worker.rb:43:in `perform'",
"lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:26:in `call'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:58:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/application_context.rb:88:in `block in use'",
"lib/gitlab/application_context.rb:88:in `use'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:11:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `block in call'",
"lib/gitlab/application_context.rb:88:in `block in use'",
"lib/gitlab/application_context.rb:88:in `use'",
"lib/gitlab/application_context.rb:41:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:46:in `within'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
"lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
"lib/gitlab/with_request_store.rb:10:in `with_request_store'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:74:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:97:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:33:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:97:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:73:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
],
"exception.sql": "/*application:sidekiq,correlation_id:2efa593fe0d072a9fa1c8572d96124d0,jid:84b97022cbd7e17a8ffe5a16,endpoint_id:Database::BatchedBackgroundMigrationWorker,db_config_name:main*/ select gin_clean_pending_list($1)",
"user.username": null,
"tags.program": "sidekiq",
"tags.locale": "en",
"tags.feature_category": "database",
"tags.correlation_id": "2efa593fe0d072a9fa1c8572d96124d0",
"extra.sidekiq": {
"retry": false,
"queue": "cronjob:database_batched_background_migration",
"version": 0,
"queue_namespace": "cronjob",
"args": [],
"class": "Database::BatchedBackgroundMigrationWorker",
"jid": "84b97022cbd7e17a8ffe5a16",
"created_at": 1651772531.4898553,
"meta.caller_id": "Cronjob",
"correlation_id": "2efa593fe0d072a9fa1c8572d96124d0",
"meta.feature_category": "database",
"worker_data_consistency": "always",
"idempotency_key": "resque:gitlab:duplicate:cronjob:database_batched_background_migration:592d9619e1997b640b70ce6a22f6713bc7793bb7a4e342b7380d90b691fcd6ae",
"enqueued_at": 1651772531.4918294
},
"extra.batched_job_id": 129
}
Checking Job's migration code, The error seems to be related to a query run by the migration job, looking for a pattern matching:
select indexname::text from pg_indexes where tablename = 'projects' and indexdef ilike '%gin%
The pattern matching "%gin%" returns the wrong index list blocking the job migration, checking the index_projects_on_id_partial_for_visibility index, i can see WHY the select lauched the job is fooled:
select indexname::text,indexdef from pg_indexes where tablename = 'projects' and indexname = 'index_projects_on_id_partial_for_visibility'"
indexname | indexdef
---------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------
-----------------------------------------
index_projects_on_id_partial_for_visibility | CREATE UNIQUE INDEX index_projects_on_id_partial_for_visibility ON public.projects USING btree (id) WHERE (visibility_level =
**ANY (ARRAY[(10)::bigint, (20)::bigint]))**
(1 row)
Unfortunately this migration is blocking our planned upgrade to the 14.10.1 version.
Any help would be kindly appriciated.
Thank you