Geo: Cannot transition state via :start from :failed due to "Verification failure can't be blank"

Summary

GitLab Version 17.11.6.

In the GEO environment, the following error frequently occurs in the synchronous state:

  • There were an excessive number of retries.
  • And while there was an error reason in the Sync phase, none was present in the verification phase.

image

Click to view the complete error log 
{
  "severity": "WARN",
  "time": "2025-08-12T20:44:04.928Z",
  "retry": 0,
  "queue": "default",
  "version": 0,
  "store": null,
  "queue_namespace": "geo",
  "dead": false,
  "status_expiration": 1800,
  "args": [
    "project_repository",
    "11018"
  ],
  "class": "Geo::SyncWorker",
  "jid": "62072d3989706f46e134c791",
  "created_at": "2025-08-12T20:44:04.465Z",
  "trace_propagation_headers": {
    "sentry-trace": "134883f491544a7b827f561194e925ba-550124174c824d46",
    "baggage": "sentry-trace_id=134883f491544a7b827f561194e925ba,sentry-release=40205c1ddfe"
  },
  "meta.caller_id": "Geo::RepositoryRegistrySyncWorker",
  "correlation_id": "f8d59e24ea00161db21d0f00314f97cb",
  "meta.root_caller_id": "Cronjob",
  "meta.feature_category": "geo_replication",
  "meta.client_id": "ip/",
  "worker_data_consistency": "sticky",
  "worker_data_consistency_per_db": {
    "main": "sticky",
    "ci": "sticky"
  },
  "wal_locations": {},
  "wal_location_sources": {
    "main": "primary",
    "ci": "primary"
  },
  "idempotency_key": "resque:gitlab:duplicate:default:d72492f498ff0c661846c4fddf2117c38d1d1b69435237d4f425fae7c8c25221",
  "size_limiter": "validated",
  "enqueued_at": "2025-08-12T20:44:04.470Z",
  "job_size_bytes": 28,
  "pid": 37896,
  "sidekiq_tid": "47q1j0k",
  "sidekiq_thread_name": "sidekiq.default/processor",
  "message": "Geo::SyncWorker JID-62072d3989706f46e134c791: fail: 0.452787 sec",
  "job_status": "fail",
  "queue_duration_s": 0.00559,
  "scheduling_latency_s": 0.005613,
  "gitaly_calls": 3,
  "gitaly_duration_s": 0.02934,
  "redis_calls": 84,
  "redis_allowed_cross_slot_calls": 8,
  "redis_duration_s": 0.361431,
  "redis_read_bytes": 680,
  "redis_write_bytes": 9231,
  "redis_cache_calls": 1,
  "redis_cache_duration_s": 9.9e-05,
  "redis_cache_read_bytes": 1,
  "redis_cache_write_bytes": 49,
  "redis_feature_flag_calls": 3,
  "redis_feature_flag_duration_s": 0.000686,
  "redis_feature_flag_read_bytes": 596,
  "redis_feature_flag_write_bytes": 188,
  "redis_queues_metadata_calls": 1,
  "redis_queues_metadata_duration_s": 0.000934,
  "redis_queues_metadata_read_bytes": 1,
  "redis_queues_metadata_write_bytes": 123,
  "redis_repository_cache_calls": 72,
  "redis_repository_cache_allowed_cross_slot_calls": 8,
  "redis_repository_cache_duration_s": 0.317554,
  "redis_repository_cache_read_bytes": 71,
  "redis_repository_cache_write_bytes": 8348,
  "redis_shared_state_calls": 7,
  "redis_shared_state_duration_s": 0.042158,
  "redis_shared_state_read_bytes": 11,
  "redis_shared_state_write_bytes": 523,
  "db_count": 3,
  "db_write_count": 0,
  "db_cached_count": 0,
  "db_txn_count": 1,
  "db_replica_txn_count": 0,
  "db_primary_txn_count": 0,
  "db_replica_count": 0,
  "db_primary_count": 3,
  "db_replica_write_count": 0,
  "db_primary_write_count": 0,
  "db_replica_cached_count": 0,
  "db_primary_cached_count": 0,
  "db_replica_wal_count": 0,
  "db_primary_wal_count": 0,
  "db_replica_wal_cached_count": 0,
  "db_primary_wal_cached_count": 0,
  "db_replica_txn_max_duration_s": 0.0,
  "db_primary_txn_max_duration_s": 0.0,
  "db_replica_txn_duration_s": 0.0,
  "db_primary_txn_duration_s": 0.0,
  "db_replica_duration_s": 0.0,
  "db_primary_duration_s": 0.015,
  "db_main_txn_count": 0,
  "db_geo_txn_count": 1,
  "db_ci_txn_count": 0,
  "db_main_replica_txn_count": 0,
  "db_ci_replica_txn_count": 0,
  "db_main_count": 2,
  "db_geo_count": 1,
  "db_ci_count": 0,
  "db_main_replica_count": 0,
  "db_ci_replica_count": 0,
  "db_main_write_count": 0,
  "db_geo_write_count": 0,
  "db_ci_write_count": 0,
  "db_main_replica_write_count": 0,
  "db_ci_replica_write_count": 0,
  "db_main_cached_count": 0,
  "db_geo_cached_count": 0,
  "db_ci_cached_count": 0,
  "db_main_replica_cached_count": 0,
  "db_ci_replica_cached_count": 0,
  "db_main_wal_count": 0,
  "db_geo_wal_count": 0,
  "db_ci_wal_count": 0,
  "db_main_replica_wal_count": 0,
  "db_ci_replica_wal_count": 0,
  "db_main_wal_cached_count": 0,
  "db_geo_wal_cached_count": 0,
  "db_ci_wal_cached_count": 0,
  "db_main_replica_wal_cached_count": 0,
  "db_ci_replica_wal_cached_count": 0,
  "db_main_txn_max_duration_s": 0.0,
  "db_geo_txn_max_duration_s": 0.001,
  "db_ci_txn_max_duration_s": 0.0,
  "db_main_replica_txn_max_duration_s": 0.0,
  "db_ci_replica_txn_max_duration_s": 0.0,
  "db_main_txn_duration_s": 0.0,
  "db_geo_txn_duration_s": 0.001,
  "db_ci_txn_duration_s": 0.0,
  "db_main_replica_txn_duration_s": 0.0,
  "db_ci_replica_txn_duration_s": 0.0,
  "db_main_duration_s": 0.012,
  "db_geo_duration_s": 0.003,
  "db_ci_duration_s": 0.0,
  "db_main_replica_duration_s": 0.0,
  "db_ci_replica_duration_s": 0.0,
  "cpu_s": 0.038194,
  "mem_objects": 18222,
  "mem_bytes": 912816,
  "mem_mallocs": 3550,
  "mem_total_bytes": 1641696,
  "worker_id": "sidekiq_1",
  "rate_limiting_gates": [],
  "exclusive_lock_requested_count": 1,
  "exclusive_lock_wait_duration_s": 0,
  "exclusive_lock_hold_duration_s": 0.39683591,
  "duration_s": 0.452787,
  "completed_at": "2025-08-12T20:44:04.928Z",
  "load_balancing_strategy": "primary_no_wal",
  "exception.class": "StateMachines::InvalidTransition",
  "exception.message": "Cannot transition state via :start from :failed (Reason(s): Verification failure can't be blank)",
  "exception.backtrace": [
    "state_machines (0.5.0) lib/state_machines/event.rb:224:in `block in add_actions'",
    "state_machines (0.5.0) lib/state_machines/machine.rb:729:in `block (2 levels) in define_helper'",
    "ee/app/services/geo/framework_repository_sync_service.rb:116:in `start_registry_sync!'",
    "ee/app/services/geo/framework_repository_sync_service.rb:40:in `sync_repository'",
    "ee/app/services/geo/framework_repository_sync_service.rb:28:in `block in execute'",
    "app/services/concerns/exclusive_lease_guard.rb:32:in `try_obtain_lease'",
    "ee/app/services/geo/framework_repository_sync_service.rb:25:in `execute'",
    "ee/app/models/concerns/geo/repository_replicator_strategy.rb:78:in `sync'",
    "ee/app/services/geo/sync_service.rb:16:in `execute'",
    "ee/app/workers/geo/sync_worker.rb:18:in `perform'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:220:in `execute_job'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:185:in `block (4 levels) in process'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:180:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "ee/lib/gitlab/sidekiq_middleware/set_session/server.rb:21:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/identity/restore.rb:12:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/skip_jobs.rb:51:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/resource_usage_limit/middleware.rb:16:in `perform'",
    "lib/gitlab/sidekiq_middleware/resource_usage_limit/server.rb:8:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:35:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/concurrency_limit/middleware.rb:37:in `perform'",
    "lib/gitlab/sidekiq_middleware/concurrency_limit/server.rb:8:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
    "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/click_house/migration_support/sidekiq_middleware.rb:7:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/pause_control/strategies/base.rb:31:in `perform'",
    "lib/gitlab/sidekiq_middleware/pause_control/strategy_handler.rb:22:in `perform'",
    "lib/gitlab/sidekiq_middleware/pause_control/server.rb:8:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
    "lib/gitlab/application_context.rb:173:in `block in use'",
    "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
    "lib/gitlab/application_context.rb:173:in `use'",
    "lib/gitlab/application_context.rb:96:in `with_context'",
    "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
    "lib/gitlab/database/query_analyzer.rb:83:in `within'",
    "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/set_ip_address.rb:7:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:111:in `block in call'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `block in instrument'",
    "lib/gitlab/metrics/background_transaction.rb:33:in `run'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:139:in `instrument'",
    "lib/gitlab/sidekiq_middleware/server_metrics.rb:110:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:8:in `block in call'",
    "gems/gitlab-safe_request_store/lib/gitlab/safe_request_store.rb:66:in `enabling_request_store'",
    "gems/gitlab-safe_request_store/lib/gitlab/safe_request_store.rb:59:in `ensure_request_store'",
    "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:7:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/server.rb:21:in `block in call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:180:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/server.rb:16:in `block in call'",
    "gitlab-labkit (0.37.0) lib/labkit/context.rb:35:in `with_context'",
    "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/context/server.rb:15:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
    "gitlab-labkit (0.37.0) lib/labkit/middleware/sidekiq/server.rb:20:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
    "lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
    "lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:10:in `block in call'",
    "lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
    "lib/gitlab/sidekiq_middleware/shard_awareness_validator.rb:9:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "marginalia (1.11.1) lib/marginalia/sidekiq_instrumentation.rb:9:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "sentry-sidekiq (5.22.1) lib/sentry/sidekiq/sentry_context_middleware.rb:54:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/job/interrupt_handler.rb:9:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:183:in `block in traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/metrics/tracking.rb:26:in `track'",
    "vendor/gems/sidekiq/lib/sidekiq/metrics/tracking.rb:134:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:182:in `traverse'",
    "vendor/gems/sidekiq/lib/sidekiq/middleware/chain.rb:173:in `invoke'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:184:in `block (3 levels) in process'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:145:in `block (6 levels) in dispatch'",
    "vendor/gems/sidekiq/lib/sidekiq/job_retry.rb:118:in `local'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:144:in `block (5 levels) in dispatch'",
    "vendor/gems/sidekiq/lib/sidekiq/rails.rb:27:in `block in call'",
    "activesupport (7.0.8.7) lib/active_support/execution_wrapper.rb:92:in `wrap'",
    "activesupport (7.0.8.7) lib/active_support/reloader.rb:72:in `block in wrap'",
    "activesupport (7.0.8.7) lib/active_support/execution_wrapper.rb:92:in `wrap'",
    "activesupport (7.0.8.7) lib/active_support/reloader.rb:71:in `wrap'",
    "vendor/gems/sidekiq/lib/sidekiq/rails.rb:26:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:139:in `block (4 levels) in dispatch'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:281:in `stats'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:134:in `block (3 levels) in dispatch'",
    "lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:133:in `block (2 levels) in dispatch'",
    "vendor/gems/sidekiq/lib/sidekiq/job_retry.rb:85:in `global'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:132:in `block in dispatch'",
    "vendor/gems/sidekiq/lib/sidekiq/job_logger.rb:40:in `prepare'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:131:in `dispatch'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:183:in `block (2 levels) in process'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:182:in `handle_interrupt'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:182:in `block in process'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:181:in `handle_interrupt'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:181:in `process'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:86:in `process_one'",
    "vendor/gems/sidekiq/lib/sidekiq/processor.rb:76:in `run'",
    "vendor/gems/sidekiq/lib/sidekiq/component.rb:10:in `watchdog'",
    "vendor/gems/sidekiq/lib/sidekiq/component.rb:19:in `block in safe_thread'"
  ],
  "db_duration_s": 0.014988,
  "urgency": "low",
  "target_duration_s": 300,
  "target_scheduling_latency_s": 60
}

Steps to reproduce

To be honest, I'm not sure. It occurs spontaneously and still persists after reinstalling GEO, but I haven't figured out how to reproduce it in the test environment.

Example Project

What is the current bug behavior?

The GEO synchronization failed, and repeated attempts to do so have also been unsuccessful.

What is the expected correct behavior?

Looking forward to a successful synchronization. Even if occasional synchronization failures occur, they can be quickly retried and succeed.

Relevant logs and/or screenshots

Output of checks

Results of GitLab environment info

Expand for output related to GitLab environment info 
(For installations with omnibus-gitlab package run and paste the output of:
`sudo gitlab-rake gitlab:env:info`)

(For installations from source run and paste the output of:
`sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production`)

Results of GitLab application Check

Expand for output related to the GitLab application check 
(For installations with omnibus-gitlab package run and paste the output of:
sudo gitlab-rake gitlab:check SANITIZE=true)


(For installations from source run and paste the output of:
sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production SANITIZE=true)


(we will only investigate if the tests are passing)

Possible fixes

Status: Confirmed still valid. This is the same invalid data state as #433182 (closed) (verification_state = verification_failed with a NULL/blank verification_failure), but it surfaces on a different sync transition that was never patched by !143158 (merged).

Root cause

  • The verification_failed state declares validates :verification_failure, presence: true (ee/app/models/concerns/geo/verification_state_definition.rb), so any validated save of a registry row in verification_failed with a blank verification_failure raises Verification failure can't be blank.
  • !143158 (merged) (which closed #433182 (closed)) fixed this for the pending sync event by making before_pending call verification_disabled (ee/app/models/concerns/geo/verifiable_registry.rb), and by widening the failed event.
  • The error here is on the start sync event. The retry-of-a-failed-registry path calls registry.start! directly via Geo::FrameworkRepositorySyncService#start_registry_sync! (see backtrace above), transitioning failed → started. The start transition is allowed from failed (ee/app/models/concerns/geo/replicable_registry.rb), but its before_transition any => :started callback only sets last_synced_at — it has no verification_disabled equivalent. So the invalid row fails validation during the start! save. This transition was never covered by !143158 (merged).

Proposed fix (mirrors !143158 (merged))

  1. In Geo::ReplicableRegistry, extract the started transition body into an overridable before_started method (mirroring the existing before_pending), and have before_transition any => :started call registry.before_started.
  2. In Geo::VerifiableRegistry, override before_started to call super and then verification_disabled — identical in shape to the existing before_pending override. This disables verification in-memory before the start! save validates, so the stale/invalid verification_failed state no longer blocks the re-sync. (after_synced re-drives verification once the sync completes, consistent with !143158 (merged)'s rationale that verification state only makes sense once a registry is synced.)
  3. Tests in the Geo verifiable/framework registry shared examples, including a regression test for a registry forced into verification_failed with a blank verification_failure (set up via update_column to bypass validations, the way these rows exist in production): start! must not raise and must land in verification_disabled.

Patch release information for backports

If the bug fix needs to be backported in a patch release to a version under the maintenance policy, please follow the steps on the patch release runbook for GitLab engineers.

Refer to the internal "Release Information" dashboard for information about the next patch release, including the targeted versions, expected release date, and current status.

High-severity bug remediation

To remediate high-severity issues requiring an internal release for single-tenant SaaS instances, refer to the internal release process for engineers.

Workaround to unstick currently stuck records

Paste this into Rails console in the secondary site:

def fix_verification_failure_cant_be_blank_for_registry(registry_class)
  affected = registry_class.where(verification_state: 3, verification_failure: nil)
  puts "Found #{affected.count} affected #{registry_class} records"
  
  update_attrs = {
    state: 3,
    last_sync_failure: "Manually setting sync failed",
    retry_count: 1,
    retry_at: registry_class.next_retry_time(1),
    verification_state: 4,
    verification_retry_count: 1,
    verification_retry_at: nil,
    verified_at: nil
  }

  affected.each_batch do |batch|
    num_updated = batch.update_all(update_attrs)
    puts "Manually set sync failed for #{num_updated} #{registry_class} records"
  end
end

def fix_verification_failure_cant_be_blank
  registry_classes = ::Gitlab::Geo.verification_enabled_replicator_classes.map(&:registry_class)

  registry_classes.each do |registry_class|
    fix_verification_failure_cant_be_blank_for_registry(registry_class)
  end

  while true do
    total_waiting = registry_classes.reduce(0) do |sum, registry_class|
      num_waiting = registry_class.where(state: 3, verification_state: 4).count
      puts "#{num_waiting} #{registry_class} affected records waiting to be resynced" if num_waiting > 0
      sum + num_waiting
    end

    break if total_waiting.zero?

    sleep 3
  end

  puts "Done"
end

fix_verification_failure_cant_be_blank
Edited by 🤖 GitLab Bot 🤖