Repository kept read only after repository move

Overview

In multiple instances such as gitlab-com/runbooks!5178 (comment 1196152397) and https://gitlab.com/gitlab-com/gl-infra/capacity-planning/-/issues/428#note_1198485442 we had failures when we scheduled a repository to move storage servers. When it failed the repository was left read only and didn't move back to read-write.

Both failures seem to be for Cannot transition state via :finish_replication from :replicated (Reason(s): State cannot transition via "finish replication")

full log

{
  "_index": "pubsub-sidekiq-inf-gstg-000426",
  "_type": "_doc",
  "_id": "oL2S4YQBnNEWeQqYBMqJ",
  "_version": 1,
  "_score": 1,
  "_source": {
    "@timestamp": "2022-12-05T09:15:52.846Z",
    "publish_time": "2022-12-05T09:15:52.822Z",
    "host": {
      "name": "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    },
    "kubernetes": {
      "container_image": "dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-sidekiq-ee:15-7-202212050320-34b74c83da5",
      "pod_ip": "10.227.5.77",
      "region": "us-east1",
      "pod_name": "gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4",
      "namespace_name": "gitlab",
      "host": "gke-gstg-gitlab-gke-generic-2-89a5eb4d-3bkd",
      "container_name": "sidekiq"
    },
    "json": {
      "db_primary_cached_count": 0,
      "db_main_wal_count": 0,
      "redis_shared_state_read_bytes": 93,
      "redis_queues_read_bytes": 1,
      "load_balancing_strategy": "primary",
      "tier": "sv",
      "db_ci_replica_wal_count": 0,
      "job_size_bytes": 34,
      "db_main_duration_s": 0.007,
      "size_limiter": "validated",
      "meta.root_caller_id": "POST /api/:version/projects/:id/repository_storage_moves",
      "jid": "393d104e7b3d3252b6f43fe4",
      "db_ci_replica_count": 0,
      "version": 0,
      "retry_count": 0,
      "db_main_wal_cached_count": 0,
      "meta.project": "steveazz/gitlab",
      "db_write_count": 3,
      "db_ci_replica_duration_s": 0,
      "args": [
        "8302702",
        "[FILTERED]",
        "2233127"
      ],
      "scheduling_latency_s": 0.001216,
      "severity": "INFO",
      "tag": "gitlab-sidekiq-gitaly-throttled.var.log.containers.gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4_gitlab_sidekiq-c00161bbe79d6512983eaccc06af274e51e1088efad5995783880244605fa918.log",
      "redis_shared_state_write_bytes": 179,
      "rate_limiting_gates": [],
      "meta.root_namespace": "steveazz",
      "meta.caller_id": "POST /api/:version/projects/:id/repository_storage_moves",
      "message": "Projects::UpdateRepositoryStorageWorker JID-393d104e7b3d3252b6f43fe4: done: 0.075734 sec",
      "db_replica_count": 0,
      "db_replica_wal_count": 0,
      "redis_cache_read_bytes": 1461,
      "db_ci_wal_cached_count": 0,
      "redis_shared_state_duration_s": 0.02882,
      "db_duration_s": 0.012159,
      "meta.remote_ip": "94.17.156.48",
      "redis_cache_calls": 7,
      "db_cached_count": 0,
      "redis_shared_state_calls": 4,
      "db_main_count": 4,
      "shard": "gitaly-throttled",
      "db_main_replica_wal_count": 0,
      "created_at": "2022-12-05T09:05:49.372Z",
      "class": "Projects::UpdateRepositoryStorageWorker",
      "db_ci_replica_wal_cached_count": 0,
      "db_main_replica_duration_s": 0,
      "db_primary_count": 4,
      "redis_duration_s": 0.032829,
      "failed_at": "2022-12-05T09:15:01.273Z",
      "stage": "main",
      "retry": 3,
      "meta.user": "steveazz",
      "meta.client_id": "user/210928",
      "db_primary_wal_count": 0,
      "db_main_replica_cached_count": 0,
      "environment": "gstg",
      "db_replica_duration_s": 0,
      "db_primary_duration_s": 0.007,
      "db_primary_wal_cached_count": 0,
      "db_ci_duration_s": 0,
      "db_replica_wal_cached_count": 0,
      "db_count": 4,
      "redis_queues_duration_s": 0.000448,
      "mem_bytes": 740984,
      "time": "2022-12-05T09:15:22.580Z",
      "job_status": "done",
      "db_main_replica_wal_cached_count": 0,
      "mem_mallocs": 1601,
      "db_main_cached_count": 0,
      "mem_objects": 6339,
      "queue": "gitaly_throttled",
      "mem_total_bytes": 994544,
      "idempotency_key": "resque:gitlab:duplicate:gitaly_throttled:9e687ad2de1a6e7d65478d9943661b8d24fbb9ee0d89d319e83640a66cca0ea0",
      "completed_at": "2022-12-05T09:15:22.580Z",
      "redis_cache_duration_s": 0.003561,
      "correlation_id": "2c8c9390e558aee813d064c571abdc69",
      "db_ci_count": 0,
      "redis_read_bytes": 1555,
      "db_ci_wal_count": 0,
      "db_replica_cached_count": 0,
      "enqueued_at": "2022-12-05T09:15:22.504Z",
      "redis_queues_write_bytes": 132,
      "worker_data_consistency": "always",
      "db_main_replica_count": 0,
      "worker_id": "sidekiq_0",
      "type": "sidekiq",
      "redis_cache_write_bytes": 542,
      "redis_calls": 12,
      "cpu_s": 0.018574,
      "db_ci_replica_cached_count": 0,
      "error_class": "StateMachines::InvalidTransition",
      "db_ci_cached_count": 0,
      "redis_write_bytes": 853,
      "meta.user_id": 210928,
      "pid": 36,
      "redis_queues_calls": 1,
      "error_message": "Cannot transition state via :finish_replication from :replicated (Reason(s): State cannot transition via \"finish replication\")",
      "meta.feature_category": "gitaly",
      "duration_s": 0.075734
    },
    "type": "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
  },
  "fields": {
    "json.type.keyword": [
      "sidekiq"
    ],
    "json.idempotency_key": [
      "resque:gitlab:duplicate:gitaly_throttled:9e687ad2de1a6e7d65478d9943661b8d24fbb9ee0d89d319e83640a66cca0ea0"
    ],
    "json.db_replica_wal_cached_count": [
      0
    ],
    "json.mem_mallocs": [
      1601
    ],
    "json.redis_duration_s": [
      0.032829
    ],
    "host.name.keyword": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "json.db_primary_cached_count": [
      0
    ],
    "json.db_replica_count": [
      0
    ],
    "json.mem_total_bytes": [
      994544
    ],
    "type": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "json.db_main_wal_count": [
      0
    ],
    "json.redis_cache_read_bytes": [
      1461
    ],
    "json.mem_bytes": [
      740984
    ],
    "json.redis_cache_write_bytes": [
      542
    ],
    "publish_time": [
      "2022-12-05T09:15:52.822Z"
    ],
    "json.redis_queues_read_bytes": [
      1
    ],
    "json.db_primary_wal_cached_count": [
      0
    ],
    "json.worker_data_consistency": [
      "always"
    ],
    "json.meta.caller_id": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.version": [
      0
    ],
    "json.class": [
      "Projects::UpdateRepositoryStorageWorker"
    ],
    "json.args.keyword": [
      "8302702",
      "[FILTERED]",
      "2233127"
    ],
    "json.jid.keyword": [
      "393d104e7b3d3252b6f43fe4"
    ],
    "json.meta.root_namespace.keyword": [
      "steveazz"
    ],
    "json.queue": [
      "gitaly_throttled"
    ],
    "json.size_limiter": [
      "validated"
    ],
    "json.db_replica_cached_count": [
      0
    ],
    "json.db_ci_replica_count": [
      0
    ],
    "json.db_ci_cached_count": [
      0
    ],
    "json.environment": [
      "gstg"
    ],
    "json.db_primary_duration_s": [
      0.007
    ],
    "json.size_limiter.keyword": [
      "validated"
    ],
    "kubernetes.pod_ip": [
      "10.227.5.77"
    ],
    "json.type": [
      "sidekiq"
    ],
    "json.load_balancing_strategy": [
      "primary"
    ],
    "json.job_status": [
      "done"
    ],
    "json.meta.feature_category": [
      "gitaly"
    ],
    "json.redis_shared_state_read_bytes": [
      93
    ],
    "json.db_main_wal_cached_count": [
      0
    ],
    "json.message.keyword": [
      "Projects::UpdateRepositoryStorageWorker JID-393d104e7b3d3252b6f43fe4: done: 0.075734 sec"
    ],
    "json.worker_id.keyword": [
      "sidekiq_0"
    ],
    "json.environment.keyword": [
      "gstg"
    ],
    "json.meta.user": [
      "steveazz"
    ],
    "json.duration_s": [
      0.075734
    ],
    "json.queue.keyword": [
      "gitaly_throttled"
    ],
    "json.redis_read_bytes": [
      1555
    ],
    "json.meta.user.keyword": [
      "steveazz"
    ],
    "json.meta.feature_category.keyword": [
      "gitaly"
    ],
    "kubernetes.namespace_name.keyword": [
      "gitlab"
    ],
    "json.redis_shared_state_calls": [
      4
    ],
    "json.redis_cache_duration_s": [
      0.003561
    ],
    "json.redis_calls": [
      12
    ],
    "json.meta.client_id.keyword": [
      "user/210928"
    ],
    "json.db_ci_replica_wal_count": [
      0
    ],
    "json.db_primary_count": [
      4
    ],
    "json.db_main_replica_duration_s": [
      0
    ],
    "json.db_main_cached_count": [
      0
    ],
    "json.severity": [
      "INFO"
    ],
    "json.db_write_count": [
      3
    ],
    "json.redis_shared_state_write_bytes": [
      179
    ],
    "json.shard.keyword": [
      "gitaly-throttled"
    ],
    "@timestamp": [
      "2022-12-05T09:15:52.846Z"
    ],
    "json.worker_id": [
      "sidekiq_0"
    ],
    "json.redis_queues_write_bytes": [
      132
    ],
    "json.db_ci_duration_s": [
      0
    ],
    "kubernetes.pod_name": [
      "gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4"
    ],
    "json.idempotency_key.keyword": [
      "resque:gitlab:duplicate:gitaly_throttled:9e687ad2de1a6e7d65478d9943661b8d24fbb9ee0d89d319e83640a66cca0ea0"
    ],
    "json.job_size_bytes": [
      34
    ],
    "json.db_replica_duration_s": [
      0
    ],
    "json.meta.remote_ip": [
      "94.17.156.48"
    ],
    "json.db_main_count": [
      4
    ],
    "json.completed_at": [
      "2022-12-05T09:15:22.580Z"
    ],
    "json.db_main_replica_wal_count": [
      0
    ],
    "json.redis_shared_state_duration_s": [
      0.02882
    ],
    "json.scheduling_latency_s": [
      0.001216
    ],
    "json.meta.project": [
      "steveazz/gitlab"
    ],
    "json.retry": [
      3
    ],
    "json.enqueued_at": [
      "2022-12-05T09:15:22.504Z"
    ],
    "json.worker_data_consistency.keyword": [
      "always"
    ],
    "kubernetes.container_image.keyword": [
      "dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-sidekiq-ee:15-7-202212050320-34b74c83da5"
    ],
    "json.db_main_replica_wal_cached_count": [
      0
    ],
    "json.redis_cache_calls": [
      7
    ],
    "type.keyword": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "kubernetes.container_image": [
      "dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-sidekiq-ee:15-7-202212050320-34b74c83da5"
    ],
    "json.db_cached_count": [
      0
    ],
    "json.args": [
      "8302702",
      "[FILTERED]",
      "2233127"
    ],
    "host.name": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "json.redis_queues_calls": [
      1
    ],
    "json.db_ci_wal_cached_count": [
      0
    ],
    "kubernetes.host.keyword": [
      "gke-gstg-gitlab-gke-generic-2-89a5eb4d-3bkd"
    ],
    "json.redis_queues_duration_s": [
      0.000448
    ],
    "json.tier.keyword": [
      "sv"
    ],
    "json.tag": [
      "gitlab-sidekiq-gitaly-throttled.var.log.containers.gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4_gitlab_sidekiq-c00161bbe79d6512983eaccc06af274e51e1088efad5995783880244605fa918.log"
    ],
    "kubernetes.container_name": [
      "sidekiq"
    ],
    "json.severity.keyword": [
      "INFO"
    ],
    "kubernetes.pod_ip.keyword": [
      "10.227.5.77"
    ],
    "json.db_replica_wal_count": [
      0
    ],
    "kubernetes.container_name.keyword": [
      "sidekiq"
    ],
    "json.pid": [
      36
    ],
    "json.db_primary_wal_count": [
      0
    ],
    "json.stage": [
      "main"
    ],
    "json.db_count": [
      4
    ],
    "json.db_main_replica_count": [
      0
    ],
    "json.retry_count": [
      0
    ],
    "kubernetes.region": [
      "us-east1"
    ],
    "json.redis_write_bytes": [
      853
    ],
    "json.meta.caller_id.keyword": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.error_message": [
      "Cannot transition state via :finish_replication from :replicated (Reason(s): State cannot transition via \"finish replication\")"
    ],
    "json.meta.root_caller_id": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.jid": [
      "393d104e7b3d3252b6f43fe4"
    ],
    "json.cpu_s": [
      0.018574
    ],
    "json.load_balancing_strategy.keyword": [
      "primary"
    ],
    "json.tag.keyword": [
      "gitlab-sidekiq-gitaly-throttled.var.log.containers.gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4_gitlab_sidekiq-c00161bbe79d6512983eaccc06af274e51e1088efad5995783880244605fa918.log"
    ],
    "json.error_class.keyword": [
      "StateMachines::InvalidTransition"
    ],
    "json.failed_at": [
      "2022-12-05T09:15:01.273Z"
    ],
    "json.meta.project.keyword": [
      "steveazz/gitlab"
    ],
    "json.time": [
      "2022-12-05T09:15:22.580Z"
    ],
    "json.error_class": [
      "StateMachines::InvalidTransition"
    ],
    "json.meta.root_namespace": [
      "steveazz"
    ],
    "json.db_main_duration_s": [
      0.007
    ],
    "kubernetes.host": [
      "gke-gstg-gitlab-gke-generic-2-89a5eb4d-3bkd"
    ],
    "json.db_main_replica_cached_count": [
      0
    ],
    "kubernetes.pod_name.keyword": [
      "gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4"
    ],
    "json.mem_objects": [
      6339
    ],
    "json.shard": [
      "gitaly-throttled"
    ],
    "json.stage.keyword": [
      "main"
    ],
    "json.correlation_id.keyword": [
      "2c8c9390e558aee813d064c571abdc69"
    ],
    "json.job_status.keyword": [
      "done"
    ],
    "json.correlation_id": [
      "2c8c9390e558aee813d064c571abdc69"
    ],
    "json.db_ci_replica_cached_count": [
      0
    ],
    "kubernetes.region.keyword": [
      "us-east1"
    ],
    "json.db_ci_replica_wal_cached_count": [
      0
    ],
    "json.db_ci_count": [
      0
    ],
    "json.message": [
      "Projects::UpdateRepositoryStorageWorker JID-393d104e7b3d3252b6f43fe4: done: 0.075734 sec"
    ],
    "json.meta.remote_ip.keyword": [
      "94.17.156.48"
    ],
    "json.db_ci_replica_duration_s": [
      0
    ],
    "json.tier": [
      "sv"
    ],
    "json.created_at": [
      "2022-12-05T09:05:49.372Z"
    ],
    "json.class.keyword": [
      "Projects::UpdateRepositoryStorageWorker"
    ],
    "kubernetes.namespace_name": [
      "gitlab"
    ],
    "json.meta.root_caller_id.keyword": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.db_duration_s": [
      0.012159
    ],
    "json.db_ci_wal_count": [
      0
    ],
    "json.meta.user_id": [
      210928
    ],
    "json.error_message.keyword": [
      "Cannot transition state via :finish_replication from :replicated (Reason(s): State cannot transition via \"finish replication\")"
    ],
    "json.meta.client_id": [
      "user/210928"
    ]
  }
}

{
  "_index": "pubsub-sidekiq-inf-gstg-000426",
  "_type": "_doc",
  "_id": "l72S4YQBnNEWeQqYBMqJ",
  "_version": 1,
  "_score": 1,
  "_source": {
    "@timestamp": "2022-12-05T09:15:52.845Z",
    "host": {
      "name": "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    },
    "type": "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj",
    "kubernetes": {
      "region": "us-east1",
      "pod_name": "gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4",
      "pod_ip": "10.227.5.77",
      "host": "gke-gstg-gitlab-gke-generic-2-89a5eb4d-3bkd",
      "namespace_name": "gitlab",
      "container_name": "sidekiq",
      "container_image": "dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-sidekiq-ee:15-7-202212050320-34b74c83da5"
    },
    "publish_time": "2022-12-05T09:15:52.822Z",
    "json": {
      "shard": "gitaly-throttled",
      "meta.user": "steveazz",
      "duration_s": 551.874423,
      "message": "Projects::UpdateRepositoryStorageWorker JID-393d104e7b3d3252b6f43fe4: fail: 551.874423 sec",
      "type": "sidekiq",
      "db_primary_cached_count": 4,
      "cpu_s": 0.671433,
      "class": "Projects::UpdateRepositoryStorageWorker",
      "time": "2022-12-05T09:15:01.275Z",
      "external_http_duration_s": 0.12479434674605727,
      "meta.root_namespace": "steveazz",
      "redis_read_bytes": 38833,
      "redis_shared_state_write_bytes": 234,
      "meta.client_id": "user/210928",
      "external_http_count": 2,
      "db_main_replica_count": 0,
      "db_ci_replica_wal_cached_count": 0,
      "environment": "gstg",
      "db_ci_wal_count": 0,
      "meta.caller_id": "POST /api/:version/projects/:id/repository_storage_moves",
      "db_replica_wal_cached_count": 0,
      "redis_shared_state_calls": 5,
      "mem_total_bytes": 26049163,
      "redis_shared_state_duration_s": 0.042916,
      "jid": "393d104e7b3d3252b6f43fe4",
      "db_primary_wal_cached_count": 0,
      "db_ci_replica_cached_count": 0,
      "db_primary_duration_s": 0.069,
      "db_write_count": 14,
      "worker_id": "sidekiq_0",
      "redis_queues_duration_s": 0.000284,
      "retry": 3,
      "db_ci_replica_duration_s": 0,
      "meta.remote_ip": "94.17.156.48",
      "redis_queues_calls": 1,
      "correlation_id": "e71cba34ab67f2cd1608753b61d90ab5",
      "redis_write_bytes": 9071,
      "db_replica_count": 0,
      "completed_at": "2022-12-05T09:15:01.274Z",
      "meta.user_id": 210928,
      "size_limiter": "validated",
      "db_replica_duration_s": 0,
      "exception.backtrace": [
        "app/services/concerns/update_repository_storage_methods.rb:26:in `block in execute'",
        "app/models/concerns/cross_database_modification.rb:92:in `block in transaction'",
        "lib/gitlab/database/load_balancing/connection_proxy.rb:120:in `block in write_using_load_balancer'",
        "lib/gitlab/database/load_balancing/load_balancer.rb:129:in `block in read_write'",
        "lib/gitlab/database/load_balancing/load_balancer.rb:207:in `retry_with_backoff'",
        "lib/gitlab/database/load_balancing/load_balancer.rb:118:in `read_write'",
        "lib/gitlab/database/load_balancing/connection_proxy.rb:119:in `write_using_load_balancer'",
        "lib/gitlab/database/load_balancing/connection_proxy.rb:71:in `transaction'",
        "lib/gitlab/database.rb:333:in `block in transaction'",
        "lib/gitlab/database.rb:332:in `transaction'",
        "app/models/concerns/cross_database_modification.rb:83:in `transaction'",
        "app/services/concerns/update_repository_storage_methods.rb:25:in `execute'",
        "app/workers/projects/update_repository_storage_worker.rb:24:in `update_repository_storage'",
        "app/workers/concerns/update_repository_storage_worker.rb:26:in `perform'",
        "lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:26:in `call'",
        "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
        "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
        "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
        "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
        "lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
        "lib/gitlab/application_context.rb:115:in `block in use'",
        "lib/gitlab/application_context.rb:115:in `use'",
        "lib/gitlab/application_context.rb:55:in `with_context'",
        "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
        "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
        "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
        "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
        "lib/gitlab/database/query_analyzer.rb:37:in `within'",
        "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
        "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
        "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
        "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
        "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
        "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
        "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
        "lib/gitlab/with_request_store.rb:10:in `with_request_store'",
        "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
        "lib/gitlab/sidekiq_middleware/server_metrics.rb:76:in `block in call'",
        "lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `block in instrument'",
        "lib/gitlab/metrics/background_transaction.rb:33:in `run'",
        "lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `instrument'",
        "lib/gitlab/sidekiq_middleware/server_metrics.rb:75:in `call'",
        "lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
        "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
        "lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
        "lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
        "lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
      ],
      "db_main_replica_cached_count": 0,
      "mem_bytes": 16766603,
      "db_ci_count": 0,
      "db_main_replica_duration_s": 0,
      "db_primary_wal_count": 0,
      "load_balancing_strategy": "primary",
      "version": 0,
      "db_cached_count": 4,
      "db_main_count": 29,
      "pid": 36,
      "db_ci_replica_wal_count": 0,
      "created_at": "2022-12-05T09:05:49.372Z",
      "redis_cache_read_bytes": 38740,
      "db_ci_cached_count": 0,
      "exception.class": "StateMachines::InvalidTransition",
      "db_replica_cached_count": 0,
      "enqueued_at": "2022-12-05T09:05:49.399Z",
      "db_main_wal_cached_count": 0,
      "tier": "sv",
      "meta.project": "steveazz/gitlab",
      "redis_cache_write_bytes": 8705,
      "db_duration_s": 0.059837,
      "args": [
        "8302702",
        "[FILTERED]",
        "2233127"
      ],
      "gitaly_duration_s": 547.453691,
      "redis_queues_write_bytes": 132,
      "db_replica_wal_count": 0,
      "db_ci_duration_s": 0,
      "worker_data_consistency": "always",
      "exception.message": "Cannot transition state via :finish_replication from :replicated (Reason(s): State cannot transition via \"finish replication\")",
      "stage": "main",
      "db_main_replica_wal_cached_count": 0,
      "rate_limiting_gates": [],
      "meta.root_caller_id": "POST /api/:version/projects/:id/repository_storage_moves",
      "db_ci_wal_cached_count": 0,
      "redis_duration_s": 0.08055000000000001,
      "redis_cache_duration_s": 0.03735,
      "redis_cache_calls": 134,
      "db_ci_replica_count": 0,
      "meta.feature_category": "gitaly",
      "redis_shared_state_read_bytes": 92,
      "mem_objects": 232064,
      "db_main_cached_count": 4,
      "scheduling_latency_s": 0.001202,
      "job_size_bytes": 34,
      "tag": "gitlab-sidekiq-gitaly-throttled.var.log.containers.gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4_gitlab_sidekiq-c00161bbe79d6512983eaccc06af274e51e1088efad5995783880244605fa918.log",
      "severity": "WARN",
      "db_primary_count": 29,
      "db_count": 29,
      "db_main_wal_count": 0,
      "idempotency_key": "resque:gitlab:duplicate:gitaly_throttled:9e687ad2de1a6e7d65478d9943661b8d24fbb9ee0d89d319e83640a66cca0ea0",
      "job_status": "fail",
      "redis_queues_read_bytes": 1,
      "queue": "gitaly_throttled",
      "mem_mallocs": 49749,
      "gitaly_calls": 14,
      "db_main_duration_s": 0.069,
      "redis_calls": 140,
      "db_main_replica_wal_count": 0
    }
  },
  "fields": {
    "json.type.keyword": [
      "sidekiq"
    ],
    "json.idempotency_key": [
      "resque:gitlab:duplicate:gitaly_throttled:9e687ad2de1a6e7d65478d9943661b8d24fbb9ee0d89d319e83640a66cca0ea0"
    ],
    "json.db_replica_wal_cached_count": [
      0
    ],
    "json.mem_mallocs": [
      49749
    ],
    "json.redis_duration_s": [
      0.08055
    ],
    "host.name.keyword": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "json.db_primary_cached_count": [
      4
    ],
    "json.db_replica_count": [
      0
    ],
    "json.mem_total_bytes": [
      26049163
    ],
    "type": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "json.db_main_wal_count": [
      0
    ],
    "json.redis_cache_read_bytes": [
      38740
    ],
    "json.mem_bytes": [
      16766603
    ],
    "json.redis_cache_write_bytes": [
      8705
    ],
    "json.exception.class.keyword": [
      "StateMachines::InvalidTransition"
    ],
    "publish_time": [
      "2022-12-05T09:15:52.822Z"
    ],
    "json.redis_queues_read_bytes": [
      1
    ],
    "json.db_primary_wal_cached_count": [
      0
    ],
    "json.worker_data_consistency": [
      "always"
    ],
    "json.meta.caller_id": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.external_http_duration_s": [
      0.12479435
    ],
    "json.version": [
      0
    ],
    "json.class": [
      "Projects::UpdateRepositoryStorageWorker"
    ],
    "json.external_http_count": [
      2
    ],
    "json.args.keyword": [
      "8302702",
      "[FILTERED]",
      "2233127"
    ],
    "json.jid.keyword": [
      "393d104e7b3d3252b6f43fe4"
    ],
    "json.meta.root_namespace.keyword": [
      "steveazz"
    ],
    "json.queue": [
      "gitaly_throttled"
    ],
    "json.size_limiter": [
      "validated"
    ],
    "json.exception.message.keyword": [
      "Cannot transition state via :finish_replication from :replicated (Reason(s): State cannot transition via \"finish replication\")"
    ],
    "json.db_replica_cached_count": [
      0
    ],
    "json.db_ci_replica_count": [
      0
    ],
    "json.db_ci_cached_count": [
      0
    ],
    "json.environment": [
      "gstg"
    ],
    "json.db_primary_duration_s": [
      0.069
    ],
    "json.size_limiter.keyword": [
      "validated"
    ],
    "kubernetes.pod_ip": [
      "10.227.5.77"
    ],
    "json.type": [
      "sidekiq"
    ],
    "json.load_balancing_strategy": [
      "primary"
    ],
    "json.job_status": [
      "fail"
    ],
    "json.meta.feature_category": [
      "gitaly"
    ],
    "json.redis_shared_state_read_bytes": [
      92
    ],
    "json.db_main_wal_cached_count": [
      0
    ],
    "json.message.keyword": [
      "Projects::UpdateRepositoryStorageWorker JID-393d104e7b3d3252b6f43fe4: fail: 551.874423 sec"
    ],
    "json.worker_id.keyword": [
      "sidekiq_0"
    ],
    "json.environment.keyword": [
      "gstg"
    ],
    "json.exception.message": [
      "Cannot transition state via :finish_replication from :replicated (Reason(s): State cannot transition via \"finish replication\")"
    ],
    "json.meta.user": [
      "steveazz"
    ],
    "json.duration_s": [
      551.87445
    ],
    "json.queue.keyword": [
      "gitaly_throttled"
    ],
    "json.redis_read_bytes": [
      38833
    ],
    "json.meta.user.keyword": [
      "steveazz"
    ],
    "json.gitaly_duration_s": [
      547.4537
    ],
    "json.gitaly_calls": [
      14
    ],
    "json.meta.feature_category.keyword": [
      "gitaly"
    ],
    "kubernetes.namespace_name.keyword": [
      "gitlab"
    ],
    "json.redis_shared_state_calls": [
      5
    ],
    "json.redis_cache_duration_s": [
      0.03735
    ],
    "json.redis_calls": [
      140
    ],
    "json.meta.client_id.keyword": [
      "user/210928"
    ],
    "json.db_ci_replica_wal_count": [
      0
    ],
    "json.db_primary_count": [
      29
    ],
    "json.db_main_replica_duration_s": [
      0
    ],
    "json.db_main_cached_count": [
      4
    ],
    "json.severity": [
      "WARN"
    ],
    "json.db_write_count": [
      14
    ],
    "json.redis_shared_state_write_bytes": [
      234
    ],
    "json.shard.keyword": [
      "gitaly-throttled"
    ],
    "@timestamp": [
      "2022-12-05T09:15:52.845Z"
    ],
    "json.worker_id": [
      "sidekiq_0"
    ],
    "json.redis_queues_write_bytes": [
      132
    ],
    "json.db_ci_duration_s": [
      0
    ],
    "json.exception.class": [
      "StateMachines::InvalidTransition"
    ],
    "kubernetes.pod_name": [
      "gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4"
    ],
    "json.idempotency_key.keyword": [
      "resque:gitlab:duplicate:gitaly_throttled:9e687ad2de1a6e7d65478d9943661b8d24fbb9ee0d89d319e83640a66cca0ea0"
    ],
    "json.job_size_bytes": [
      34
    ],
    "json.db_replica_duration_s": [
      0
    ],
    "json.meta.remote_ip": [
      "94.17.156.48"
    ],
    "json.db_main_count": [
      29
    ],
    "json.completed_at": [
      "2022-12-05T09:15:01.274Z"
    ],
    "json.db_main_replica_wal_count": [
      0
    ],
    "json.redis_shared_state_duration_s": [
      0.042916
    ],
    "json.scheduling_latency_s": [
      0.001202
    ],
    "json.meta.project": [
      "steveazz/gitlab"
    ],
    "json.retry": [
      3
    ],
    "json.enqueued_at": [
      "2022-12-05T09:05:49.399Z"
    ],
    "json.worker_data_consistency.keyword": [
      "always"
    ],
    "kubernetes.container_image.keyword": [
      "dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-sidekiq-ee:15-7-202212050320-34b74c83da5"
    ],
    "json.db_main_replica_wal_cached_count": [
      0
    ],
    "json.redis_cache_calls": [
      134
    ],
    "type.keyword": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "kubernetes.container_image": [
      "dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-sidekiq-ee:15-7-202212050320-34b74c83da5"
    ],
    "json.db_cached_count": [
      4
    ],
    "json.args": [
      "8302702",
      "[FILTERED]",
      "2233127"
    ],
    "host.name": [
      "pubsubbeat-pubsub-sidekiq-inf-gstg-7f985b494f-kpvjj"
    ],
    "json.redis_queues_calls": [
      1
    ],
    "json.db_ci_wal_cached_count": [
      0
    ],
    "kubernetes.host.keyword": [
      "gke-gstg-gitlab-gke-generic-2-89a5eb4d-3bkd"
    ],
    "json.redis_queues_duration_s": [
      0.000284
    ],
    "json.tier.keyword": [
      "sv"
    ],
    "json.tag": [
      "gitlab-sidekiq-gitaly-throttled.var.log.containers.gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4_gitlab_sidekiq-c00161bbe79d6512983eaccc06af274e51e1088efad5995783880244605fa918.log"
    ],
    "kubernetes.container_name": [
      "sidekiq"
    ],
    "json.severity.keyword": [
      "WARN"
    ],
    "kubernetes.pod_ip.keyword": [
      "10.227.5.77"
    ],
    "json.db_replica_wal_count": [
      0
    ],
    "kubernetes.container_name.keyword": [
      "sidekiq"
    ],
    "json.pid": [
      36
    ],
    "json.db_primary_wal_count": [
      0
    ],
    "json.exception.backtrace.keyword": [
      "app/services/concerns/update_repository_storage_methods.rb:26:in `block in execute'",
      "app/models/concerns/cross_database_modification.rb:92:in `block in transaction'",
      "lib/gitlab/database/load_balancing/connection_proxy.rb:120:in `block in write_using_load_balancer'",
      "lib/gitlab/database/load_balancing/load_balancer.rb:129:in `block in read_write'",
      "lib/gitlab/database/load_balancing/load_balancer.rb:207:in `retry_with_backoff'",
      "lib/gitlab/database/load_balancing/load_balancer.rb:118:in `read_write'",
      "lib/gitlab/database/load_balancing/connection_proxy.rb:119:in `write_using_load_balancer'",
      "lib/gitlab/database/load_balancing/connection_proxy.rb:71:in `transaction'",
      "lib/gitlab/database.rb:333:in `block in transaction'",
      "lib/gitlab/database.rb:332:in `transaction'",
      "app/models/concerns/cross_database_modification.rb:83:in `transaction'",
      "app/services/concerns/update_repository_storage_methods.rb:25:in `execute'",
      "app/workers/projects/update_repository_storage_worker.rb:24:in `update_repository_storage'",
      "app/workers/concerns/update_repository_storage_worker.rb:26:in `perform'",
      "lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:26:in `call'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
      "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
      "lib/gitlab/application_context.rb:115:in `block in use'",
      "lib/gitlab/application_context.rb:115:in `use'",
      "lib/gitlab/application_context.rb:55:in `with_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
      "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
      "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
      "lib/gitlab/database/query_analyzer.rb:37:in `within'",
      "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
      "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
      "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
      "lib/gitlab/with_request_store.rb:10:in `with_request_store'",
      "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:76:in `block in call'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `block in instrument'",
      "lib/gitlab/metrics/background_transaction.rb:33:in `run'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `instrument'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:75:in `call'",
      "lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
      "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
      "lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
      "lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
    ],
    "json.stage": [
      "main"
    ],
    "json.db_count": [
      29
    ],
    "json.db_main_replica_count": [
      0
    ],
    "kubernetes.region": [
      "us-east1"
    ],
    "json.redis_write_bytes": [
      9071
    ],
    "json.meta.caller_id.keyword": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.meta.root_caller_id": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.jid": [
      "393d104e7b3d3252b6f43fe4"
    ],
    "json.cpu_s": [
      0.671433
    ],
    "json.load_balancing_strategy.keyword": [
      "primary"
    ],
    "json.tag.keyword": [
      "gitlab-sidekiq-gitaly-throttled.var.log.containers.gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4_gitlab_sidekiq-c00161bbe79d6512983eaccc06af274e51e1088efad5995783880244605fa918.log"
    ],
    "json.meta.project.keyword": [
      "steveazz/gitlab"
    ],
    "json.time": [
      "2022-12-05T09:15:01.275Z"
    ],
    "json.meta.root_namespace": [
      "steveazz"
    ],
    "json.db_main_duration_s": [
      0.069
    ],
    "kubernetes.host": [
      "gke-gstg-gitlab-gke-generic-2-89a5eb4d-3bkd"
    ],
    "json.db_main_replica_cached_count": [
      0
    ],
    "kubernetes.pod_name.keyword": [
      "gitlab-sidekiq-gitaly-throttled-v2-5bb679bd57-qrnf4"
    ],
    "json.exception.backtrace": [
      "app/services/concerns/update_repository_storage_methods.rb:26:in `block in execute'",
      "app/models/concerns/cross_database_modification.rb:92:in `block in transaction'",
      "lib/gitlab/database/load_balancing/connection_proxy.rb:120:in `block in write_using_load_balancer'",
      "lib/gitlab/database/load_balancing/load_balancer.rb:129:in `block in read_write'",
      "lib/gitlab/database/load_balancing/load_balancer.rb:207:in `retry_with_backoff'",
      "lib/gitlab/database/load_balancing/load_balancer.rb:118:in `read_write'",
      "lib/gitlab/database/load_balancing/connection_proxy.rb:119:in `write_using_load_balancer'",
      "lib/gitlab/database/load_balancing/connection_proxy.rb:71:in `transaction'",
      "lib/gitlab/database.rb:333:in `block in transaction'",
      "lib/gitlab/database.rb:332:in `transaction'",
      "app/models/concerns/cross_database_modification.rb:83:in `transaction'",
      "app/services/concerns/update_repository_storage_methods.rb:25:in `execute'",
      "app/workers/projects/update_repository_storage_worker.rb:24:in `update_repository_storage'",
      "app/workers/concerns/update_repository_storage_worker.rb:26:in `perform'",
      "lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:26:in `call'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
      "lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
      "lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
      "lib/gitlab/application_context.rb:115:in `block in use'",
      "lib/gitlab/application_context.rb:115:in `use'",
      "lib/gitlab/application_context.rb:55:in `with_context'",
      "lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
      "lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
      "lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
      "lib/gitlab/database/query_analyzer.rb:37:in `within'",
      "lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
      "lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
      "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
      "lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
      "lib/gitlab/with_request_store.rb:10:in `with_request_store'",
      "lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:76:in `block in call'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `block in instrument'",
      "lib/gitlab/metrics/background_transaction.rb:33:in `run'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `instrument'",
      "lib/gitlab/sidekiq_middleware/server_metrics.rb:75:in `call'",
      "lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
      "lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
      "lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
      "lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
      "lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
    ],
    "json.mem_objects": [
      232064
    ],
    "json.shard": [
      "gitaly-throttled"
    ],
    "json.stage.keyword": [
      "main"
    ],
    "json.correlation_id.keyword": [
      "e71cba34ab67f2cd1608753b61d90ab5"
    ],
    "json.job_status.keyword": [
      "fail"
    ],
    "json.correlation_id": [
      "e71cba34ab67f2cd1608753b61d90ab5"
    ],
    "json.db_ci_replica_cached_count": [
      0
    ],
    "kubernetes.region.keyword": [
      "us-east1"
    ],
    "json.db_ci_replica_wal_cached_count": [
      0
    ],
    "json.db_ci_count": [
      0
    ],
    "json.message": [
      "Projects::UpdateRepositoryStorageWorker JID-393d104e7b3d3252b6f43fe4: fail: 551.874423 sec"
    ],
    "json.meta.remote_ip.keyword": [
      "94.17.156.48"
    ],
    "json.db_ci_replica_duration_s": [
      0
    ],
    "json.tier": [
      "sv"
    ],
    "json.created_at": [
      "2022-12-05T09:05:49.372Z"
    ],
    "json.class.keyword": [
      "Projects::UpdateRepositoryStorageWorker"
    ],
    "kubernetes.namespace_name": [
      "gitlab"
    ],
    "json.meta.root_caller_id.keyword": [
      "POST /api/:version/projects/:id/repository_storage_moves"
    ],
    "json.db_duration_s": [
      0.059837
    ],
    "json.db_ci_wal_count": [
      0
    ],
    "json.meta.user_id": [
      210928
    ],
    "json.meta.client_id": [
      "user/210928"
    ]
  }
}