User contribution mapping: Reassignment of placeholder users to real users is stuck
Hey folks, we had deployed Gitlab CE 17.8.5 on a Kubernetes cluster using the official helm chart. We had migrated Groups from Gitlab CE 15 (don't have minor version as of now, will update incase we find it out) using direct transfer. After migration was completed, we started the process of reassigning placeholder users to real users. We had 250+ users but the reassignment got stuck after 39 users where reassigned.Rest of the Users are stuck with Reassigning status.
We tried going through the logs of webservice, sidekiq and gitaly but weren't able to see much related to this. We had also looked at sideiq metrics but couldn't identify any issue there as well. We tried going through the docs in order to understand how does reassignment of users work and which components are responsible for the same but couldn't find/understand much.
Please do let us know how can we help in providing more context. We'll try enabling debug logs to get more visibility. It would be really helpful if someone can help us to narrow down to areas that are related to the issue. The only log that we can see when we search reassignment is as following on webservice container
Click to expand
{
"component": "gitlab",
"subcomponent": "production_json",
"method": "POST",
"path": "/api/graphql",
"format": "*/*",
"controller": "GraphqlController",
"action": "execute",
"status": 200,
"time": "2025-03-17T14:30:54.733Z",
"params": [
{
"key": "operationName",
"value": "importSourceUsers"
},
{
"key": "variables",
"value": "[FILTERED]"
},
{
"key": "query",
"value": "[FILTERED]"
},
{
"key": "graphql",
"value": {
"operationName": "importSourceUsers",
"variables": "[FILTERED]",
"query": "[FILTERED]"
}
}
],
"correlation_id": "01JPJ87HXNWW7QNDXDR9K85REE",
"meta.caller_id": "GraphqlController#execute",
"meta.feature_category": "groups_and_projects",
"meta.organization_id": 1,
"meta.remote_ip": "114.143.63.226",
"meta.user": "bhabesh.mahato",
"meta.user_id": 375,
"meta.client_id": "user/375",
"graphql": [
{
"complexity": 42,
"depth": 5,
"used_fields": [
"Namespace.id",
"ImportSourceUser.id",
"ImportSourceUser.sourceUsername",
"ImportSourceUser.sourceName",
"ImportSourceUser.sourceHostname",
"ImportSourceUser.sourceUserIdentifier",
"ImportSourceUser.status",
"ImportSourceUser.reassignmentError",
"User.id",
"User.avatarUrl",
"User.name",
"User.username",
"User.webUrl",
"User.webPath",
"User.__typename",
"UserCore.__typename",
"ImportSourceUser.placeholderUser",
"ImportSourceUser.reassignToUser",
"ImportSourceUser.__typename",
"ImportSourceUserConnection.nodes",
"PageInfo.hasNextPage",
"PageInfo.hasPreviousPage",
"PageInfo.startCursor",
"PageInfo.endCursor",
"PageInfo.__typename",
"ImportSourceUserConnection.pageInfo",
"ImportSourceUserConnection.__typename",
"Namespace.importSourceUsers",
"Namespace.__typename",
"Query.namespace"
],
"used_deprecated_fields": [
"Namespace.importSourceUsers"
],
"used_deprecated_arguments": [],
"variables": "{\"fullPath\"=>\"fplabs\", \"before\"=>nil, \"after\"=>nil, \"first\"=>20, \"statuses\"=>[\"COMPLETED\", \"KEEP_AS_PLACEHOLDER\"], \"search\"=>\"[FILTERED]\", \"sort\"=>nil}",
"operation_name": "importSourceUsers"
}
],
"remote_ip": "114.143.63.226",
"user_id": 375,
"username": "bhabesh.mahato",
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36",
"queue_duration_s": 0.010762,
"request_urgency": "low",
"target_duration_s": 5,
"redis_calls": 5,
"redis_allowed_cross_slot_calls": 1,
"redis_duration_s": 0.001379,
"redis_read_bytes": 513,
"redis_write_bytes": 721,
"redis_db_load_balancing_calls": 2,
"redis_db_load_balancing_duration_s": 0.00054,
"redis_db_load_balancing_write_bytes": 108,
"redis_sessions_calls": 3,
"redis_sessions_allowed_cross_slot_calls": 1,
"redis_sessions_duration_s": 0.000839,
"redis_sessions_read_bytes": 513,
"redis_sessions_write_bytes": 613,
"db_count": 8,
"db_write_count": 0,
"db_cached_count": 0,
"db_txn_count": 0,
"db_replica_txn_count": 0,
"db_primary_txn_count": 0,
"db_replica_count": 0,
"db_primary_count": 8,
"db_replica_write_count": 0,
"db_primary_write_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_replica_txn_max_duration_s": 0,
"db_primary_txn_max_duration_s": 0,
"db_replica_txn_duration_s": 0,
"db_primary_txn_duration_s": 0,
"db_replica_duration_s": 0,
"db_primary_duration_s": 0.014,
"db_main_txn_count": 0,
"db_ci_txn_count": 0,
"db_main_replica_txn_count": 0,
"db_ci_replica_txn_count": 0,
"db_main_count": 8,
"db_ci_count": 0,
"db_main_replica_count": 0,
"db_ci_replica_count": 0,
"db_main_write_count": 0,
"db_ci_write_count": 0,
"db_main_replica_write_count": 0,
"db_ci_replica_write_count": 0,
"db_main_cached_count": 0,
"db_ci_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_ci_replica_cached_count": 0,
"db_main_wal_count": 0,
"db_ci_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_ci_replica_wal_count": 0,
"db_main_wal_cached_count": 0,
"db_ci_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_ci_replica_wal_cached_count": 0,
"db_main_txn_max_duration_s": 0,
"db_ci_txn_max_duration_s": 0,
"db_main_replica_txn_max_duration_s": 0,
"db_ci_replica_txn_max_duration_s": 0,
"db_main_txn_duration_s": 0,
"db_ci_txn_duration_s": 0,
"db_main_replica_txn_duration_s": 0,
"db_ci_replica_txn_duration_s": 0,
"db_main_duration_s": 0.014,
"db_ci_duration_s": 0,
"db_main_replica_duration_s": 0,
"db_ci_replica_duration_s": 0,
"cpu_s": 0.071606,
"mem_objects": 39600,
"mem_bytes": 2366664,
"mem_mallocs": 13025,
"mem_total_bytes": 3950664,
"pid": 47,
"worker_id": "puma_1",
"rate_limiting_gates": [],
"db_duration_s": 0.01094,
"view_duration_s": 0.0009,
"duration_s": 0.06767
}Update
we also noticed that the users who were stuck had there jobs in following state
Click to expand
{
"component": "gitlab",
"subcomponent": "sidekiq_client",
"severity": "INFO",
"time": "2025-03-17T05:17:45.198Z",
"retry": 5,
"queue": "default",
"version": 0,
"store": null,
"dead": false,
"max_retries_after_interruption": 20,
"args": [
"72"
],
"class": "Import::ReassignPlaceholderUserRecordsWorker",
"jid": "90d4abe69122aefe6122b51e",
"created_at": "2025-03-17T05:17:45.197Z",
"trace_propagation_headers": {
"sentry-trace": "dce2d677a5c4448d9ca94e4845fe0229-7588313699bf45a5",
"baggage": "sentry-trace_id=dce2d677a5c4448d9ca94e4845fe0229,sentry-environment=,sentry-release=752f3e291f7"
},
"correlation_id": "01JPH8JP94J601Y5229ZCBSH13",
"meta.caller_id": "Import::SourceUsersController#accept",
"meta.feature_category": "importers",
"meta.organization_id": 1,
"meta.remote_ip": "114.143.63.226",
"meta.user": "jitendra",
"meta.user_id": 457,
"meta.client_id": "user/457",
"meta.root_caller_id": "Import::SourceUsersController#accept",
"worker_data_consistency": "sticky",
"worker_data_consistency_per_db": {
"main": "sticky",
"ci": "sticky"
},
"wal_locations": {},
"wal_location_sources": {
"main": "primary",
"ci": "primary"
},
"job_size_bytes": 4,
"pid": 44,
"job_status": "concurrency_limit",
"message": "Import::ReassignPlaceholderUserRecordsWorker JID-90d4abe69122aefe6122b51e: concurrency_limit: paused"
}Looks like the concurrency limit was hit. But why didn't the job failed or was retried? Is there a way to make these jobs fail? maybe updating entries in redis might work?
Edited by 🤖 GitLab Bot 🤖