The fork projects are missing alternates as intermittent after restore using bundle exec rake gitlab:backup:restore.

Everyone can contribute. Help move this issue forward while earning points, leveling up and collecting rewards.

• Close this issue

I forked a project and backup the GitLab instance using bundle exec rake gitlab:backup:create

Then I restored it with bundle exec rake gitlab:backup:restore, the fork project can not access, and can not fetch data.

When it is failed.

This is a log.

{
  "command": "restore",
  "gl_project_path": "system/awx-ee",
  "level": "info",
  "msg": "started restore",
  "pid": 112,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:44.224Z"
}
{
  "command": "restore",
  "gl_project_path": "system/awx-ee.wiki",
  "level": "info",
  "msg": "started restore",
  "pid": 112,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:44.224Z"
}
{
  "command": "restore",
  "gl_project_path": "system/awx-ee.wiki",
  "level": "warning",
  "msg": "skipped restore",
  "pid": 112,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:44.224Z"
}
{
  "command": "restore",
  "gl_project_path": "system/awx-ee",
  "level": "info",
  "msg": "completed restore",
  "pid": 112,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:44.648Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee",
  "level": "info",
  "msg": "started restore",
  "pid": 112,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:44.648Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee.wiki",
  "level": "info",
  "msg": "started restore",
  "pid": 112,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:45.310Z"
}
{
  "backup_id": "20240712063949",
  "error": "no refs in backup",
  "level": "warning",
  "msg": "unable to reset refs. Proceeding with a normal restore",
  "pid": 112,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.wiki.git",
  "storage": "default",
  "time": "2024-07-12T07:29:45.310Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee",
  "level": "info",
  "msg": "completed restore",
  "pid": 112,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:45.578Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee.wiki",
  "level": "info",
  "msg": "completed restore",
  "pid": 112,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:29:45.950Z"
}
 - Object pool @pools/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b...

This is a log from gitlab-gitaly.

{
  "args": [
    "/tmp/gitaly-1191926314/git-exec-2886327144.d/git",
    "--git-dir",
    "/home/git/repositories/@cluster/repositories/ae/a9/38",
    "-c",
    "gc.auto=0",
    "-c",
    "maintenance.auto=0",
    "-c",
    "core.autocrlf=input",
    "-c",
    "core.useReplaceRefs=false",
    "-c",
    "core.fsync=objects,derived-metadata,reference",
    "-c",
    "core.fsyncMethod=fsync",
    "-c",
    "core.packedRefsTimeout=10000",
    "-c",
    "core.filesRefLockTimeout=1000",
    "-c",
    "core.bigFileThreshold=50m",
    "cat-file",
    "--use-mailmap",
    "-Z",
    "--batch-command",
    "--buffer",
    "--end-of-options"
  ],
  "command.cpu_time_ms": 1,
  "command.exitCode": 0,
  "command.inblock": 0,
  "command.maxrss": 390492,
  "command.oublock": 0,
  "command.real_time_ms": 10442,
  "command.system_time_ms": 1,
  "command.user_time_ms": 0,
  "component": "gitaly.StreamServerInterceptor",
  "correlation_id": "01J2K215J6D1Q2EJ787A54W1A3",
  "grpc.meta.auth_version": "v2",
  "grpc.meta.client_name": "gitlab-web",
  "grpc.meta.deadline_type": "regular",
  "grpc.meta.method_operation": "accessor",
  "grpc.meta.method_scope": "repository",
  "grpc.meta.method_type": "server_stream",
  "grpc.method": "FindLocalBranches",
  "grpc.request.deadline": "2024-07-12T08:48:01.063",
  "grpc.request.fullMethod": "/gitaly.RefService/FindLocalBranches",
  "grpc.request.glProjectPath": "ops-i-system-engineers/awx-ee",
  "grpc.request.glRepository": "project-9",
  "grpc.request.repoPath": "@cluster/repositories/ae/a9/38",
  "grpc.request.repoStorage": "gitlab-gitaly-default-0",
  "grpc.service": "gitaly.RefService",
  "grpc.start_time": "2024-07-12T08:47:51.943",
  "level": "error",
  "msg": "error: unable to normalize alternate object path: /home/git/repositories/@cluster/repositories/ae/a9/38/objects/../../../../../pools/d5/9e/40/objects\nerror: unable to normalize alternate object path: /home/git/repositories/@cluster/repositories/ae/a9/38/objects/../../../../../pools/d5/9e/40/objects\n",
  "path": "/tmp/gitaly-1191926314/git-exec-2886327144.d/git",
  "pid": 1,
  "remote_ip": "120.72.83.34",
  "span.kind": "server",
  "system": "grpc",
  "time": "2024-07-12T08:48:02.386Z",
  "user_id": "2",
  "username": "system1"
}

After restoration, the alternates folder is gone.

When it succeeds

This is a log

{
  "command": "restore",
  "gl_project_path": "system/awx-ee",
  "level": "info",
  "msg": "started restore",
  "pid": 239,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:13.036Z"
}
{
  "command": "restore",
  "gl_project_path": "system/awx-ee.wiki",
  "level": "info",
  "msg": "started restore",
  "pid": 239,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:13.092Z"
}
{
  "command": "restore",
  "gl_project_path": "system/awx-ee.wiki",
  "level": "warning",
  "msg": "skipped restore",
  "pid": 239,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:13.092Z"
}
{
  "command": "restore",
  "gl_project_path": "system/awx-ee",
  "level": "info",
  "msg": "completed restore",
  "pid": 239,
  "relative_path": "@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:13.520Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee",
  "level": "info",
  "msg": "started restore",
  "pid": 239,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:13.521Z"
}
{
  "backup_id": "20240712063949",
  "error": "reset refs: close stream: rpc error: code = Internal desc = committing update: exit status 128",
  "level": "warning",
  "msg": "unable to reset refs. Proceeding with a normal restore",
  "pid": 239,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.git",
  "storage": "default",
  "time": "2024-07-12T07:38:13.943Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee.wiki",
  "level": "info",
  "msg": "started restore",
  "pid": 239,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:14.060Z"
}
{
  "backup_id": "20240712063949",
  "error": "no refs in backup",
  "level": "warning",
  "msg": "unable to reset refs. Proceeding with a normal restore",
  "pid": 239,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.wiki.git",
  "storage": "default",
  "time": "2024-07-12T07:38:14.060Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee.wiki",
  "level": "info",
  "msg": "completed restore",
  "pid": 239,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.wiki.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:14.793Z"
}
{
  "command": "restore",
  "gl_project_path": "ops-i-system-engineers/awx-ee",
  "level": "info",
  "msg": "completed restore",
  "pid": 239,
  "relative_path": "@hashed/19/58/19581e27de7ced00ff1ce50b2047e7a567c76b1cbaebabe5ef03f7c3017bb5b7.git",
  "storage_name": "default",
  "time": "2024-07-12T07:38:15.119Z"
}
 - Object pool @pools/6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b...

FYI, I ran the command to restore multiple times, sometimes the fork project is unavailable, and sometimes it is not.

Please help me with this issue.