Restore of gitlab ends in Backup::Error: gitaly-backup exit status 1
Summary
Testing backup and restore on fresh Gitlab install.
I created some repo, triggered backup, deleted repo and triggered restore.
Steps to reproduce
create backup, delete repo, restore
Configuration used
(Please provide a sanitized version of the configuration used wrapped in a code block (```yaml))
global:
serviceAccount:
enabled: true
create: false
name: aws-access
platform:
eksRoleArn: _sanitized_
shell:
authToken:
secret: shell-secret
key: password
## https://docs.gitlab.com/charts/installation/deployment#deploy-the-community-edition
edition: ee
# gitlab:
## Enterprise license for this GitLab installation
## Secret created according to https://docs.gitlab.com/charts/installation/secrets#initial-enterprise-license
## If allowing shared-secrets generation, this is OPTIONAL.
# license:
# secret: gitlab-license
# key: license
registry:
bucket: eu-central-1-gitlab-registry
tls:
enabled: false
hosts:
domain: _sanitized_
https: true
externalIP:
gitlab:
name: _sanitized_
https: true
registry:
name: _sanitized_
https: true
# pages:
# name: pages.{gitlab_dns_name}
# https: true
ssh: ~
ingress:
enabled: false
tls:
enabled: false
configureCertmanager: false
psql:
password:
useSecret: true
secret: gitlab-postgres
key: psql-password
host: _sanitized_
port: 5432
username: _sanitized_
database: gitlab_production
redis:
password:
enabled: false
host: _sanitized_
minio:
enabled: false
pages:
enabled: false
accessControl: false
port: 443
# objectStore:
# enabled: true
# bucket: {s3_pages}
# connection:
# secret: s3-storage-credentials
# key: connection
appConfig:
enableUsagePing: true
backups:
bucket: eu-central-1-gitlab-backups
#tmpBucket: eu-central-1-gitlab-backups-tmp
lfs:
enabled: true
bucket: eu-central-1-git-lfs
connection:
secret: s3-storage-credentials
key: connection
artifacts:
enabled: true
bucket: eu-central-1-gitlab-artifacts
connection:
secret: s3-storage-credentials
key: connection
uploads:
enabled: true
bucket: eu-central-1-gitlab-uploads
connection:
secret: s3-storage-credentials
key: connection
packages:
enabled: true
bucket: eu-central-1-gitlab-packages
connection:
secret: s3-storage-credentials
key: connection
terraformState:
enabled: true
bucket: eu-central-1-gitlab-terraformstate
connection:
secret: s3-storage-credentials
key: connection
omniauth:
enabled: true
autoLinkSamlUser: true
autoLinkLdapUser: true
blockAutoCreatedUsers: false
auto_sign_in_with_provider: 'saml'
syncProfileFromProvider: ['saml']
syncProfileAttributes: [email]
allowSingleSignOn:
- saml
providers:
- secret: saml-credentials
key: provider
cron_jobs:
personal_access_tokens_expiring_worker: # (prod only) Send emails for personal tokens which are about to expire
cron: "0 0 * * *,1,4"
admin_email_worker: # Send admin emails once a week
cron: "0 0 * * 0"
# Periodically update ci_runner_versions table with up-to-date versions and status
ci_runner_versions_reconciliation_worker:
cron: "@daily"
# Flag stuck CI builds as failed
stuck_ci_jobs_worker:
cron: "0 * * * *"
pipeline_schedule_worker:
cron: "19 * * * *"
expire_build_artifacts_worker:
cron: "50 * * * *"
repository_check_worker:
cron: "20 * * * *"
smtp:
enabled: true
address: _sanitized_
domain: _sanitized_
port: 587
user_name: _sanitized_
password:
secret: gitlab-smtp-password
key: password
authentication: "login"
starttls_auto: true
openssl_verify_mode: "peer"
pool: false
email:
from: _sanitized_
display_name: "GitLab"
nginx-ingress:
enabled: false
certmanager:
install: false
installCRDs: false
rbac:
create: false
prometheus:
install: false
redis:
install: false
upgradeCheck:
enabled: false
gitlab:
gitaly:
persistence:
enabled: true
storageClass: efs-sc
size: 50Gi
accessMode: ReadWriteOnce
toolbox:
backups:
cron:
enabled: true
schedule: "0 1 * * *"
extraArgs: "--maximum-backups 28"
objectStorage:
backend: s3
config:
secret: s3-storage-credentials
key: connection
persistence:
enabled: true
storageClass: efs-sc
accessMode: ReadWriteOnce
size: '10Gi'
annotations:
_sanitized_
webservice:
annotations:
_sanitized_
sidekiq:
annotations:
_sanitized_
migrations:
psql:
password:
secret: gitlab-postgres
key: psql-password
host: _sanitized_
port: 5432
postgresql:
install: false
gitlab-runner:
install: true
rbac:
create: true
runners:
locked: false
registry:
enabled: true
annotations:
_sanitized_
storage:
secret: s3-registry-storage-credentials
key: configCurrent behavior
Failure with exit code 1.
{"level":"fatal","msg":"restore: pipeline: 1 failures encountered:\n - @hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git (root/audiobookshelf): manager: remove repository: rpc error: code = Internal desc = staging repository for removal: rename /home/git/repositories/@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git /home/git/repositories/+gitaly/tmp/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git+removed: file exists\n","pid":64,"time":"2022-11-01T11:55:01.654Z"}
rake aborted!
Backup::Error: gitaly-backup exit status 1But it looks like the restore is ok, I can't explain why its doing this.
Expected behavior
Finish without error
Versions
- Chart: (tagged version | branch | hash
git rev-parse HEAD) - Platform:
- Cloud: EKS
- Kubernetes: (
kubectl version)- Client:
- Server:
- Helm: (
helm version)- Client: version.Info{Major:"1", Minor:"23+", GitVersion:"v1.23.7-eks-4721010", GitCommit:"b77d9473a02fbfa834afa67d677fd12d690b195f", GitTreeState:"clean", BuildDate:"2022-06-27T22:22:16Z", GoVersion:"go1.17.10", Compiler:"gc", Platform:"linux/amd64"}
- Server: version.Info{Major:"1", Minor:"23+", GitVersion:"v1.23.10-eks-15b7512", GitCommit:"cd6399691d9b1fed9ec20c9c5e82f5993c3f42cb", GitTreeState:"clean", BuildDate:"2022-08-31T19:17:01Z", GoVersion:"go1.17.13", Compiler:"gc", Platform:"linux/amd64"}
Relevant logs
.
.
.
{"command":"restore","gl_project_path":"test-group/test-pro1","level":"info","msg":"started restore","relative_path":"@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.design.git","storage_name":"default","time":"2022-11-01T11:55:00.864Z"}
{"command":"restore","gl_project_path":"test-group/test-pro1","level":"info","msg":"completed restore","relative_path":"@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.git","storage_name":"default","time":"2022-11-01T11:55:01.170Z"}
{"command":"restore","gl_project_path":"root/audiobookshelf","level":"info","msg":"started restore","relative_path":"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git","storage_name":"default","time":"2022-11-01T11:55:01.170Z"}
{"command":"restore","error":"manager: remove repository: rpc error: code = Internal desc = staging repository for removal: rename /home/git/repositories/@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git /home/git/repositories/+gitaly/tmp/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git+removed: file exists","gl_project_path":"root/audiobookshelf","level":"error","msg":"restore failed","relative_path":"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git","storage_name":"default","time":"2022-11-01T11:55:01.231Z"}
{"command":"restore","gl_project_path":"root/audiobookshelf.wiki","level":"info","msg":"started restore","relative_path":"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.wiki.git","storage_name":"default","time":"2022-11-01T11:55:01.231Z"}
{"command":"restore","error":"manager: repository skipped: restore bundle: filesystem sink: get reader for \"@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.design.bundle\": doesn't exist","gl_project_path":"test-group/test-pro1","level":"warning","msg":"skipped restore","relative_path":"@hashed/79/02/7902699be42c8a8e46fbbb4501726517e86b22c56a189f7625a6da49081b2451.design.git","storage_name":"default","time":"2022-11-01T11:55:01.255Z"}
{"command":"restore","gl_project_path":"root/audiobookshelf","level":"info","msg":"started restore","relative_path":"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.design.git","storage_name":"default","time":"2022-11-01T11:55:01.255Z"}
{"command":"restore","error":"manager: repository skipped: restore bundle: filesystem sink: get reader for \"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.wiki.bundle\": doesn't exist","gl_project_path":"root/audiobookshelf.wiki","level":"warning","msg":"skipped restore","relative_path":"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.wiki.git","storage_name":"default","time":"2022-11-01T11:55:01.633Z"}
{"command":"restore","error":"manager: repository skipped: restore bundle: filesystem sink: get reader for \"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.design.bundle\": doesn't exist","gl_project_path":"root/audiobookshelf","level":"warning","msg":"skipped restore","relative_path":"@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.design.git","storage_name":"default","time":"2022-11-01T11:55:01.653Z"}
{"level":"fatal","msg":"restore: pipeline: 1 failures encountered:\n - @hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git (root/audiobookshelf): manager: remove repository: rpc error: code = Internal desc = staging repository for removal: rename /home/git/repositories/@hashed/2c/62/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git /home/git/repositories/+gitaly/tmp/2c624232cdd221771294dfbb310aca000a0df6ac8b66b696d90ef06fdefb64a3.git+removed: file exists\n","pid":64,"time":"2022-11-01T11:55:01.654Z"}
rake aborted!
Backup::Error: gitaly-backup exit status 1
/srv/gitlab/lib/backup/gitaly_backup.rb:59:in `finish!'
/srv/gitlab/lib/backup/repositories.rb:37:in `restore'
/srv/gitlab/lib/backup/manager.rb:101:in `run_restore_task'
/srv/gitlab/lib/tasks/gitlab/backup.rake:29:in `block (4 levels) in <main>'
/srv/gitlab/vendor/bundle/ruby/2.7.0/gems/sentry-ruby-core-5.1.1/lib/sentry/rake.rb:26:in `execute'
/srv/gitlab/vendor/bundle/ruby/2.7.0/gems/rake-13.0.6/exe/rake:27:in `<top (required)>'
/srv/gitlab/bin/bundle:5:in `load'
/srv/gitlab/bin/bundle:5:in `<main>'
Tasks: TOP => gitlab:backup:repo:restore
(See full trace by running task with --trace)