Migration container exits with error during new deployment
Summary
Migration container exits with error during new deployment using helm chart gitlab-2.0.3
and app version 12.0.3
. The log output of the migration container shows:
Missing Rails.application.secrets.secret_key_base for production environment. The secret will be generated and stored in config/secrets.yml.
rake aborted!
Errno::EACCES: Permission denied @ rb_sysopen - /srv/gitlab/config/secrets.yml
Steps to reproduce
helm install --timeout 600 --namespace=gitlab --name=gitlab gitlab/gitlab -f values.yaml
Configuration used
# Default values for gitlab/gitlab chart
## NOTICE
# Due to the scope and complexity of this chart, all possible values are
# not documented in this file. Extensive documentation for these values
# and more can be found at https://gitlab.com/charts/gitlab/
## Advanced Configuration
# Documentation for advanced configuration can be found under doc/advanced
# - external PostgreSQL
# - external Gitaly
# - external Redis
# - external NGINX
# - PersistentVolume configuration
# - external Object Storage providers
## The global properties are used to configure multiple charts at once.
## Extended documenation at doc/charts/globals.md
global:
## GitLab operator is Alpha. Not for production use.
operator:
enabled: false
## doc/installation/deployment.md#deploy-the-community-edition
# edition: ee
## doc/charts/globals.md#gitlab-version
# gitlabVersion: master
## doc/charts/globals.md#application-resource
application:
create: false
links: []
allowClusterRoles: true
## doc/charts/globals.md#configure-host-settings
hosts:
domain: ########
# hostSuffix:
https: true
externalIP: #########
ssh: ~
## doc/charts/globals.md#configure-ingress-settings
ingress:
configureCertmanager: false
annotations:
kubernetes.io/tls-acme: "true"
#certmanager.k8s.io/cluster-issuer: letsencrypt-prod
kubernetes.io/ingress.class: nginx
tls:
secretName: gitlab-gitlab-tls
enabled: true
# tls:
# enabled: true
## Initial root password for this GitLab installation
## Secret created according to doc/installation/secrets.md#initial-root-password
## If allowing shared-secrets generation, this is OPTIONAL.
initialRootPassword: {}
# secret: RELEASE-gitlab-initial-root-password
# key: password
## doc/charts/globals.md#configure-postgresql-settings
psql:
password: {}
#secret: gitlab-postgresql-password
#key: postgres-password
#host: gitlab-postgresql
port: 5432
username: gitlab
database: gitlabhq_production
## doc/charts/globals.md#configure-redis-settings
redis:
password:
enabled: true
# secret:
# key:
# host: redis.hostedsomewhere.else
# port: 6379
## doc/charts/globals.md#configure-gitaly-settings
gitaly:
authToken: {}
# secret:
# key:
internal:
names: ['default']
external: []
## doc/charts/globals.md#configure-minio-settings
minio:
enabled: true
credentials: {}
# secret:
## doc/charts/globals.md#configure-appconfig-settings
## Rails based portions of this chart share many settings
appConfig:
## doc/charts/globals.md#general-application-settings
enableUsagePing: true
enableImpersonation:
defaultCanCreateGroup: true
usernameChangingEnabled: true
issueClosingPattern:
defaultTheme:
defaultProjectsFeatures:
issues: true
mergeRequests: true
wiki: true
snippets: true
builds: true
webhookTimeout:
## doc/charts/globals.md#cron-jobs-related-settings
cron_jobs: {}
# stuck_ci_jobs_worker:
# cron: "0 * * * *"
# pipeline_schedule_worker:
# cron: "19 * * * *"
# expire_build_artifacts_worker:
# cron: "50 * * * *"
# repository_check_worker:
# cron: "20 * * * *"
# admin_email_worker:
# cron: "0 0 * * 0"
# repository_archive_cache_worker:
# cron: "0 * * * *"
# pages_domain_verification_cron_worker:
# cron: "*/15 * * * *"
# pseudonymizer_worker:
# cron: "0 * * * *"
# schedule_migrate_external_diffs_worker
# cron: "15 * * * *"
## doc/charts/globals.md#gravatarlibravatar-settings
gravatar:
plainUrl:
sslUrl:
## doc/charts/globals.md#hooking-analytics-services-to-the-gitlab-instance
extra:
googleAnalyticsId:
piwikUrl:
piwikSiteId:
## doc/charts/globals.md#lfs-artifacts-uploads-packages-external-mr-diffs
lfs:
bucket: git-lfs
connection: {}
# secret:
# key:
artifacts:
bucket: gitlab-artifacts
connection: {}
# secret:
# key:
uploads:
bucket: gitlab-uploads
connection: {}
# secret:
# key:
packages:
bucket: gitlab-packages
connection: {}
externalDiffs:
when:
bucket: gitlab-mr-diffs
connection: {}
## doc/charts/globals.md#pseudonymizer-settings
pseudonymizer:
configMap:
bucket: gitlab-pseudo
connection: {}
# secret:
# key:
backups:
bucket: gitlab-backups
tmpBucket: tmp
## doc/charts/globals.md#incoming-email-settings
## doc/installation/deployment.md#incoming-email
incomingEmail:
enabled: false
address: ""
host: "imap.gmail.com"
port: 993
ssl: true
startTls: false
user: ""
password:
secret: ""
key: password
mailbox: inbox
idleTimeout: 60
## doc/charts/globals.md#ldap
ldap:
servers: {}
## 'main' is the GitLab 'provider ID' of this LDAP server
# main:
# label: 'LDAP'
# host: '_your_ldap_server'
# port: 636
# uid: 'sAMAccountName'
# bind_dn: '_the_full_dn_of_the_user_you_will_bind_with'
# password:
# secret: _the_secret_containing_your_ldap_password
# key: _the_key_which_holds_your_ldap_password
# encryption: 'plain'
## doc/charts/globals.md#omniauth
omniauth:
enabled: false
autoSignInWithProvider:
syncProfileFromProvider: []
syncProfileAttributes: ['email']
allowSingleSignOn: ['saml']
blockAutoCreatedUsers: true
autoLinkLdapUser: false
autoLinkSamlUser: false
externalProviders: []
providers: []
# - secret: gitlab-google-oauth2
# key: provider
## End of global.appConfig
## doc/charts/globals.md#configure-gitlab-shell-settings
shell:
authToken: {}
# secret:
# key:
hostKeys: {}
image:
tag: 9.3.0
# secret:
## Rails application secrets
## Secret created according to doc/installation/secrets.md#gitlab-rails-secret
## If allowing shared-secrets generation, this is OPTIONAL.
railsSecrets: {}
# secret: gitlab-rails-secret
## doc/charts/globals.md#configure-registry-settings
##registry:
## bucket: registry
## certificate: {}
## # secret:
## httpSecret: {}
## # secret:
## # key:
## GitLab Runner
## Secret created according to doc/installation/secrets.md#gitlab-runner-secret
## If allowing shared-secrets generation, this is OPTIONAL.
runner:
registrationToken: {}
# secret:
## doc/installation/deployment.md#outgoing-email
## Outgoing email server settings
smtp:
enabled: false
address: smtp.mailgun.org
port: 2525
user_name: ""
## doc/installation/secrets.md#smtp-password
password:
secret: ""
key: password
# domain:
authentication: "plain"
starttls_auto: false
openssl_verify_mode: "peer"
## doc/installation/deployment.md#outgoing-email
## Email persona used in email sent by GitLab
email:
from: ''
display_name: GitLab
reply_to: ''
subject_suffix: ''
## Timezone for containers.
time_zone: UTC
## Global Service Annotations
service:
annotations: {}
antiAffinity: soft
## doc/installation/secrets.md#gitlab-workhorse-secret
workhorse: {}
# secret:
# key:
## doc/charts/globals.md#custom-certificate-authorities
# configuration of certificates container & custom CA injection
certificates:
image:
repository: registry.gitlab.com/gitlab-org/build/cng/alpine-certificates
tag: 20171114-r3
customCAs: []
# - secret: custom-CA
# - secret: more-custom-CAs
## End of global
upgradeCheck:
enabled: true
image:
repository: busybox
tag: latest
tolerations: []
resources:
requests:
cpu: 50m
## Settings to for the Let's Encrypt ACME Issuer
# certmanager-issuer:
## The email address to register certificates requested from Let's Encrypt.
## Required if using Let's Encrypt.
# email: email@example.com
## Installation & configuration of stable/cert-manager
## See requirements.yaml for current version
certmanager:
# Install cert-manager chart. Set to false if you already have cert-manager
# installed or if you are not using cert-manager.
install: false
# Other cert-manager configurations from upstream
# See https://github.com/kubernetes/charts/tree/master/stable/cert-manager#configuration
rbac:
create: true
## doc/charts/nginx/index.md
## doc/architecture/decisions.md#nginx-ingress
## Installation & configuration of charts/nginx
nginx-ingress:
enabled: false
tcpExternalConfig: "true"
controller:
config:
hsts-include-subdomains: "false"
server-name-hash-bucket-size: "256"
enable-vts-status: "true"
use-http2: "true"
ssl-ciphers: "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"
ssl-protocols: "TLSv1.1 TLSv1.2"
server-tokens: "false"
extraArgs:
force-namespace-isolation: ""
service:
externalTrafficPolicy: "Local"
resources:
requests:
cpu: 100m
memory: 100Mi
publishService:
enabled: true
replicaCount: 3
minAvailable: 2
scope:
enabled: true
stats:
enabled: true
metrics:
enabled: true
service:
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "10254"
defaultBackend:
minAvailable: 1
replicaCount: 2
resources:
requests:
cpu: 5m
memory: 5Mi
rbac:
create: true
serviceAccount:
create: true
## Installation & configuration of stable/prometheus
## See requirements.yaml for current version
prometheus:
install: false
rbac:
create: true
alertmanager:
enabled: false
alertmanagerFiles:
alertmanager.yml: {}
kubeStateMetrics:
enabled: false
nodeExporter:
enabled: false
pushgateway:
enabled: false
## Configuration of Redis
## doc/architecture/decisions.md#redis
## doc/charts/redis
redis:
enabled: true
## doc/architecture/decisions.md#redis-ha
## doc/charts/redis-ha
redis-ha:
enabled: false
nameOverride: redis
## Instllation & configuration of stable/prostgresql
## See requirements.yaml for current version
postgresql:
install: true
postgresUser: gitlab
postgresDatabase: gitlabhq_production
imageTag: 9.6.8
usePasswordFile: true
existingSecret: 'secret'
metrics:
enabled: true
## Optionally define additional custom metrics
## ref: https://github.com/wrouesnel/postgres_exporter#adding-new-metrics-via-a-config-file
## Installation & configuration charts/registry
## doc/architecture/decisions.md#registry
## doc/charts/registry/
registry:
enabled: true
ingress:
tls:
secretName: gitlab-registry-tls
minio:
#enabled: true
ingress:
tls:
secretName: gitlab-minio-tls
## Automatic shared secret generation
## doc/installation/secrets.md
## doc/charts/shared-secrets
shared-secrets:
enabled: true
rbac:
create: true
## Installation & configuration of gitlab/gitlab-runner
## See requirements.yaml for current version
gitlab-runner:
install: true
rbac:
create: true
runners:
locked: false
cache:
cacheType: s3
s3BucketName: runner-cache
cacheShared: true
s3BucketLocation: us-east-1
s3CachePath: gitlab-runner
s3CacheInsecure: false
gitlab:
unicorn:
enabled: true
ingress:
tls:
secretName: gitlab-gitlab-tls
sidekiq:
enabled: true
## Settings for individual sub-charts under GitLab
## Note: Many of these settings are configurable via globals
# gitlab:
## doc/charts/gitlab/migrations
# migrations:
# enabled: false
## doc/charts/gitlab/unicorn
# unicorn:
# enabled: false
## doc/charts/gitlab/sidekiq
# sidekiq:
# enabled: false
## doc/charts/gitlab/gitaly
# gitaly:
# enabled: false
## doc/charts/gitlab/gitlab-shell
# gitlab-shell:
# enabled: false
Current behavior
sidekiq-all-in-1 and unicorn PODs ends up in Init:CrashLoopBackOff due to the migration POD is not completed successfully.
Expected behavior
Migration container should complete successfully so that the dependency init containers for sidekiq and unicorn can proceed.
Versions
- Chart: gitlab-2.0.3
- Platform:
- Cloud: (GKE | AKS | EKS | ?)
- Self-hosted: Ubuntu 16.04; kubernetes v1.15.0 deployed via kubeadm
- Kubernetes:
- Client: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", BuildDate:"2019-06-20T04:49:16Z", GoVersion:"go1.12.6", Compiler:"gc", Platform:"darwin/amd64"}
- Server: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", BuildDate:"2019-06-19T16:32:14Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
- Helm: (
helm version
)- Client: &version.Version{SemVer:"v2.14.1", GitCommit:"5270352a09c7e8b6e8c9593002a73535276507c0", GitTreeState:"clean"}
- Server: &version.Version{SemVer:"v2.14.1", GitCommit:"5270352a09c7e8b6e8c9593002a73535276507c0", GitTreeState:"clean"}
Relevant logs
Log from migration pod:
+ /scripts/set-config /var/opt/gitlab/templates /srv/gitlab/config
Begin parsing .erb files from /var/opt/gitlab/templates
Writing /srv/gitlab/config/database.yml
Writing /srv/gitlab/config/resque.yml
Writing /srv/gitlab/config/gitlab.yml
Copying other config files found in /var/opt/gitlab/templates
+ exec /scripts/wait-for-deps /scripts/db-migrate
Checking database connection and schema version
Missing Rails.application.secrets.secret_key_base for production environment. The secret will be generated and stored in config/secrets.yml.
rake aborted!
Errno::EACCES: Permission denied @ rb_sysopen - /srv/gitlab/config/secrets.yml
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write'
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write_secrets_yml'
/srv/gitlab/config/initializers/01_secret_token.rb:46:in `create_tokens'
/srv/gitlab/config/initializers/01_secret_token.rb:106:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `block in load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:655:in `block in load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/notifications.rb:168:in `instrument'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:654:in `load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:612:in `block (2 levels) in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `block in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `instance_exec'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `run'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:59:in `block in run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `tsort_each_child'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:58:in `run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:353:in `initialize!'
/srv/gitlab/config/environment.rb:6:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `block in require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:329:in `require_environment!'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:445:in `block in run_tasks_blocks'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/rake-12.3.2/exe/rake:27:in `<top (required)>'
/srv/gitlab/bin/bundle:3:in `load'
/srv/gitlab/bin/bundle:3:in `<main>'
Tasks: TOP => db:version => environment
(See full trace by running task with --trace)
Checking database connection and schema version
Missing Rails.application.secrets.secret_key_base for production environment. The secret will be generated and stored in config/secrets.yml.
rake aborted!
Errno::EACCES: Permission denied @ rb_sysopen - /srv/gitlab/config/secrets.yml
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write'
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write_secrets_yml'
/srv/gitlab/config/initializers/01_secret_token.rb:46:in `create_tokens'
/srv/gitlab/config/initializers/01_secret_token.rb:106:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `block in load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:655:in `block in load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/notifications.rb:168:in `instrument'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:654:in `load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:612:in `block (2 levels) in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `block in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `instance_exec'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `run'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:59:in `block in run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `tsort_each_child'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:58:in `run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:353:in `initialize!'
/srv/gitlab/config/environment.rb:6:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `block in require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:329:in `require_environment!'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:445:in `block in run_tasks_blocks'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/rake-12.3.2/exe/rake:27:in `<top (required)>'
/srv/gitlab/bin/bundle:3:in `load'
/srv/gitlab/bin/bundle:3:in `<main>'
Tasks: TOP => db:version => environment
(See full trace by running task with --trace)
Checking database connection and schema version