Migration container exits with error during new deployment

Summary

Migration container exits with error during new deployment using helm chart gitlab-2.0.3 and app version 12.0.3. The log output of the migration container shows:

Missing Rails.application.secrets.secret_key_base for production environment. The secret will be generated and stored in config/secrets.yml.
rake aborted!
Errno::EACCES: Permission denied @ rb_sysopen - /srv/gitlab/config/secrets.yml

Steps to reproduce

helm install --timeout 600 --namespace=gitlab --name=gitlab gitlab/gitlab -f values.yaml

Configuration used

# Default values for gitlab/gitlab chart

## NOTICE
# Due to the scope and complexity of this chart, all possible values are
# not documented in this file. Extensive documentation for these values
# and more can be found at https://gitlab.com/charts/gitlab/

## Advanced Configuration
# Documentation for advanced configuration can be found under doc/advanced
# - external PostgreSQL
# - external Gitaly
# - external Redis
# - external NGINX
# - PersistentVolume configuration
# - external Object Storage providers

## The global properties are used to configure multiple charts at once.
## Extended documenation at doc/charts/globals.md
global:
  ## GitLab operator is Alpha. Not for production use.
  operator:
    enabled: false

  ## doc/installation/deployment.md#deploy-the-community-edition
  # edition: ee

  ## doc/charts/globals.md#gitlab-version
  # gitlabVersion: master

  ## doc/charts/globals.md#application-resource
  application:
    create: false
    links: []
    allowClusterRoles: true
  ## doc/charts/globals.md#configure-host-settings
  hosts:
    domain: ########
    # hostSuffix:
    https: true
    externalIP: #########
    ssh: ~

  ## doc/charts/globals.md#configure-ingress-settings
  ingress:
    configureCertmanager: false
    annotations:
      kubernetes.io/tls-acme: "true"
      #certmanager.k8s.io/cluster-issuer: letsencrypt-prod
      kubernetes.io/ingress.class: nginx
    tls:
      secretName: gitlab-gitlab-tls
    
    enabled: true
    # tls:
    #   enabled: true

  ## Initial root password for this GitLab installation
  ## Secret created according to doc/installation/secrets.md#initial-root-password
  ## If allowing shared-secrets generation, this is OPTIONAL.
  initialRootPassword: {}
    # secret: RELEASE-gitlab-initial-root-password
    # key: password

  ## doc/charts/globals.md#configure-postgresql-settings
  psql:
    password: {}
      #secret: gitlab-postgresql-password
      #key: postgres-password
    #host: gitlab-postgresql
    port: 5432
    username: gitlab
    database: gitlabhq_production

  ## doc/charts/globals.md#configure-redis-settings
  redis:
    password:
      enabled: true
      # secret:
      # key:
    # host: redis.hostedsomewhere.else
    # port: 6379

  ## doc/charts/globals.md#configure-gitaly-settings
  gitaly:
    authToken: {}
      # secret:
      # key:
    internal:
      names: ['default']
    external: []

  ## doc/charts/globals.md#configure-minio-settings
  minio:
    enabled: true
    credentials: {}
      # secret:

  ## doc/charts/globals.md#configure-appconfig-settings
  ## Rails based portions of this chart share many settings
  appConfig:
    ## doc/charts/globals.md#general-application-settings
    enableUsagePing: true
    enableImpersonation:
    defaultCanCreateGroup: true
    usernameChangingEnabled: true
    issueClosingPattern:
    defaultTheme:
    defaultProjectsFeatures:
      issues: true
      mergeRequests: true
      wiki: true
      snippets: true
      builds: true
    webhookTimeout:

    ## doc/charts/globals.md#cron-jobs-related-settings
    cron_jobs: {}
      # stuck_ci_jobs_worker:
      #   cron: "0 * * * *"
      # pipeline_schedule_worker:
      #   cron: "19 * * * *"
      # expire_build_artifacts_worker:
      #   cron: "50 * * * *"
      # repository_check_worker:
      #   cron: "20 * * * *"
      # admin_email_worker:
      #   cron: "0 0 * * 0"
      # repository_archive_cache_worker:
      #   cron: "0 * * * *"
      # pages_domain_verification_cron_worker:
      #   cron: "*/15 * * * *"
      # pseudonymizer_worker:
      #   cron: "0 * * * *"
      # schedule_migrate_external_diffs_worker
      #   cron: "15 * * * *"

    ## doc/charts/globals.md#gravatarlibravatar-settings
    gravatar:
      plainUrl:
      sslUrl:

    ## doc/charts/globals.md#hooking-analytics-services-to-the-gitlab-instance
    extra:
      googleAnalyticsId:
      piwikUrl:
      piwikSiteId:

    ## doc/charts/globals.md#lfs-artifacts-uploads-packages-external-mr-diffs
    lfs:
      bucket: git-lfs
      connection: {}
        # secret:
        # key:
    artifacts:
      bucket: gitlab-artifacts
      connection: {}
        # secret:
        # key:
    uploads:
      bucket: gitlab-uploads
      connection: {}
        # secret:
        # key:
    packages:
      bucket: gitlab-packages
      connection: {}
    externalDiffs:
      when:
      bucket: gitlab-mr-diffs
      connection: {}

    ## doc/charts/globals.md#pseudonymizer-settings
    pseudonymizer:
      configMap:
      bucket: gitlab-pseudo
      connection: {}
        # secret:
        # key:
    backups:
      bucket: gitlab-backups
      tmpBucket: tmp

    ## doc/charts/globals.md#incoming-email-settings
    ## doc/installation/deployment.md#incoming-email
    incomingEmail:
      enabled: false
      address: ""
      host: "imap.gmail.com"
      port: 993
      ssl: true
      startTls: false
      user: ""
      password:
        secret: ""
        key: password
      mailbox: inbox
      idleTimeout: 60

    ## doc/charts/globals.md#ldap
    ldap:
      servers: {}
      ## 'main' is the GitLab 'provider ID' of this LDAP server
      # main:
      #   label: 'LDAP'
      #   host: '_your_ldap_server'
      #   port: 636
      #   uid: 'sAMAccountName'
      #   bind_dn: '_the_full_dn_of_the_user_you_will_bind_with'
      #   password:
      #     secret: _the_secret_containing_your_ldap_password
      #     key: _the_key_which_holds_your_ldap_password
      #   encryption: 'plain'

    ## doc/charts/globals.md#omniauth
    omniauth:
      enabled: false
      autoSignInWithProvider:
      syncProfileFromProvider: []
      syncProfileAttributes: ['email']
      allowSingleSignOn: ['saml']
      blockAutoCreatedUsers: true
      autoLinkLdapUser: false
      autoLinkSamlUser: false
      externalProviders: []
      providers: []
      # - secret: gitlab-google-oauth2
      #   key: provider
  ## End of global.appConfig

  ## doc/charts/globals.md#configure-gitlab-shell-settings
  shell:
    authToken: {}
      # secret:
      # key:
    hostKeys: {}
    image:
      tag: 9.3.0
      # secret:

  ## Rails application secrets
  ## Secret created according to doc/installation/secrets.md#gitlab-rails-secret
  ## If allowing shared-secrets generation, this is OPTIONAL.
  railsSecrets: {}
  #   secret: gitlab-rails-secret

  ## doc/charts/globals.md#configure-registry-settings
  ##registry:
  ##  bucket: registry
  ##  certificate: {}
  ##    # secret:
  ##  httpSecret: {}
  ##    # secret:
  ##    # key:

  ## GitLab Runner
  ## Secret created according to doc/installation/secrets.md#gitlab-runner-secret
  ## If allowing shared-secrets generation, this is OPTIONAL.
  runner:
    registrationToken: {}
      # secret:

  ## doc/installation/deployment.md#outgoing-email
  ## Outgoing email server settings
  smtp:
    enabled: false
    address: smtp.mailgun.org
    port: 2525
    user_name: ""
    ## doc/installation/secrets.md#smtp-password
    password:
      secret: ""
      key: password
    # domain:
    authentication: "plain"
    starttls_auto: false
    openssl_verify_mode: "peer"

  ## doc/installation/deployment.md#outgoing-email
  ## Email persona used in email sent by GitLab
  email:
    from: ''
    display_name: GitLab
    reply_to: ''
    subject_suffix: ''

  ## Timezone for containers.
  time_zone: UTC

  ## Global Service Annotations
  service:
    annotations: {}

  antiAffinity: soft

  ## doc/installation/secrets.md#gitlab-workhorse-secret
  workhorse: {}
    # secret:
    # key:

  ## doc/charts/globals.md#custom-certificate-authorities
  # configuration of certificates container & custom CA injection
  certificates:
    image:
      repository: registry.gitlab.com/gitlab-org/build/cng/alpine-certificates
      tag: 20171114-r3
    customCAs: []
    # - secret: custom-CA
    # - secret: more-custom-CAs
## End of global

upgradeCheck:
  enabled: true
  image:
    repository: busybox
    tag: latest
  tolerations: []
  resources:
    requests:
      cpu: 50m

## Settings to for the Let's Encrypt ACME Issuer
# certmanager-issuer:
  ## The email address to register certificates requested from Let's Encrypt.
  ## Required if using Let's Encrypt.
  # email: email@example.com

## Installation & configuration of stable/cert-manager
## See requirements.yaml for current version
certmanager:
  # Install cert-manager chart. Set to false if you already have cert-manager
  # installed or if you are not using cert-manager.
  install: false
  # Other cert-manager configurations from upstream
  # See https://github.com/kubernetes/charts/tree/master/stable/cert-manager#configuration
  rbac:
    create: true

## doc/charts/nginx/index.md
## doc/architecture/decisions.md#nginx-ingress
## Installation & configuration of charts/nginx
nginx-ingress:
  enabled: false
  tcpExternalConfig: "true"
  controller:
    config:
      hsts-include-subdomains: "false"
      server-name-hash-bucket-size: "256"
      enable-vts-status: "true"
      use-http2: "true"
      ssl-ciphers: "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"
      ssl-protocols: "TLSv1.1 TLSv1.2"
      server-tokens: "false"
    extraArgs:
      force-namespace-isolation: ""
    service:
      externalTrafficPolicy: "Local"
    resources:
      requests:
        cpu: 100m
        memory: 100Mi
    publishService:
      enabled: true
    replicaCount: 3
    minAvailable: 2
    scope:
      enabled: true
    stats:
      enabled: true
    metrics:
      enabled: true
      service:
        annotations:
          prometheus.io/scrape: "true"
          prometheus.io/port: "10254"
  defaultBackend:
    minAvailable: 1
    replicaCount: 2
    resources:
      requests:
        cpu: 5m
        memory: 5Mi
  rbac:
    create: true
  serviceAccount:
    create: true

## Installation & configuration of stable/prometheus
## See requirements.yaml for current version
prometheus:
  install: false
  rbac:
    create: true
  alertmanager:
    enabled: false
  alertmanagerFiles:
    alertmanager.yml: {}
  kubeStateMetrics:
    enabled: false
  nodeExporter:
    enabled: false
  pushgateway:
    enabled: false

## Configuration of Redis
## doc/architecture/decisions.md#redis
## doc/charts/redis
redis:
  enabled: true
## doc/architecture/decisions.md#redis-ha
## doc/charts/redis-ha
redis-ha:
  enabled: false
  nameOverride: redis

## Instllation & configuration of stable/prostgresql
## See requirements.yaml for current version
postgresql:
  install: true
  postgresUser: gitlab
  postgresDatabase: gitlabhq_production
  imageTag: 9.6.8
  usePasswordFile: true
  existingSecret: 'secret'
  metrics:
    enabled: true
    ## Optionally define additional custom metrics
    ## ref: https://github.com/wrouesnel/postgres_exporter#adding-new-metrics-via-a-config-file

## Installation & configuration charts/registry
## doc/architecture/decisions.md#registry
## doc/charts/registry/
registry:
  enabled: true
  ingress:
    tls:
      secretName: gitlab-registry-tls

minio:
  #enabled: true
  ingress:
    tls:
      secretName: gitlab-minio-tls

## Automatic shared secret generation
## doc/installation/secrets.md
## doc/charts/shared-secrets
shared-secrets:
  enabled: true
  rbac:
    create: true

## Installation & configuration of gitlab/gitlab-runner
## See requirements.yaml for current version
gitlab-runner:
  install: true
  rbac:
    create: true
  runners:
    locked: false
    cache:
      cacheType: s3
      s3BucketName: runner-cache
      cacheShared: true
      s3BucketLocation: us-east-1
      s3CachePath: gitlab-runner
      s3CacheInsecure: false

gitlab:
  unicorn:
    enabled: true
    ingress:
      tls:
        secretName: gitlab-gitlab-tls
  sidekiq:
    enabled: true  

## Settings for individual sub-charts under GitLab
## Note: Many of these settings are configurable via globals
# gitlab:
## doc/charts/gitlab/migrations
#   migrations:
#     enabled: false
## doc/charts/gitlab/unicorn
#   unicorn:
#     enabled: false
## doc/charts/gitlab/sidekiq
# sidekiq:
#     enabled: false
## doc/charts/gitlab/gitaly
#   gitaly:
#     enabled: false
## doc/charts/gitlab/gitlab-shell
#   gitlab-shell:
#     enabled: false

Current behavior

sidekiq-all-in-1 and unicorn PODs ends up in Init:CrashLoopBackOff due to the migration POD is not completed successfully.

Expected behavior

Migration container should complete successfully so that the dependency init containers for sidekiq and unicorn can proceed.

Versions

• Chart: gitlab-2.0.3
• Platform:
  • Cloud: (GKE | AKS | EKS | ?)
  • Self-hosted: Ubuntu 16.04; kubernetes v1.15.0 deployed via kubeadm
• Kubernetes:
  • Client: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", BuildDate:"2019-06-20T04:49:16Z", GoVersion:"go1.12.6", Compiler:"gc", Platform:"darwin/amd64"}
  • Server: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", BuildDate:"2019-06-19T16:32:14Z", GoVersion:"go1.12.5", Compiler:"gc", Platform:"linux/amd64"}
• Helm: (helm version)
  • Client: &version.Version{SemVer:"v2.14.1", GitCommit:"5270352a09c7e8b6e8c9593002a73535276507c0", GitTreeState:"clean"}
  • Server: &version.Version{SemVer:"v2.14.1", GitCommit:"5270352a09c7e8b6e8c9593002a73535276507c0", GitTreeState:"clean"}

Relevant logs

Log from migration pod:

+ /scripts/set-config /var/opt/gitlab/templates /srv/gitlab/config
Begin parsing .erb files from /var/opt/gitlab/templates
Writing /srv/gitlab/config/database.yml
Writing /srv/gitlab/config/resque.yml
Writing /srv/gitlab/config/gitlab.yml
Copying other config files found in /var/opt/gitlab/templates
+ exec /scripts/wait-for-deps /scripts/db-migrate
Checking database connection and schema version
Missing Rails.application.secrets.secret_key_base for production environment. The secret will be generated and stored in config/secrets.yml.
rake aborted!
Errno::EACCES: Permission denied @ rb_sysopen - /srv/gitlab/config/secrets.yml
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write'
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write_secrets_yml'
/srv/gitlab/config/initializers/01_secret_token.rb:46:in `create_tokens'
/srv/gitlab/config/initializers/01_secret_token.rb:106:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `block in load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:655:in `block in load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/notifications.rb:168:in `instrument'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:654:in `load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:612:in `block (2 levels) in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `block in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `instance_exec'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `run'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:59:in `block in run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `tsort_each_child'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:58:in `run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:353:in `initialize!'
/srv/gitlab/config/environment.rb:6:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `block in require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:329:in `require_environment!'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:445:in `block in run_tasks_blocks'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/rake-12.3.2/exe/rake:27:in `<top (required)>'
/srv/gitlab/bin/bundle:3:in `load'
/srv/gitlab/bin/bundle:3:in `<main>'
Tasks: TOP => db:version => environment
(See full trace by running task with --trace)
Checking database connection and schema version
Missing Rails.application.secrets.secret_key_base for production environment. The secret will be generated and stored in config/secrets.yml.
rake aborted!
Errno::EACCES: Permission denied @ rb_sysopen - /srv/gitlab/config/secrets.yml
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write'
/srv/gitlab/config/initializers/01_secret_token.rb:103:in `write_secrets_yml'
/srv/gitlab/config/initializers/01_secret_token.rb:46:in `create_tokens'
/srv/gitlab/config/initializers/01_secret_token.rb:106:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `block in load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:286:in `load'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:655:in `block in load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/notifications.rb:168:in `instrument'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:654:in `load_config_initializer'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:612:in `block (2 levels) in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/engine.rb:611:in `block in <class:Engine>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `instance_exec'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:30:in `run'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:59:in `block in run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `each'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:48:in `tsort_each_child'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/initializable.rb:58:in `run_initializers'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:353:in `initialize!'
/srv/gitlab/config/environment.rb:6:in `<top (required)>'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `block in require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:258:in `load_dependency'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/activesupport-5.1.7/lib/active_support/dependencies.rb:292:in `require'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:329:in `require_environment!'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/railties-5.1.7/lib/rails/application.rb:445:in `block in run_tasks_blocks'
/srv/gitlab/vendor/bundle/ruby/2.6.0/gems/rake-12.3.2/exe/rake:27:in `<top (required)>'
/srv/gitlab/bin/bundle:3:in `load'
/srv/gitlab/bin/bundle:3:in `<main>'
Tasks: TOP => db:version => environment
(See full trace by running task with --trace)
Checking database connection and schema version