Skip to content

Runner job creation 500 error with version 3.3.0 due to "Neither PUB key nor PRIV key: nested asn1 error"

Summary

Last night our cluster upgraded to the GitLab helm chart version 3.3.0. GitLab itself is running perfectly fine, but new jobs are "stuck" and don't seem to start at all.

GitLab runner error:

WARNING: Checking for jobs... failed                runner=1kKZP15G status=500 Internal Server Error

Unicorn error:

{
  "time": "2020-04-23T07:01:25.624Z",
  "severity": "INFO",
  "duration_s": 0.14,
  "db_duration_s": 0.03,
  "view_duration_s": 0.11,
  "status": 500,
  "method": "POST",
  "path": "/api/v4/jobs/request",
  "params": [
    {
      "key": "info",
      "value": {
        "name": "gitlab-runner",
        "version": "12.9.0",
        "revision": "4c96e5ad",
        "platform": "linux",
        "architecture": "amd64",
        "executor": "kubernetes",
        "shell": "bash",
        "features": {
          "variables": "[FILTERED]",
          "image": null,
          "services": null,
          "artifacts": null,
          "cache": null,
          "shared": null,
          "upload_multiple_artifacts": null,
          "upload_raw_artifacts": null,
          "session": null,
          "terminal": null,
          "refspecs": null,
          "masking": null,
          "proxy": null
        }
      }
    },
    {
      "key": "token",
      "value": "[FILTERED]"
    },
    {
      "key": "last_update",
      "value": "e91d3a82241ee367d1440ecd8a4110ea"
    }
  ],
  "host": "gitlab.enrise.com",
  "remote_ip": [FILTERED]",
  "ua": "gitlab-runner 12.9.0 (12-9-stable; go1.13.8; linux/amd64)",
  "route": "/api/:version/jobs/request",
  "exception.class": "OpenSSL::PKey::RSAError",
  "exception.message": "Neither PUB key nor PRIV key: nested asn1 error",
  "exception.backtrace": [
    "lib/gitlab/ci/jwt.rb:63:in `initialize'",
    "lib/gitlab/ci/jwt.rb:63:in `new'",
    "lib/gitlab/ci/jwt.rb:63:in `key'",
    "lib/gitlab/ci/jwt.rb:67:in `public_key'",
    "lib/gitlab/ci/jwt.rb:71:in `kid'",
    "lib/gitlab/ci/jwt.rb:23:in `encoded'",
    "lib/gitlab/ci/jwt.rb:10:in `for_build'",
    "app/models/ci/build.rb:983:in `block in job_jwt_variables'",
    "app/models/ci/build.rb:980:in `tap'",
    "app/models/ci/build.rb:980:in `job_jwt_variables'",
    "app/models/ci/build.rb:528:in `block in variables'",
    "lib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'",
    "app/models/ci/build.rb:525:in `variables'",
    "app/presenters/ci/build_runner_presenter.rb:121:in `block in git_depth_variable'",
    "lib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'",
    "app/presenters/ci/build_runner_presenter.rb:120:in `git_depth_variable'",
    "app/presenters/ci/build_runner_presenter.rb:28:in `git_depth'",
    "app/presenters/ci/build_runner_presenter.rb:40:in `refspecs'"
  ],
  "queue_duration_s": 0.01,
  "gitaly_calls": 2,
  "gitaly_duration_s": 0,
  "redis_calls": 11,
  "redis_duration_s": 0.0,
  "correlation_id": "xeVwShyQwk5"
}

We fixed this issue by rolling back to version 3.2.4, which is not a long term solution obviously.

Steps to reproduce

Upgrade from 3.2.4 to 3.3.0 and create a new job in the GitLab runner.

Configuration used

(Please provide a sanitized version of the configuration used wrapped in a code block (```yaml))

# Global GitLab config
global:
  # Ingress
  ingress:
    class: nginx
    configureCertmanager: false
    annotations:
      cert-manager.io/cluster-issuer: letsencrypt

  # Email
  email:
    [FILTERED]

  # Service
  hosts:
    domain: enrise.com

  # PostgreSQL database connection
  psql:
    password:
      key: password
      secret: postgres-password
    host: [FILTERED]
    username: enrise-gitlab
    database: enrise-gitlab

  # Disable local Minio persistent storage
  minio:
    enabled: false

  appConfig:
    # Persistent storage settings
    lfs:
      bucket: enrise-gitlab-lfs-storage
      connection:
        secret: global-bucket-config
        key: config
    artifacts:
      bucket: enrise-gitlab-artifacts-storage
      connection:
        secret: global-bucket-config
        key: config
    uploads:
      bucket: enrise-gitlab-uploads-storage
      connection:
        secret: global-bucket-config
        key: config
    packages:
      bucket: enrise-gitlab-packages-storage
      connection:
        secret: global-bucket-config
        key: config
    registry:
      bucket: enrise-gitlab-registry-storage
    backups:
      bucket: enrise-gitlab-backup-storage
      tmpBucket: enrise-gitlab-tmp-storage

    # LDAP settings
    ldap:
      [FILTERED]

  # Email service provider
  smtp:
    [FILTERED]

  # Timezone
  time_zone: Europe/Amsterdam

  # GitLab Edition
  edition: ce

# Disable PostgreSQL cluster database
postgresql:
  install: false

# Task runner object storage config
gitlab:

  unicorn:
    service:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/-/metrics"
    ingress:
      tls:
        secretName: enrise-gitlab-gitlab-tls
    # Temporary workaround for artifacts in 12.4.0, see https://gitlab.com/gitlab-org/charts/gitlab/issues/1647
    extraVolumes: |
      - name: shared-tmp
        emptyDir:
          medium: "Memory"
    extraVolumeMounts: |
      - name: shared-tmp
        mountPath: /tmp
        readOnly: false

  task-runner:
    backups:
      cron:
        enabled: true
        failedJobsHistoryLimit: 3
        schedule: "45 0 * * *"
        extraArgs: "--skip registry --skip lfs"
        successfulJobsHistoryLimit: 2
        resources:
          requests:
            cpu: 1
            memory: 6Gi
      objectStorage:
        config:
          secret: backup-bucket-config
          key: config

  gitaly:
    resources:
      requests:
        cpu: 100m
        memory: 4Gi

  sidekiq:
    resources:
      requests:
        cpu: 100m
        memory: 1.2Gi

  gitlab-shell:
    hpa:
      targetAverageValue: 250m

# Registry object storage config
registry:
  ingress:
    tls:
      secretName: enrise-gitlab-registry-tls
  storage:
    secret: registry-bucket-config
    key: config
    extraKey: gcs.json
  nodeSelector:
      gitlab.enrise.com/task: app
  resources:
    requests:
      cpu: 100m
      memory: 120Mi

# CI runner config
gitlab-runner:
  # Check for new jobs interval, in seconds
  checkInterval: 5
  # How many runners can be spawned in the cluster
  concurrent: 12
  # Node selector on which to run the main gitlab-runner controller
  nodeSelector:
      gitlab.enrise.com/task: app
  # Runner configuration
  runners:
    image: docker:latest
    # Do not lock for specific projects
    locked: false
    # https://docs.gitlab.com/runner/install/kubernetes.html#running-privileged-containers-for-the-runners
    privileged: true
    # Wait 10 minutes for a pod to become ready. This is needed so the runner node pool can scale up.
    pollTimeout: 600
    # Default runner environment variables
    env:
      DOCKER_HOST: tcp://localhost:2375
      DOCKER_TLS_CERTDIR: "" # Temporary workaround for https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
    # Build Container specific configuration
    builds:
      cpuRequests: 500m
      memoryRequests: 512Mi
    # Service Container specific configuration
    services:
      cpuRequests: 200m
      memoryRequests: 256Mi
    # Helper Container specific configuration
    helpers:
      cpuRequests: 200m
      memoryRequests: 256Mi
    # Build cache
    cache:
      cacheType: gcs
      cachePath: "gitlab_runner"
      cacheShared: true
      gcsBucketName: enrise-gitlab-runner-cache
      secretName: google-storage-sa
    # Node selector to select nodes on which to run CI jobs
    nodeSelector:
      gitlab.enrise.com/task: runner

minio:
  ingress:
    tls:
      secretName: enrise-gitlab-minio-tls

# Disable builtin ingress
nginx-ingress:
  enabled: false

# Disable builtin certificate manager
certmanager:
  install: false

# Disable builtin prometheus
prometheus:
  install: false

Expected behavior

Jobs are created just fine

Versions

  • Chart: 3.3.0
  • Platform:
    • Cloud: GKE
  • Kubernetes: (kubectl version)
    • Client: 1.14.10-gke.27
    • Server: 1.14.10-gke.27