Skip to content

Support the use of AWS Key Management Service for object storage

Summary

The AWS Key Management Service (KMS) can be used to manage keys that protect data stored in S3 buckets. While GitLab supports the use of external storage such as S3 to store objects used by GitLab, it does not have full support to configure and use KMS encryption.

Solution

The proposed solution is to pass the following Fog parameters with each object download/upload request.

  • encryption
  • x-amz-server-side-encryption-aws-kms-key-id
  • encryption_key
  • storage_class

Currently some of these parameters are passed with the connection request which causes warnings.

Proposal

  • Add the required parameters to the GitLab configuration files
  • Pass the required parameters with each upload/download request
  • Pass the required parameters to Workhorse (now that we've added an S3 client to Workhorse)

Original issue description as created by @szymonrychu

KMS encryption key mishandled during webUI pipeline rendering

Summary

KMS s3 object encryption doesn't work- objects gets created, pipeline webUI are crashing when trying to show the job/pipeline details.

Steps to reproduce

Create s3 buckets with KMS encryption enabled. Provide configuration for s3 connection as it states in: https://gitlab.com/charts/gitlab/tree/master/doc/advanced/external-object-storage#lfs-artifacts-uploads-packages. Extend the configuration with options from official AWS supported ruby library (options are not documented in chart repo) https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#put-instance_method. Example rails.s3.yaml:

provider: AWS
region: us-east-1
aws_access_key_id: BOGUS_ACCESS_KEY
aws_secret_access_key: BOGUS_SECRET_KEY
ssekms_key_id: arn:aws:kms:us-east-1:something:key/something-something-something
server_side_encryption: aws:kms

Configuration used

---
# Default values for gitlab-chart.
# This is a YAML-formatted file.
global:
  operator:
    enabled: false
  # gitlabVersion: master
  application:
    create: false
    links: []
  hosts:
    domain: something.something.com
    https: true
    externalIP:
    ssh: ~
  ingress:
    configureCertmanager: false
    annotations:
      kubernetes.io/ingress.class: "nginx-internal"
      nginx.ingress.kubernetes.io/ssl-redirect: "true"
    enabled: true
    tls:
      secretName: star-something-something-com
  initialRootPassword:
    secret: gitlab-gitlab-initial-root-password
    key: password

  psql:
    host: something-postgres.cluster-something.eu-west-1.rds.amazonaws.com
    port: 5432
    username: gitlab
    database: gitlab
    password:
      secret: 'gitlab-database'
      key: 'password'
  redis:
    password: {}
    persistence:
      storageClass: efs
      volumeName: gitlab-redis
  gitaly:
    authToken: {}
    internal:
      names: ['default']
    external: []
    persistence:
      storageClass: efs
      volumeName: gitlab-gitaly
  minio:
    enabled: false
  appConfig:
    omniauth:
      enabled: true
      autoSignInWithProvider: saml
      autoLinkSamlUser: true
      allowSingleSignOn: true
      providers:
        - secret: keycloak-saml
          key: provider
    enableUsagePing: true
    defaultCanCreateGroup: true
    usernameChangingEnabled: true
    issueClosingPattern:
    defaultTheme:
    defaultProjectsFeatures:
      issues: true
      mergeRequests: true
      wiki: true
      snippets: true
      builds: true
    webhookTimeout:
    gravatar:
      plainUrl:
      sslUrl:
    extra:
      googleAnalyticsId:
      piwikUrl:
      piwikSiteId:
    lfs:
      bucket: something-gitlab-lfs-storage
      connection:
        secret: s3access
        key: data
    artifacts:
      bucket: something-gitlab-artifacts-storage
      connection:
        secret: s3access
        key: data
    uploads:
      bucket: something-gitlab-uploads-storage
      connection:
        secret: s3access
        key: data
    backups:
      bucket: something-gitlab-backup-storage
      tmpBucket: gitlab-tmp-storage
    packages:
      bucket: something-gitlab-packages-storage
      connection:
        secret: s3access
        key: data
    incomingEmail:
      enabled: false
      address: ""
      host: "imap.gmail.com"
      port: 993
      ssl: true
      startTls: false
      user: ""
      password:
        secret: ""
        key: password
      mailbox: inbox
      idleTimeout: 60
  shell:
    port: 10022
    authToken: {}
    hostKeys: {}
  railsSecrets: {}
  registry:
    bucket: registry
    certificate: {}
    httpSecret: {}
  runner:
    registrationToken: {}
  # Outgoing email server settings
  smtp:
    enabled: false
    address: smtp.mailgun.org
    port: 2525
    user_name: ""
    password:
      secret: ""
      key: password
    # domain:
    authentication: "plain"
    starttls_auto: false
    openssl_verify_mode: "peer"
  # Email persona used in email sent by GitLab
  email:
    from: ''
    display_name: GitLab
    reply_to: ''
    subject_suffix: ''
  time_zone: UTC
  service:
    annotations:
      service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
  antiAffinity: soft
  workhorse: {}
  # configuration of certificates container & custom CA injection
  certificates:
    image:
      repository: registry.gitlab.com/gitlab-org/build/cng/alpine-certificates
      tag: 20171114-r3
    customCAs: []
    # - secret: custom-CA
    # - secret: more-custom-CAs

certmanager-issuer:
  email: "administrator@something.com"

certmanager:
  install: false

nginx-ingress:
  enabled: false

gitlab:
  task-runner:
    backups:
      objectStorage:
        config:
          secret: s3access
          key: s3cmd

prometheus:
  install: true
  rbac:
    create: true
  alertmanager:
    enabled: false
  alertmanagerFiles:
    alertmanager.yml: {}
  kubeStateMetrics:
    enabled: false
  nodeExporter:
    enabled: false
  pushgateway:
    enabled: false

redis-ha:
  nameOverride: redis
  enabled: false

postgresql:
  install: false

ingress:
  annotations: {}

shared-secrets:
  enabled: true
  rbac:
    create: true

registry:
  enabled: false

gitlab-runner:
  install: false

Current behavior

I want to use KMS for encrypting data at-rest in s3. Once I trigger the pipeline and the job is done I get an error occurred while fetching the job log. error banner. When I exit the view and get back to it I get http 500 error and can't view the pipeline or the job. It seems, that artifacts get uploaded to s3 bucket, view handler doesn't get how to show them up properly.

Expected behavior

Use KMS and be able to see the pipeline/job results. Documentation showing how to use KMS encryption for objects in s3.

Versions

  • Chart: v1.3.3
  • Platform:
    • Cloud: EKS
  • Kubernetes:
    • Client: v1.12.2
    • Server: v1.10.11-eks
  • Helm: (helm version)
    • Client: v2.11.0
    • Server: v2.11.0

Relevant logs

stern logs:

gitlab-unicorn-5659f7987b-llqmr unicorn [fog][WARNING] Unrecognized arguments: ssekms_key_id, server_side_encryption
gitlab-gitaly-0 gitaly time="2018-12-12T10:21:10Z" level=info msg="finished streaming call with code OK" grpc.code=OK grpc.meta.auth_version=v2 grpc.meta.client_name=gitlab-web grpc.method=ListCommitsByOid grpc.request.deadline="2018-12-12T10:21:40Z" grpc.request.fullMethod=/gitaly.CommitService/ListCommitsByOid grpc.request.glRepository=project-1 grpc.request.repoPath=something/something.git grpc.request.repoStorage=default grpc.request.topLevelGroup=something grpc.service=gitaly.CommitService grpc.start_time="2018-12-12T10:21:10Z" grpc.time_ms=38.462 peer.address="10.137.146.151:37556" span.kind=server system=grpc
gitlab-gitaly-0 gitaly time="2018-12-12T10:21:10Z" level=info msg="finished unary call with code OK" grpc.code=OK grpc.meta.auth_version=v2 grpc.meta.client_name=gitlab-web grpc.method=FindCommit grpc.request.deadline="2018-12-12T10:21:40Z" grpc.request.fullMethod=/gitaly.CommitService/FindCommit grpc.request.glRepository=project-1 grpc.request.repoPath=something/something.git grpc.request.repoStorage=default grpc.request.topLevelGroup=something grpc.service=gitaly.CommitService grpc.start_time="2018-12-12T10:21:10Z" grpc.time_ms=41.68 peer.address="10.137.146.151:37556" span.kind=server system=grpc
gitlab-unicorn-5659f7987b-n49rq unicorn Completed 200 OK in 272ms (Views: 1.1ms | ActiveRecord: 22.7ms | Elasticsearch: 0.0ms)
gitlab-unicorn-5659f7987b-n49rq unicorn
gitlab-unicorn-5659f7987b-n49rq unicorn ==> /var/log/gitlab/production_json.log <==
gitlab-unicorn-5659f7987b-n49rq unicorn {"method":"GET","path":"/something/something/-/jobs/1.json","format":"json","controller":"Projects::JobsController","action":"show","status":200,"duration":273.12,"view":1.09,"db":22.66,"time":"2018-12-12T10:21:10.601Z","params":[{"key":"namespace_id","value":"something"},{"key":"project_id","value":"something"},{"key":"id","value":"1"}],"remote_ip":"1.2.3.4","user_id":2,"username":"something.something","ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36","gitaly_calls":2}
gitlab-unicorn-5659f7987b-n49rq unicorn
gitlab-unicorn-5659f7987b-n49rq unicorn ==> /var/log/gitlab/production.log <==
gitlab-unicorn-5659f7987b-n49rq unicorn Started GET "/something/something/pipelines/1.json" for 1.2.3.4 at 2018-12-12 10:21:10 +0000
gitlab-unicorn-5659f7987b-n49rq unicorn Processing by Projects::PipelinesController#show as JSON
gitlab-unicorn-5659f7987b-n49rq unicorn   Parameters: {"namespace_id"=>"something", "project_id"=>"something", "id"=>"1"}
gitlab-unicorn-5659f7987b-llqmr unicorn Completed 500 Internal Server Error in 395ms (ActiveRecord: 7.7ms | Elasticsearch: 0.0ms)
gitlab-unicorn-5659f7987b-llqmr unicorn
gitlab-unicorn-5659f7987b-llqmr unicorn ==> /var/log/gitlab/production_json.log <==
gitlab-unicorn-5659f7987b-llqmr unicorn {"method":"GET","path":"/something/something/-/jobs/1/trace.json","format":"json","controller":"Projects::JobsController","action":"trace","status":500,"error":"Gitlab::HttpIO::FailedToGetChunkError: Gitlab::HttpIO::FailedToGetChunkError","duration":396.79,"view":0.0,"db":7.74,"time":"2018-12-12T10:21:10.594Z","params":[{"key":"state","value":""},{"key":"namespace_id","value":"something"},{"key":"project_id","value":"something"},{"key":"id","value":"1"}],"remote_ip":"1.2.3.4","user_id":2,"username":"something.something","ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"}
gitlab-unicorn-5659f7987b-llqmr unicorn
gitlab-unicorn-5659f7987b-llqmr unicorn ==> /var/log/gitlab/production.log <==
gitlab-unicorn-5659f7987b-llqmr unicorn
gitlab-unicorn-5659f7987b-llqmr unicorn Gitlab::HttpIO::FailedToGetChunkError (Gitlab::HttpIO::FailedToGetChunkError):
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/http_io.rb:156:in `get_chunk'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/http_io.rb:109:in `readline'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/http_io.rb:71:in `each_line'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/ci/ansi2html.rb:158:in `convert'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/ci/ansi2html.rb:30:in `convert'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/ci/trace/stream.rb:72:in `html_with_state'
gitlab-unicorn-5659f7987b-llqmr unicorn   app/controllers/projects/jobs_controller.rb:83:in `block (3 levels) in trace'
gitlab-unicorn-5659f7987b-llqmr unicorn   app/controllers/projects/jobs_controller.rb:74:in `block in trace'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/ci/trace.rb:80:in `read'
gitlab-unicorn-5659f7987b-llqmr unicorn   app/controllers/projects/jobs_controller.rb:73:in `trace'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/i18n.rb:55:in `with_locale'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/i18n.rb:61:in `with_user_locale'
gitlab-unicorn-5659f7987b-llqmr unicorn   app/controllers/application_controller.rb:427:in `set_locale'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/middleware/multipart.rb:101:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/request_profiler/middleware.rb:14:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   ee/lib/gitlab/jira/middleware.rb:15:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/middleware/go.rb:17:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/etag_caching/middleware.rb:11:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/middleware/rails_queue_duration.rb:22:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/metrics/rack_middleware.rb:15:in `block in call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/metrics/transaction.rb:53:in `run'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/metrics/rack_middleware.rb:15:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/middleware/read_only/controller.rb:40:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/middleware/read_only.rb:16:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/middleware/basic_health_check.rb:25:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/request_context.rb:20:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/metrics/requests_rack_middleware.rb:27:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn   lib/gitlab/middleware/release_env.rb:10:in `call'
gitlab-unicorn-5659f7987b-llqmr unicorn
gitlab-unicorn-5659f7987b-llqmr unicorn
Edited by Steven Wilson