Ontap s3 issues with GitLab-Uploads bucket

Summary

We are encountering an issue if try to run our GitLab instance with Ontap s3-Buckets (provided by NetApp) (reference: #3519 (closed)). This just seems to affect the GitLab-Uploads bucket, or more specifically, displaying avatars.

As you can see in the image, there are three types of icons that are not displayed here. Image is of course available on the s3 bucket in the specified path. The strange is, the problem varies depending on the http call:

/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png?width=48 -> Works

/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png?width=400 -> Doesn't work

/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png -> Doesn't work

Steps to reproduce

Use Ontap s3-Buckets (Version 9.13.1)
Deploy GitLab-Helm-Chart with reference to this buckets
Upload some project icons and user avatars

Configuration used

global:
  image:
    pullPolicy: IfNotPresent
  minio:
    enabled: false
  hosts:
    domain: gitlab.domain.example
    https: true
    gitlab:
      name: gitlab.domain.example
  email:
    from: git@gitlab.domain.example
  smtp:
    address: smtp.domain.example
    authentication: ""
    enabled: true
    port: 25
  ingress:
    enabled: true
    class: nginx
    configureCertmanager: false
    customCAs:
      - secret: ourcustomca
  time_zone: CET
  appConfig:
    enableUsagePing: false
    omniauth:
      enabled: true
      autoSignInWithProvider: ...
      syncProfileFromProvider: ...
      syncProfileAttributes: ['...']
      allowSingleSignOn: ['...']
      blockAutoCreatedUsers: ...
      autoLinkLdapUser: ...
      autoLinkSamlUser: ...
      autoLinkUser: ['...']
      allowBypassTwoFactor: ...
      providers:
        - secret: oursecret
    
    lfs:
      enabled: true
      bucket: gitlab-lfs
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    artifacts:
      enabled: true
      bucket: gitlab-artifacts
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    uploads:
      enabled: true
      bucket: gitlab-uploads
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    packages:
      enabled: true
      bucket: gitlab-packages
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    externalDiffs:
      enabled: true
      bucket: gitlab-external-diffs
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    terraformState:
      enabled: true
      bucket: gitlab-terraform-state
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    pseudonymizer:
      enabled: true
      bucket: gitlab-pseudonymizer
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    dependencyProxy:
      enabled: true
      bucket: gitlab-dependency-proxy
      connection:
        secret: gitlab-rails-storage
        key: rails.s3.yaml
    backups:
      bucket: gitlab-backup
      tmpBucket: gitlab-tmp-storage

postgresql:
  primary:
    configuration: |-
      # -----------------------------
      # PostgreSQL configuration file
      # -----------------------------
      #
      #------------------------------------------------------------------------------
      # CONNECTIONS AND AUTHENTICATION
      #------------------------------------------------------------------------------
      # - Connection Settings -
      listen_addresses = '*'
                                              # comma-separated list of addresses;
                                              # defaults to 'localhost'; use '*' for all
      port = '5432'
      max_connections = 1024
      superuser_reserved_connections = 3
      #------------------------------------------------------------------------------
      # WRITE-AHEAD LOG
      #------------------------------------------------------------------------------
      # - Settings -
      wal_level = 'replica'
      fsync = 'on'
                                              # (turning this off can cause
                                              # unrecoverable data corruption)
      max_wal_size = '400MB'
      #------------------------------------------------------------------------------
      # REPLICATION
      #------------------------------------------------------------------------------
      # - Sending Servers -
      # Set these on the master and on any standby that will send replication data.
      max_wal_senders = '16'
      wal_keep_size = '128MB'
      hot_standby = 'on'
      log_connections = 'false'
      log_disconnections = 'false'
      log_hostname = 'false'
      #------------------------------------------------------------------------------
      # CLIENT CONNECTION DEFAULTS
      #------------------------------------------------------------------------------
      # - Statement Behavior -
      client_min_messages = 'error'
                                              #   debug5
                                              #   debug4
                                              #   debug3
                                              #   debug2
                                              #   debug1
                                              #   log
                                              #   notice
                                              #   warning
                                              #   error
      # - Shared Library Preloading -
      shared_preload_libraries = 'pgaudit'
      #------------------------------------------------------------------------------
      # CONFIG FILE INCLUDES
      #------------------------------------------------------------------------------
      # These options allow settings to be loaded from files other than the
      # default postgresql.conf.  Note that these are directives, not variable
      # assignments, so they can usefully be given more than once.
      include_dir = 'conf.d'
      #include_dir = '...'                    # include files ending in '.conf' from
                                              # a directory, e.g., 'conf.d'
      #include_if_exists = '...'              # include file only if it exists
      #include = '...'                        # include file
      #------------------------------------------------------------------------------
      # CUSTOMIZED OPTIONS
      #------------------------------------------------------------------------------
      # Add settings for extensions here
      pgaudit.log_catalog = 'off'
  install: true
  image:
    tag: 13.14.0
  metrics:
    enabled: true
    serviceMonitor:
      enabled: true

redis:
  install: true
  metrics:
    enabled: true
    serviceMonitor:
      enabled: true

certmanager:
  install: false

prometheus:
  install: false
  rbac:
    create: false

nginx-ingress:
  enabled: false

gitlab:
  toolbox:
    extraEnvFrom:
      AWS_ACCESS_KEY_ID:
        secretKeyRef:
          name: storage-config
          key: accesskey
      AWS_SECRET_ACCESS_KEY:
        secretKeyRef:
          name: storage-config
          key: secretkey
    extraEnv:
      AWS_DEFAULT_REGION: us-east-1
      AWS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
    backups:
      objectStorage:
        config:
          secret: storage-config
          key: config
      cron:
        enabled: true
        schedule: "@daily"
        extraArgs: "--maximum-backups 14 --rsyncable --s3tool awscli --aws-s3-endpoint-url https://s3.gitlab.domain.example/"
        backoffLimit: 1
        failedJobsHistoryLimit: 3
        successfulJobsHistoryLimit: 3
        restartPolicy: "Never"
  webservice:
    metrics:
      enabled: true
      serviceMonitor:
        enabled: true
  gitlab-exporter:
    metrics:
      enabled: true
      serviceMonitor:
        enabled: true
  gitaly:
    metrics:
      enabled: true
      serviceMonitor:
        enabled: true
  kas:
    metrics:
      enabled: true
      serviceMonitor:
        enabled: true

# gitlab-rails-storage / rails.s3.yaml

provider: AWS
region: us-east-1
aws_access_key_id: ...
aws_secret_access_key: ...
enable_signature_v4_streaming: false
aws_signature_version: 4
endpoint: "https://s3.gitlab.domain.example:443"
path_style: true

Current behavior

Icons and avatars are not displayed continuously.

Expected behavior

All icons and avatars should be displayed as intended.

Versions

Chart: 7.9.4
Platform:
- Self-hosted: RKE2
Kubernetes: (kubectl version)
- Client: v1.26.10
- Server: v1.26.13+rke2r1
Helm: (helm version)
- Client: v3.13.2
- Server: v3.13.2

Relevant logs

gitlab-test-webservice-default-6d855c876f-frvkv webservice {"component": "gitlab","subcomponent":"production_json","method":"GET","path":"/uploads/-/system/user/avatar/6/avatar.png","format":"html","controller":"UploadsController","action":"show","status":200,"time":"2024-04-22T16:02:47.532Z","params":[{"key":"width","value":"64"},{"key":"model","value":"user"},{"key":"mounted_as","value":"avatar"},{"key":"id","value":"6"},{"key":"filename","value":"avatar.png"}],"correlation_id":"01HW38QAHCZQYH2FW02ZH6J2ZR","meta.caller_id":"UploadsController#show","meta.remote_ip":"....","meta.feature_category":"groups_and_projects","meta.user":"....","meta.user_id":6,"meta.client_id":"user/6","remote_ip":"...","user_id":6,"username":"....","ua":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","queue_duration_s":0.030544,"request_urgency":"default","target_duration_s":1,"redis_calls":6,"redis_allowed_cross_slot_calls":1,"redis_duration_s":0.016067,"redis_read_bytes":379,"redis_write_bytes":781,"redis_cache_calls":1,"redis_cache_duration_s":0.002399,"redis_cache_read_bytes":199,"redis_cache_write_bytes":76,"redis_sessions_calls":1,"redis_sessions_duration_s":0.002513,"redis_sessions_read_bytes":177,"redis_sessions_write_bytes":85,"redis_shared_state_calls":4,"redis_shared_state_allowed_cross_slot_calls":1,"redis_shared_state_duration_s":0.011155,"redis_shared_state_read_bytes":3,"redis_shared_state_write_bytes":620,"db_count":3,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":3,"db_main_count":3,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.013,"db_main_duration_s":0.013,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.032268,"mem_objects":14896,"mem_bytes":838064,"mem_mallocs":2629,"mem_total_bytes":1433904,"pid":42,"worker_id":"puma_0","rate_limiting_gates":[],"db_duration_s":0.00884,"view_duration_s":0.0,"duration_s":0.02091}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"correlation_id":"01HW38QAHCZQYH2FW02ZH6J2ZR","duration_s":0.062420468,"imageresizer.content_type":"image/png","imageresizer.original_filesize":75806,"imageresizer.status":"success","imageresizer.target_width":"64","level":"info","method":"GET","msg":"success","subsystem":"imageresizer","time":"2024-04-22T16:02:47Z","uri":"/uploads/-/system/user/avatar/6/avatar.png?width=64","written_bytes":8483}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"content_type":"image/png","correlation_id":"01HW38QAHCZQYH2FW02ZH6J2ZR","duration_ms":132,"host":"gitlab.domain.example","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"https://gitlab.domain.example/MyUser","remote_addr":"....:35506","remote_ip":"....","route":"","status":200,"system":"http","time":"2024-04-22T16:02:47Z","ttfb_ms":132,"uri":"/uploads/-/system/user/avatar/6/avatar.png?width=64","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","written_bytes":8483}
- gitlab-test-sidekiq-all-in-1-v2-788cc987b-p4klf › certificates
gitlab-test-webservice-default-6d855c876f-frvkv webservice {"component": "gitlab","subcomponent":"production_json","method":"GET","path":"/uploads/-/system/user/avatar/6/avatar.png","format":"html","controller":"UploadsController","action":"show","status":200,"time":"2024-04-22T16:02:47.874Z","params":[{"key":"model","value":"user"},{"key":"mounted_as","value":"avatar"},{"key":"id","value":"6"},{"key":"filename","value":"avatar.png"}],"correlation_id":"01HW38QAWA3G0QVYB4JYSS5E69","meta.caller_id":"UploadsController#show","meta.remote_ip":"....","meta.feature_category":"groups_and_projects","meta.user":"MyUser","meta.user_id":6,"meta.client_id":"user/6","remote_ip":"....","user_id":6,"username":"MyUser","ua":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","queue_duration_s":0.027487,"request_urgency":"default","target_duration_s":1,"redis_calls":5,"redis_allowed_cross_slot_calls":1,"redis_duration_s":0.012162000000000001,"redis_read_bytes":180,"redis_write_bytes":705,"redis_sessions_calls":1,"redis_sessions_duration_s":0.002399,"redis_sessions_read_bytes":177,"redis_sessions_write_bytes":85,"redis_shared_state_calls":4,"redis_shared_state_allowed_cross_slot_calls":1,"redis_shared_state_duration_s":0.009763,"redis_shared_state_read_bytes":3,"redis_shared_state_write_bytes":620,"db_count":3,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":3,"db_main_count":3,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.012,"db_main_duration_s":0.012,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.030334,"mem_objects":14420,"mem_bytes":782464,"mem_mallocs":2528,"mem_total_bytes":1359264,"pid":44,"worker_id":"puma_1","rate_limiting_gates":[],"db_duration_s":0.00676,"view_duration_s":0.0,"duration_s":0.01615}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"correlation_id":"01HW38QAWA3G0QVYB4JYSS5E69","level":"info","msg":"SendURL: sending","path":"/uploads/-/system/user/avatar/6/avatar.png","time":"2024-04-22T16:02:47Z","url":"https://s3.gitlab.domain.example/gitlab-test-uploads/user/avatar/6/avatar.png?response-content-disposition=inline%3B%20filename%3D%22avatar.png%22%3B%20filename%2A%3DUTF-8%27%27avatar.png\u0026response-content-type=image%2Fpng\u0026X-Amz-Expires=600\u0026X-Amz-Date=20240422T160247Z\u0026X-Amz-Algorithm=AWS4-HMAC-SHA256\u0026X-Amz-Credential=yourSuperCoolCred%2F20240422%2Fus-east-1%2Fs3%2Faws4_request\u0026X-Amz-SignedHeaders=host\u0026X-Amz-Signature=[FILTERED]"}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"content_type":"text/plain; charset=utf-8","correlation_id":"01HW38QAWA3G0QVYB4JYSS5E69","duration_ms":117,"host":"gitlab.domain.example","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"https://gitlab.domain.example/MyUser","remote_addr":"....:35506","remote_ip":"....","route":"","status":501,"system":"http","time":"2024-04-22T16:02:47Z","ttfb_ms":117,"uri":"/uploads/-/system/user/avatar/6/avatar.png","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","written_bytes":176}

gitlab-test-webservice-default-6d855c876f-frvkv webservice {"component": "gitlab","subcomponent":"production_json","method":"GET","path":"/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png","format":"html","controller":"UploadsController","action":"show","status":200,"time":"2024-04-22T16:07:31.711Z","params":[{"key":"width","value":"400"},{"key":"model","value":"project"},{"key":"mounted_as","value":"avatar"},{"key":"id","value":"13"},{"key":"filename","value":"gitlab-icon-rgb.png"}],"correlation_id":"01HW39001CD19PETJRD5K2F0A5","meta.caller_id":"UploadsController#show","meta.remote_ip":"....","meta.feature_category":"groups_and_projects","meta.user":"MyUser","meta.user_id":6,"meta.client_id":"user/6","remote_ip":"....","user_id":6,"username":"MyUser","ua":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","queue_duration_s":0.031905,"request_urgency":"default","target_duration_s":1,"redis_calls":5,"redis_allowed_cross_slot_calls":1,"redis_duration_s":0.012819,"redis_read_bytes":180,"redis_write_bytes":704,"redis_sessions_calls":1,"redis_sessions_duration_s":0.003445,"redis_sessions_read_bytes":177,"redis_sessions_write_bytes":85,"redis_shared_state_calls":4,"redis_shared_state_allowed_cross_slot_calls":1,"redis_shared_state_duration_s":0.009374,"redis_shared_state_read_bytes":3,"redis_shared_state_write_bytes":619,"db_count":6,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":6,"db_main_count":6,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.026,"db_main_duration_s":0.026,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.043464,"mem_objects":17816,"mem_bytes":1666136,"mem_mallocs":3655,"mem_total_bytes":2378776,"pid":44,"worker_id":"puma_1","rate_limiting_gates":[],"db_duration_s":0.01792,"view_duration_s":0.0,"duration_s":0.03798}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"correlation_id":"01HW39001CD19PETJRD5K2F0A5","level":"info","msg":"SendURL: sending","path":"/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png","time":"2024-04-22T16:07:31Z","url":"https://s3.gitlab.domain.example/gitlab-test-uploads/project/avatar/13/gitlab-icon-rgb.png?response-content-disposition=inline%3B%20filename%3D%22gitlab-icon-rgb.png%22%3B%20filename%2A%3DUTF-8%27%27gitlab-icon-rgb.png\u0026response-content-type=image%2Fpng\u0026X-Amz-Expires=600\u0026X-Amz-Date=20240422T160731Z\u0026X-Amz-Algorithm=AWS4-HMAC-SHA256\u0026X-Amz-Credential=yourSuperCoolCred%2F20240422%2Fus-east-1%2Fs3%2Faws4_request\u0026X-Amz-SignedHeaders=host\u0026X-Amz-Signature=[FILTERED]"}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"content_type":"text/plain; charset=utf-8","correlation_id":"01HW39001CD19PETJRD5K2F0A5","duration_ms":761,"host":"s3.gitlab.domain.example","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":".....:55688","remote_ip":"....","route":"","status":501,"system":"http","time":"2024-04-22T16:07:32Z","ttfb_ms":761,"uri":"/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png?width=400","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","written_bytes":176}