Ontap s3 issues with GitLab-Uploads bucket
Summary
We are encountering an issue if try to run our GitLab instance with Ontap s3-Buckets (provided by NetApp) (reference: #3519 (closed)). This just seems to affect the GitLab-Uploads bucket, or more specifically, displaying avatars.
As you can see in the image, there are three types of icons that are not displayed here. Image is of course available on the s3 bucket in the specified path. The strange is, the problem varies depending on the http call:
- /uploads/-/system/project/avatar/13/gitlab-icon-rgb.png?width=48 -> Works
- /uploads/-/system/project/avatar/13/gitlab-icon-rgb.png?width=400 -> Doesn't work
Steps to reproduce
- Use Ontap s3-Buckets (Version 9.13.1)
- Deploy GitLab-Helm-Chart with reference to this buckets
- Upload some project icons and user avatars
Configuration used
global:
image:
pullPolicy: IfNotPresent
minio:
enabled: false
hosts:
domain: gitlab.domain.example
https: true
gitlab:
name: gitlab.domain.example
email:
from: git@gitlab.domain.example
smtp:
address: smtp.domain.example
authentication: ""
enabled: true
port: 25
ingress:
enabled: true
class: nginx
configureCertmanager: false
customCAs:
- secret: ourcustomca
time_zone: CET
appConfig:
enableUsagePing: false
omniauth:
enabled: true
autoSignInWithProvider: ...
syncProfileFromProvider: ...
syncProfileAttributes: ['...']
allowSingleSignOn: ['...']
blockAutoCreatedUsers: ...
autoLinkLdapUser: ...
autoLinkSamlUser: ...
autoLinkUser: ['...']
allowBypassTwoFactor: ...
providers:
- secret: oursecret
lfs:
enabled: true
bucket: gitlab-lfs
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
artifacts:
enabled: true
bucket: gitlab-artifacts
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
uploads:
enabled: true
bucket: gitlab-uploads
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
packages:
enabled: true
bucket: gitlab-packages
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
externalDiffs:
enabled: true
bucket: gitlab-external-diffs
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
terraformState:
enabled: true
bucket: gitlab-terraform-state
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
pseudonymizer:
enabled: true
bucket: gitlab-pseudonymizer
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
dependencyProxy:
enabled: true
bucket: gitlab-dependency-proxy
connection:
secret: gitlab-rails-storage
key: rails.s3.yaml
backups:
bucket: gitlab-backup
tmpBucket: gitlab-tmp-storage
postgresql:
primary:
configuration: |-
# -----------------------------
# PostgreSQL configuration file
# -----------------------------
#
#------------------------------------------------------------------------------
# CONNECTIONS AND AUTHENTICATION
#------------------------------------------------------------------------------
# - Connection Settings -
listen_addresses = '*'
# comma-separated list of addresses;
# defaults to 'localhost'; use '*' for all
port = '5432'
max_connections = 1024
superuser_reserved_connections = 3
#------------------------------------------------------------------------------
# WRITE-AHEAD LOG
#------------------------------------------------------------------------------
# - Settings -
wal_level = 'replica'
fsync = 'on'
# (turning this off can cause
# unrecoverable data corruption)
max_wal_size = '400MB'
#------------------------------------------------------------------------------
# REPLICATION
#------------------------------------------------------------------------------
# - Sending Servers -
# Set these on the master and on any standby that will send replication data.
max_wal_senders = '16'
wal_keep_size = '128MB'
hot_standby = 'on'
log_connections = 'false'
log_disconnections = 'false'
log_hostname = 'false'
#------------------------------------------------------------------------------
# CLIENT CONNECTION DEFAULTS
#------------------------------------------------------------------------------
# - Statement Behavior -
client_min_messages = 'error'
# debug5
# debug4
# debug3
# debug2
# debug1
# log
# notice
# warning
# error
# - Shared Library Preloading -
shared_preload_libraries = 'pgaudit'
#------------------------------------------------------------------------------
# CONFIG FILE INCLUDES
#------------------------------------------------------------------------------
# These options allow settings to be loaded from files other than the
# default postgresql.conf. Note that these are directives, not variable
# assignments, so they can usefully be given more than once.
include_dir = 'conf.d'
#include_dir = '...' # include files ending in '.conf' from
# a directory, e.g., 'conf.d'
#include_if_exists = '...' # include file only if it exists
#include = '...' # include file
#------------------------------------------------------------------------------
# CUSTOMIZED OPTIONS
#------------------------------------------------------------------------------
# Add settings for extensions here
pgaudit.log_catalog = 'off'
install: true
image:
tag: 13.14.0
metrics:
enabled: true
serviceMonitor:
enabled: true
redis:
install: true
metrics:
enabled: true
serviceMonitor:
enabled: true
certmanager:
install: false
prometheus:
install: false
rbac:
create: false
nginx-ingress:
enabled: false
gitlab:
toolbox:
extraEnvFrom:
AWS_ACCESS_KEY_ID:
secretKeyRef:
name: storage-config
key: accesskey
AWS_SECRET_ACCESS_KEY:
secretKeyRef:
name: storage-config
key: secretkey
extraEnv:
AWS_DEFAULT_REGION: us-east-1
AWS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
backups:
objectStorage:
config:
secret: storage-config
key: config
cron:
enabled: true
schedule: "@daily"
extraArgs: "--maximum-backups 14 --rsyncable --s3tool awscli --aws-s3-endpoint-url https://s3.gitlab.domain.example/"
backoffLimit: 1
failedJobsHistoryLimit: 3
successfulJobsHistoryLimit: 3
restartPolicy: "Never"
webservice:
metrics:
enabled: true
serviceMonitor:
enabled: true
gitlab-exporter:
metrics:
enabled: true
serviceMonitor:
enabled: true
gitaly:
metrics:
enabled: true
serviceMonitor:
enabled: true
kas:
metrics:
enabled: true
serviceMonitor:
enabled: true
# gitlab-rails-storage / rails.s3.yaml
provider: AWS
region: us-east-1
aws_access_key_id: ...
aws_secret_access_key: ...
enable_signature_v4_streaming: false
aws_signature_version: 4
endpoint: "https://s3.gitlab.domain.example:443"
path_style: true
Current behavior
Icons and avatars are not displayed continuously.
Expected behavior
All icons and avatars should be displayed as intended.
Versions
- Chart: 7.9.4
- Platform:
- Self-hosted: RKE2
- Kubernetes: (
kubectl version
)- Client: v1.26.10
- Server: v1.26.13+rke2r1
- Helm: (
helm version
)- Client: v3.13.2
- Server: v3.13.2
Relevant logs
gitlab-test-webservice-default-6d855c876f-frvkv webservice {"component": "gitlab","subcomponent":"production_json","method":"GET","path":"/uploads/-/system/user/avatar/6/avatar.png","format":"html","controller":"UploadsController","action":"show","status":200,"time":"2024-04-22T16:02:47.532Z","params":[{"key":"width","value":"64"},{"key":"model","value":"user"},{"key":"mounted_as","value":"avatar"},{"key":"id","value":"6"},{"key":"filename","value":"avatar.png"}],"correlation_id":"01HW38QAHCZQYH2FW02ZH6J2ZR","meta.caller_id":"UploadsController#show","meta.remote_ip":"....","meta.feature_category":"groups_and_projects","meta.user":"....","meta.user_id":6,"meta.client_id":"user/6","remote_ip":"...","user_id":6,"username":"....","ua":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","queue_duration_s":0.030544,"request_urgency":"default","target_duration_s":1,"redis_calls":6,"redis_allowed_cross_slot_calls":1,"redis_duration_s":0.016067,"redis_read_bytes":379,"redis_write_bytes":781,"redis_cache_calls":1,"redis_cache_duration_s":0.002399,"redis_cache_read_bytes":199,"redis_cache_write_bytes":76,"redis_sessions_calls":1,"redis_sessions_duration_s":0.002513,"redis_sessions_read_bytes":177,"redis_sessions_write_bytes":85,"redis_shared_state_calls":4,"redis_shared_state_allowed_cross_slot_calls":1,"redis_shared_state_duration_s":0.011155,"redis_shared_state_read_bytes":3,"redis_shared_state_write_bytes":620,"db_count":3,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":3,"db_main_count":3,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.013,"db_main_duration_s":0.013,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.032268,"mem_objects":14896,"mem_bytes":838064,"mem_mallocs":2629,"mem_total_bytes":1433904,"pid":42,"worker_id":"puma_0","rate_limiting_gates":[],"db_duration_s":0.00884,"view_duration_s":0.0,"duration_s":0.02091}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"correlation_id":"01HW38QAHCZQYH2FW02ZH6J2ZR","duration_s":0.062420468,"imageresizer.content_type":"image/png","imageresizer.original_filesize":75806,"imageresizer.status":"success","imageresizer.target_width":"64","level":"info","method":"GET","msg":"success","subsystem":"imageresizer","time":"2024-04-22T16:02:47Z","uri":"/uploads/-/system/user/avatar/6/avatar.png?width=64","written_bytes":8483}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"content_type":"image/png","correlation_id":"01HW38QAHCZQYH2FW02ZH6J2ZR","duration_ms":132,"host":"gitlab.domain.example","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"https://gitlab.domain.example/MyUser","remote_addr":"....:35506","remote_ip":"....","route":"","status":200,"system":"http","time":"2024-04-22T16:02:47Z","ttfb_ms":132,"uri":"/uploads/-/system/user/avatar/6/avatar.png?width=64","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","written_bytes":8483}
- gitlab-test-sidekiq-all-in-1-v2-788cc987b-p4klf › certificates
gitlab-test-webservice-default-6d855c876f-frvkv webservice {"component": "gitlab","subcomponent":"production_json","method":"GET","path":"/uploads/-/system/user/avatar/6/avatar.png","format":"html","controller":"UploadsController","action":"show","status":200,"time":"2024-04-22T16:02:47.874Z","params":[{"key":"model","value":"user"},{"key":"mounted_as","value":"avatar"},{"key":"id","value":"6"},{"key":"filename","value":"avatar.png"}],"correlation_id":"01HW38QAWA3G0QVYB4JYSS5E69","meta.caller_id":"UploadsController#show","meta.remote_ip":"....","meta.feature_category":"groups_and_projects","meta.user":"MyUser","meta.user_id":6,"meta.client_id":"user/6","remote_ip":"....","user_id":6,"username":"MyUser","ua":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","queue_duration_s":0.027487,"request_urgency":"default","target_duration_s":1,"redis_calls":5,"redis_allowed_cross_slot_calls":1,"redis_duration_s":0.012162000000000001,"redis_read_bytes":180,"redis_write_bytes":705,"redis_sessions_calls":1,"redis_sessions_duration_s":0.002399,"redis_sessions_read_bytes":177,"redis_sessions_write_bytes":85,"redis_shared_state_calls":4,"redis_shared_state_allowed_cross_slot_calls":1,"redis_shared_state_duration_s":0.009763,"redis_shared_state_read_bytes":3,"redis_shared_state_write_bytes":620,"db_count":3,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":3,"db_main_count":3,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.012,"db_main_duration_s":0.012,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.030334,"mem_objects":14420,"mem_bytes":782464,"mem_mallocs":2528,"mem_total_bytes":1359264,"pid":44,"worker_id":"puma_1","rate_limiting_gates":[],"db_duration_s":0.00676,"view_duration_s":0.0,"duration_s":0.01615}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"correlation_id":"01HW38QAWA3G0QVYB4JYSS5E69","level":"info","msg":"SendURL: sending","path":"/uploads/-/system/user/avatar/6/avatar.png","time":"2024-04-22T16:02:47Z","url":"https://s3.gitlab.domain.example/gitlab-test-uploads/user/avatar/6/avatar.png?response-content-disposition=inline%3B%20filename%3D%22avatar.png%22%3B%20filename%2A%3DUTF-8%27%27avatar.png\u0026response-content-type=image%2Fpng\u0026X-Amz-Expires=600\u0026X-Amz-Date=20240422T160247Z\u0026X-Amz-Algorithm=AWS4-HMAC-SHA256\u0026X-Amz-Credential=yourSuperCoolCred%2F20240422%2Fus-east-1%2Fs3%2Faws4_request\u0026X-Amz-SignedHeaders=host\u0026X-Amz-Signature=[FILTERED]"}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"content_type":"text/plain; charset=utf-8","correlation_id":"01HW38QAWA3G0QVYB4JYSS5E69","duration_ms":117,"host":"gitlab.domain.example","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"https://gitlab.domain.example/MyUser","remote_addr":"....:35506","remote_ip":"....","route":"","status":501,"system":"http","time":"2024-04-22T16:02:47Z","ttfb_ms":117,"uri":"/uploads/-/system/user/avatar/6/avatar.png","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","written_bytes":176}
gitlab-test-webservice-default-6d855c876f-frvkv webservice {"component": "gitlab","subcomponent":"production_json","method":"GET","path":"/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png","format":"html","controller":"UploadsController","action":"show","status":200,"time":"2024-04-22T16:07:31.711Z","params":[{"key":"width","value":"400"},{"key":"model","value":"project"},{"key":"mounted_as","value":"avatar"},{"key":"id","value":"13"},{"key":"filename","value":"gitlab-icon-rgb.png"}],"correlation_id":"01HW39001CD19PETJRD5K2F0A5","meta.caller_id":"UploadsController#show","meta.remote_ip":"....","meta.feature_category":"groups_and_projects","meta.user":"MyUser","meta.user_id":6,"meta.client_id":"user/6","remote_ip":"....","user_id":6,"username":"MyUser","ua":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","queue_duration_s":0.031905,"request_urgency":"default","target_duration_s":1,"redis_calls":5,"redis_allowed_cross_slot_calls":1,"redis_duration_s":0.012819,"redis_read_bytes":180,"redis_write_bytes":704,"redis_sessions_calls":1,"redis_sessions_duration_s":0.003445,"redis_sessions_read_bytes":177,"redis_sessions_write_bytes":85,"redis_shared_state_calls":4,"redis_shared_state_allowed_cross_slot_calls":1,"redis_shared_state_duration_s":0.009374,"redis_shared_state_read_bytes":3,"redis_shared_state_write_bytes":619,"db_count":6,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":6,"db_main_count":6,"db_ci_count":0,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.026,"db_main_duration_s":0.026,"db_ci_duration_s":0.0,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.043464,"mem_objects":17816,"mem_bytes":1666136,"mem_mallocs":3655,"mem_total_bytes":2378776,"pid":44,"worker_id":"puma_1","rate_limiting_gates":[],"db_duration_s":0.01792,"view_duration_s":0.0,"duration_s":0.03798}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"correlation_id":"01HW39001CD19PETJRD5K2F0A5","level":"info","msg":"SendURL: sending","path":"/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png","time":"2024-04-22T16:07:31Z","url":"https://s3.gitlab.domain.example/gitlab-test-uploads/project/avatar/13/gitlab-icon-rgb.png?response-content-disposition=inline%3B%20filename%3D%22gitlab-icon-rgb.png%22%3B%20filename%2A%3DUTF-8%27%27gitlab-icon-rgb.png\u0026response-content-type=image%2Fpng\u0026X-Amz-Expires=600\u0026X-Amz-Date=20240422T160731Z\u0026X-Amz-Algorithm=AWS4-HMAC-SHA256\u0026X-Amz-Credential=yourSuperCoolCred%2F20240422%2Fus-east-1%2Fs3%2Faws4_request\u0026X-Amz-SignedHeaders=host\u0026X-Amz-Signature=[FILTERED]"}
gitlab-test-webservice-default-6d855c876f-frvkv gitlab-workhorse {"content_type":"text/plain; charset=utf-8","correlation_id":"01HW39001CD19PETJRD5K2F0A5","duration_ms":761,"host":"s3.gitlab.domain.example","level":"info","method":"GET","msg":"access","proto":"HTTP/1.1","referrer":"","remote_addr":".....:55688","remote_ip":"....","route":"","status":501,"system":"http","time":"2024-04-22T16:07:32Z","ttfb_ms":761,"uri":"/uploads/-/system/project/avatar/13/gitlab-icon-rgb.png?width=400","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36","written_bytes":176}