Wiki cannot attach file error 500
Hi I have an issue with a self-managed docker gitlab instance : when I upload an image on wiki I have an internal error 500 with no more details.
If I look at the logs I can see ( is the ip of the reverse proxy, is the client ip) :
gitlab_1 | ==> /var/log/gitlab/gitlab-rails/production.log <==
gitlab_1 | Started POST "/api/v4/internal/workhorse/authorize_upload" for <proxy-ip> at 2023-01-09 14:08:33 +0000
gitlab_1 | Processing by Gitlab::RequestForgeryProtection::Controller#index as HTML
gitlab_1 | Can't verify CSRF token authenticity.
gitlab_1 | Completed 422 Unprocessable Entity in 1ms (ActiveRecord: 0.0ms | Elasticsearch: 0.0ms | Allocations: 202)
gitlab_1 | Processing by Gitlab::RequestForgeryProtection::Controller#index as HTML
gitlab_1 | Can't verify CSRF token authenticity.
gitlab_1 | Completed 422 Unprocessable Entity in 0ms (ActiveRecord: 0.0ms | Elasticsearch: 0.0ms | Allocations: 250)
gitlab_1 |
gitlab_1 | ==> /var/log/gitlab/gitlab-rails/api_json.log <==
gitlab_1 | {"time":"2023-01-09T14:08:33.786Z","severity":"INFO","duration_s":0.00251,"db_duration_s":0.0,"view_duration_s":0.00251,"status":401,"method":"POST","path":"/api/v4/internal/workhorse/authorize_upload
","params":[],"host":"gitlab.FQDN","remote_ip":"<my-ip>, <proxy-ip>","ua":"Mozilla/5.0 (X11; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0","route":"/api/:version/interna
l/workhorse/authorize_upload","db_count":1,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":1,"db_main_count":1,"db_main_replica_count":0,"db_replica_cached_count":0,"db_primary_cach
ed_count":0,"db_main_cached_count":0,"db_main_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_main_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.008,"db_main_duration_s":0.008,"db_main_replica_duration_s":0.0,"cpu_s":0.034177,"mem_objects":12125,"mem_bytes":1017246,"mem_mallocs":3048,"mem_total_bytes":1502246,"pid":1226131,"worker_id":"puma_7","rate_limiting_gates":[],"correlation_id":"01GPBDS2EDDY1W25ATGRMGR05E","meta.caller_id":"POST /api/:version/internal/workhorse/authorize_upload","meta.remote_ip":"<proxy-ip>","meta.feature_category":"not_owned","meta.client_id":"ip/<proxy-ip>","content_length":"0","request_urgency":"default
","target_duration_s":1}
gitlab_1 |
gitlab_1 | ==> /var/log/gitlab/gitlab-workhorse/current <==
gitlab_1 | {"correlation_id":"01GPBDS2EDDY1W25ATGRMGR05E","error":"handleFileUploads: extract files from multipart: no api response: status 401","level":"error","method":"POST","msg":"","time":"2023-01-09T14:08:
33Z","uri":"/api/v4/projects/27/wikis/attachments"}
gitlab_1 | {"content_type":"text/plain; charset=utf-8","correlation_id":"01GPBDS2EDDY1W25ATGRMGR05E","duration_ms":47,"host":"gitlab.FQDN","level":"info","method":"POST","msg":"acces
s","proto":"HTTP/1.1","referrer":"","remote_addr":"<my-ip>","remote_ip":"<my-ip>","route":"^/api/v4/projects/[^/]+/wikis/attachments\\z","status":500,"system":"http","time":"2023-01-09T14:08:33Z","t
tfb_ms":46,"uri":"/api/v4/projects/27/wikis/attachments","user_agent":"Mozilla/5.0 (X11; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0","written_bytes":22}
gitlab_1 |
gitlab_1 | ==> /var/log/gitlab/nginx/gitlab_access.log <==
gitlab_1 | <proxy-ip> - - [09/Jan/2023:14:08:33 +0000] "POST /api/v4/projects/27/wikis/attachments HTTP/1.1" 500 22 "" "Mozilla/5.0 (X11; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0" -
It looks like gitlab-foss#52243 (closed) but the request is send in https as we can see below, also I have the same issue if I forge the request with curl with a command like (where the CSRF token is retrieved manually from the page) :
curl --request POST --header 'X-CSRF-Token: <token>' -F 'data=@/path/to/image.png' https://FQDN/api/v4/projects/27/wikis/attachments
Configuration details
+ Gitlab 15.7.2 ce docker image behind a traefik reverse proxy
docker-compose.yml
version: '3'
services:
gitlab:
image: 'gitlab/gitlab-ce:15.7.2-ce.0'
restart: always
hostname: 'FQDN'
networks:
- ldap
- traefik
environment:
TZ:
GITLAB_OMNIBUS_CONFIG: |
gitlab_rails['gitlab_shell_ssh_port'] = 2222
external_url = 'https://FQDN'
nginx['listen_port'] = 80
nginx['listen_https'] = false
nginx['proxy_set_headers'] = {
"X-Forwarded-Proto" => "https",
"X-Forwarded-Ssl" => "on"
}
nginx['redirect_http_to_https'] = true
nginx['redirect_http_to_https_port'] = 80
gitlab_rails['allowed_hosts'] = ['FQDN', 'localhost', '127.0.0.1']
# Each address is added to the the NGINX config as 'set_real_ip_from <address>;'
nginx['real_ip_trusted_addresses'] = [ '<proxy-ip>' ]
# other real_ip config options
nginx['real_ip_header'] = 'X-Forwarded-For'
nginx['real_ip_recursive'] = 'on'
letsencrypt['enable'] = false
# Limit backup lifetime to 3 days - 259200 seconds
gitlab_rails['backup_keep_time'] = 259200
gitlab_rails['rack_attack_git_basic_auth'] = {
'enabled' => true,
'ip_whitelist' => ["127.0.0.1", '<proxy-ip>'],
'maxretry' => 10, # Limit the number of Git HTTP authentication attempts per IP
'findtime' => 60, # Reset the auth attempt counter per IP after 60 seconds
'bantime' => 3600 # Ban an IP for one hour (3600s) after too many auth attempts
}
# Mail
gitlab_rails['smtp_enable'] = true
gitlab_rails['smtp_address'] = "providerFQDN"
gitlab_rails['smtp_port'] = 465
gitlab_rails['smtp_user_name'] = "<addr>"
gitlab_rails['smtp_password'] = "<password>"
gitlab_rails['smtp_domain'] = "providerFQDN"
gitlab_rails['smtp_authentication'] = "login"
gitlab_rails['smtp_enable_starttls_auto'] = true
gitlab_rails['smtp_tls'] = true
gitlab_rails['smtp_openssl_verify_mode'] = 'peer' # If your SMTP server does not like the default 'From: gitlab@localhost' you
# # can change the 'From' with this setting.
gitlab_rails['gitlab_email_from'] = 'services@FQDN'
gitlab_rails['incoming_email_enabled'] = true
# The email address including the `%{key}` placeholder that will be replaced to reference the item being replied to.
# The placeholder can be omitted but if present, it must appear in the "user" part of the address (before the `@`).
gitlab_rails['incoming_email_address'] = "services+%{key}@FQDN"
# Email account username
# With third party providers, this is usually the full email address.
# With self-hosted email servers, this is usually the user part of the email address.
gitlab_rails['incoming_email_email'] = "<adress>"
# Email account password
gitlab_rails['incoming_email_password'] = "<password>"
# IMAP server host
gitlab_rails['incoming_email_host'] = "smtproviderFQDN"
# IMAP server port
gitlab_rails['incoming_email_port'] = 993
# Whether the IMAP server uses SSL
gitlab_rails['incoming_email_ssl'] = true
# Whether the IMAP server uses StartTLS
gitlab_rails['incoming_email_start_tls'] = false
# The mailbox where incoming mail will end up. Usually "inbox".
gitlab_rails['incoming_email_mailbox_name'] = "inbox"
# The IDLE command timeout.
gitlab_rails['incoming_email_idle_timeout'] = 60
# Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery
gitlab_rails['incoming_email_expunge_deleted'] = true
# Add any other gitlab.rb configuration here, each on its own line
gitlab_rails['ldap_enabled'] = true
gitlab_rails['ldap_servers'] = YAML.load <<-'EOS'
main: # 'main' is the GitLab 'provider ID' of this LDAP server
label: 'LDAP'
host: 'ldap'
port: 389
uid: 'uid'
bind_dn: '<dn>'
password: '<password>'
encryption: 'plain' # "start_tls" or "simple_tls" or "plain"
verify_certificates: false
active_directory: false
allow_username_or_email_login: false
lowercase_usernames: false
block_auto_created_users: false
base: '<somedn>'
user_filter: '<somefilter>'
timeout: 10
attributes:
username: ['uid']
email: ['mail']
name: 'cn'
first_name: 'givenName'
last_name: 'sn'
EOS
# Add any other gitlab.rb configuration here, each on its own line
ports:
- '2222:22'
volumes:
- 'gitlab_config:/etc/gitlab'
- 'gitlab_logs:/var/log/gitlab'
- 'gitlab_data:/var/opt/gitlab'
shm_size: '256m'
labels:
- "traefik.enable=true"
- "traefik.docker.network=traefik"
- "traefik.http.routers.gitlab.rule=Host(`${HOST}`)"
- "traefik.http.routers.gitlab.tls.certresolver=myresolver"
- "traefik.http.routers.gitlab.entrypoints=web,websecure"
- "traefik.http.services.gitlab.loadbalancer.server.port=80"
- "traefik.http.routers.gitlab.middlewares=hardening@docker"
volumes:
gitlab_config:
gitlab_logs:
gitlab_data:
networks:
ldap:
external: true
traefik:
external: true