Grafana has failed to load its application files
Summary
I'm running Gitlab Omnibus.
Trying to open https://gitlab.corp.com/-/grafana shows error message
If you're seeing this Grafana has failed to load its application files
1. This could be caused by your reverse proxy settings.
2. If you host grafana under subpath make sure your grafana.ini root_url setting includes subpath
3. If you have a local dev build make sure you build frontend using: yarn start, yarn start:hot, or yarn build
4. Sometimes restarting grafana-server can helpI've tried restarting Grafana using sudo gitlab-ctl restart grafana.
I've tried restarting Gitlab using sudo gitlab-ctl restart
What is the current bug behavior?
Grafana shows error message.
What is the expected correct behavior?
I expected to see Grafana dashboard
Relevant logs
Relevant logs
Grafana:2019-11-18_11:52:58.86528 t=2019-11-18T14:52:58+0300 lvl=info msg="Starting Grafana" logger=server version=6.3.5 commit=67bad72 branch=HEAD compiled=2019-09-02T12:34:49+0300 2019-11-18_11:52:58.86539 t=2019-11-18T14:52:58+0300 lvl=info msg="Config loaded from" logger=settings file=/var/opt/gitlab/grafana/conf/defaults.ini 2019-11-18_11:52:58.86541 t=2019-11-18T14:52:58+0300 lvl=info msg="Config loaded from" logger=settings file=/var/opt/gitlab/grafana/grafana.ini 2019-11-18_11:52:58.86542 t=2019-11-18T14:52:58+0300 lvl=info msg="Path Home" logger=settings path=/var/opt/gitlab/grafana 2019-11-18_11:52:58.86544 t=2019-11-18T14:52:58+0300 lvl=info msg="Path Data" logger=settings path=/var/opt/gitlab/grafana/data 2019-11-18_11:52:58.86545 t=2019-11-18T14:52:58+0300 lvl=info msg="Path Logs" logger=settings path=/var/log/gitlab/grafana 2019-11-18_11:52:58.86547 t=2019-11-18T14:52:58+0300 lvl=info msg="Path Plugins" logger=settings path=/var/opt/gitlab/grafana/data/plugins 2019-11-18_11:52:58.86548 t=2019-11-18T14:52:58+0300 lvl=info msg="Path Provisioning" logger=settings path=/var/opt/gitlab/grafana/provisioning 2019-11-18_11:52:58.86556 t=2019-11-18T14:52:58+0300 lvl=info msg="App mode production" logger=settings 2019-11-18_11:52:58.86728 t=2019-11-18T14:52:58+0300 lvl=info msg="Initializing SqlStore" logger=server 2019-11-18_11:52:58.86739 t=2019-11-18T14:52:58+0300 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2019-11-18_11:52:58.86988 t=2019-11-18T14:52:58+0300 lvl=info msg="Starting DB migration" logger=migrator 2019-11-18_11:52:58.88900 t=2019-11-18T14:52:58+0300 lvl=info msg="Initializing HTTPServer" logger=server 2019-11-18_11:52:58.89051 t=2019-11-18T14:52:58+0300 lvl=info msg="Initializing InternalMetricsService" logger=server 2019-11-18_11:52:58.89054 t=2019-11-18T14:52:58+0300 lvl=info msg="Initializing RemoteCache" logger=server 2019-11-18_11:52:58.89062 t=2019-11-18T14:52:58+0300 lvl=info msg="Initializing QuotaService" logger=server 2019-11-18_11:52:58.89064 t=2019-11-18T14:52:58+0300 lvl=info msg="Initializing PluginManager" logger=server 2019-11-18_11:52:58.89071 t=2019-11-18T14:52:58+0300 lvl=info msg="Starting plugin search" logger=plugins 2019-11-18_11:52:59.03319 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing RenderingService" logger=server 2019-11-18_11:52:59.03403 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing DatasourceCacheService" logger=server 2019-11-18_11:52:59.03454 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing HooksService" logger=server 2019-11-18_11:52:59.03459 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing LoginService" logger=server 2019-11-18_11:52:59.03460 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing SearchService" logger=server 2019-11-18_11:52:59.03462 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing ServerLockService" logger=server 2019-11-18_11:52:59.03463 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing TracingService" logger=server 2019-11-18_11:52:59.03470 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing UsageStatsService" logger=server 2019-11-18_11:52:59.03497 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing UserAuthTokenService" logger=server 2019-11-18_11:52:59.03500 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing CleanUpService" logger=server 2019-11-18_11:52:59.03501 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing NotificationService" logger=server 2019-11-18_11:52:59.04037 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing provisioningServiceImpl" logger=server 2019-11-18_11:52:59.23588 t=2019-11-18T14:52:59+0300 lvl=info msg="Initializing Stream Manager" 2019-11-18_11:52:59.25828 t=2019-11-18T14:52:59+0300 lvl=info msg="HTTP Server Listen" logger=http.server address=localhost:3000 protocol=http subUrl=/-/grafana socket= 2019-11-18_11:55:26.69261 t=2019-11-18T14:55:26+0300 lvl=info msg="Request Completed" logger=context userId=0 orgId=0 uname= method=GET path=/-/grafana/ status=404 remote_addr=127.0.0.1 time_ms=9 size=23318 referer= 2019-11-18_11:55:26.72153 t=2019-11-18T14:55:26+0300 lvl=info msg="Request Completed" logger=context userId=0 orgId=0 uname= method=GET path=/-/grafana/public/build/grafana.dark.75ccb77dbb4b5e3c9285.css status=404 remote_addr=127.0.0.1 time_ms=6 size=23318 referer=https://example.corp.com/-/grafana/ 2019-11-18_11:55:26.77370 t=2019-11-18T14:55:26+0300 lvl=info msg="Request Completed" logger=context userId=0 orgId=0 uname= method=GET path=/-/grafana/public/build/app.75ccb77dbb4b5e3c9285.js status=404 remote_addr=127.0.0.1 time_ms=15 size=23318 referer=https://example.corp.com/-/grafana/ 2019-11-18_11:55:26.77693 t=2019-11-18T14:55:26+0300 lvl=info msg="Request Completed" logger=context userId=0 orgId=0 uname= method=GET path=/-/grafana/public/build/vendor.75ccb77dbb4b5e3c9285.js status=404 remote_addr=127.0.0.1 time_ms=18 size=23318 referer=https://example.corp.com/-/grafana/ 2019-11-18_11:55:26.82094 t=2019-11-18T14:55:26+0300 lvl=info msg="Request Completed" logger=context userId=0 orgId=0 uname= method=GET path=/-/grafana/public/build/app.75ccb77dbb4b5e3c9285.js status=404 remote_addr=127.0.0.1 time_ms=3 size=23318 referer=https://example.corp.com/-/grafana/
Nginx
xxx.xxx.xxx.xxx - - [18/Nov/2019:14:55:26 +0300] "GET /-/grafana/ HTTP/1.1" 404 23354 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36 OPR/65.0.3467.38" xxx.xxx.xxx.xxx - - [18/Nov/2019:14:55:26 +0300] "GET /-/grafana/public/build/grafana.dark.75ccb77dbb4b5e3c9285.css HTTP/1.1" 404 23354 "https://example.corp.com/-/grafana/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36 OPR/65.0.3467.38" xxx.xxx.xxx.xxx - - [18/Nov/2019:14:55:26 +0300] "GET /-/grafana/public/build/app.75ccb77dbb4b5e3c9285.js HTTP/1.1" 404 23354 "https://example.corp.com/-/grafana/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36 OPR/65.0.3467.38" xxx.xxx.xxx.xxx - - [18/Nov/2019:14:55:26 +0300] "GET /-/grafana/public/build/vendor.75ccb77dbb4b5e3c9285.js HTTP/1.1" 404 23354 "https://example.corp.com/-/grafana/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36 OPR/65.0.3467.38" xxx.xxx.xxx.xxx - - [18/Nov/2019:14:55:26 +0300] "GET /-/grafana/public/build/app.75ccb77dbb4b5e3c9285.js HTTP/1.1" 404 23354 "https://example.corp.com/-/grafana/" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36 OPR/65.0.3467.38" xxx.xxx.xxx.xxx - - [18/Nov/2019:14:55:28 +0300] "GET / HTTP/1.1" 200 82991 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36 OPR/65.0.3467.38"
Details of package version
Provide the package version installation details
Desired=Unknown/Install/Remove/Purge/Hold | Status=Not/Inst/Conf-files/Unpacked/halF-conf/Half-inst/trig-aWait/Trig-pend |/ Err?=(none)/Reinst-required (Status,Err: uppercase=bad) ||/ Name Version Architecture Description +++-====================================-=======================-=======================-============================================================================== ii gitlab-ce 12.4.2-ce.0 amd64 GitLab Community Edition (including NGINX, Postgres, Redis) un gitlab-ee (no description available)
Environment details
- Operating System:
Ubuntu 16.04 - Installation Target, remove incorrect values:
- VMware virtual machine
- Installation Type, remove incorrect values:
- Upgrade from version
12.4 - Other:
REPLACE-WITH-DETAILS
- Upgrade from version
- Is there any other software running on the machine:
Jenkins,nginx - Is this a single or multiple node installation? - Single-node installation
- Resources
- CPU:
8 cores - Memory total:
12Gb
- CPU:
Configuration details
Provide the relevant sections of `/etc/gitlab/gitlab.rb`
external_url 'https://example.corp.com'
prometheus['enable'] = false
prometheus['listen_address'] = ':9090'
grafana['enable'] = true
grafana['home'] = '/var/opt/gitlab/grafana'
grafana['env_directory'] = '/opt/gitlab/etc/grafana/env'
gitlab_rails['time_zone'] = 'Europe/Moscow'
gitlab_rails['gitlab_email_enabled'] = true
gitlab_rails['gitlab_email_from'] = 'git@example.corp.com'
gitlab_rails['gitlab_email_display_name'] = 'Algorithmic Lab gitlab server'
gitlab_rails['gitlab_default_projects_features_builds'] = false
gitlab_rails['trusted_proxies'] = ['106.109.129.127']
gitlab_rails['lfs_enabled'] = true
gitlab_rails['lfs_storage_path'] = "/data/lfs-objects/lfs-objects"
gitlab_rails['lfs_object_store_enabled'] = true
gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects"
gitlab_rails['lfs_object_store_connection'] = {
'provider' => 'AWS',
'region' => 'eu-central-1',
'aws_access_key_id' => 'LTGK6LGSDKOBOLZ8ECZK',
'aws_secret_access_key' => 'HzoScQ9VoO4lnhIl3hTFBi5cRMsXiLwLvlzKnFv6',
# Options below configure an S3 compatible host instead of AWS
'host' => 'robotics-storage.rnd.samsung.ru',
'endpoint' => 'https://106.109.128.252:9000',
'path_style' => true
}
gitlab_rails['lfs_object_store_background_upload'] = true
gitlab_rails['pages_path'] = "/mnt/storage/pages"
gitlab_rails['ldap_enabled'] = true
#method: "tls" or "ssl" or "plain"
gitlab_rails['ldap_servers'] = YAML.load <<-'EOS' # remember to close this block with 'EOS' below
main: # 'main' is the GitLab 'provider ID' of this LDAP server
label: 'LDAP'
host: 'dc.corp.net'
port: 389
uid: 'sAMAccountName'
method: 'plain'
bind_dn: 'CN=ldapbind'
password: ''
active_directory: true
allow_username_or_email_login: false
block_auto_created_users: false
user_filter: ''
attributes:
username: ['sAMAccountName']
email: ['mail', 'userPrincipalName']
name: ['EnCn', 'cn']
first_name: ['enGivenName', 'givenName']
last_name: ['EnSn']
EOS
gitlab_rails['omniauth_enabled'] = false
gitlab_rails['backup_path'] = "/mnt/gitlab-backups"
gitlab_rails['backup_keep_time'] = 86400 # seconds, default 604800 = 7 days
gitlab_rails['backup_upload_connection'] = {
:provider => 'Local',
:local_root => '/data'
}
gitlab_rails['backup_upload_remote_directory'] = 'gitlab-backups'
git_data_dirs({"default"=> {"path" => "/mnt/gitlab"} })
gitlab_rails['smtp_enable'] = true
gitlab_rails['smtp_address'] = ""
gitlab_rails['smtp_port'] = 25
gitlab_rails['smtp_enable_starttls_auto'] = false
gitlab_rails['smtp_tls'] = false
web_server['external_users'] = ['www-data']
nginx['enable'] = false
nginx['redirect_http_to_https'] = true
nginx['redirect_http_to_https_port'] = 80
nginx['ssl_client_certificate'] = "/etc/gitlab/ssl/crt.crt" # Most root CA's are included by default
nginx['ssl_certificate'] = "/etc/gitlab/ssl/example.corp.com.crt"
nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/example.corp.com.key"
nginx['custom_gitlab_server_config'] = "include /etc/nginx/conf.d/locations.conf;"
manage_accounts['enable'] = false
NGINX location
location /-/grafana {
root /opt/gitlab/embedded/service;
proxy_pass http://localhost:3000;
}
Edited by Vladimir Eremeev