How to configure correctly external Prometheus to use GitLab Omnibus Grafana dashboard
Hello,
I write this issue because I am trying to configure an exernal Prometheus server to monitor all my Docker GitLab instance (CE and EE).
I have disable the embeded GitLab Prometheus in my docker-compose file and setup exporter as mentionned in this documentation (https://docs.gitlab.com/ee/administration/monitoring/prometheus/#using-an-external-prometheus-server) like this :
version: '3'
services:
web:
image: 'gitlab/gitlab-ce:latest'
restart: always
hostname: 'mycustomurl'
environment:
TZ: Europe/Paris
GITLAB_OMNIBUS_CONFIG: |
external_url 'https://mycustomurl'
gitlab_rails['monitoring_whitelist'] = ['127.0.0.1', 'PROMETHEUS_IP']
prometheus['listen_address'] = '0.0.0.0:9090'
prometheus['enable'] = false
gitlab_monitor['listen_address'] = '0.0.0.0'
gitlab_monitor['listen_port'] = '9168'
gitaly['prometheus_listen_addr'] = "0.0.0.0:9236"
node_exporter['listen_address'] = '0.0.0.0:9100'
redis_exporter['listen_address'] = '0.0.0.0:9121'
postgres_exporter['listen_address'] = '0.0.0.0:9187'
#gitlab_rails['gitlab_default_projects_features_builds'] = false
gitlab_rails['gitlab_default_projects_features_container_registry'] = false
#gitlab_rails['lfs_enabled'] = false
gitlab_rails['lfs_storage_path'] = "/var/opt/git-lfs"
# limit backup lifetime to 2 days - 172800 seconds
gitlab_rails['backup_keep_time'] = 172800
gitlab_rails['gitlab_shell_ssh_port'] = 2224
gitlab_rails['smtp_enable'] = true;
gitlab_rails['smtp_address'] = '172.17.0.1';
gitlab_rails['smtp_port'] = 25;
gitlab_rails['smtp_domain'] = '172.17.0.1';
gitlab_rails['smtp_tls'] = false;
gitlab_rails['smtp_openssl_verify_mode'] = 'none'
gitlab_rails['smtp_enable_starttls_auto'] = false
gitlab_rails['smtp_ssl'] = false
gitlab_rails['smtp_force_ssl'] = false
gitlab_rails['omniauth_enabled'] = false
ports:
- '80:80'
- '443:443'
- '2224:22'
- '9168:9168'
- '9236:9236'
- '9100:9100'
- '9121:9121'
- '9187:9187'
- '8060:8060'
- '9229:9229'
- '8080:8080'
- '8082:8082'
volumes:
- '/srv/git_nfs/gitlab/config:/etc/gitlab'
- '/srv/git_nfs/gitlab/logs:/var/log/gitlab'
- '/srv/git_nfs/gitlab/data:/var/opt/gitlab'
- '/srv/git_nfs/gitlab/secret:/secret/gitlab/backups'
- '/tmp/lfs-objects:/var/opt/git-lfs'
Here is my prometheus.yml config file (it's a copy of the GitLab one) :
---
global:
scrape_interval: 15s
scrape_timeout: 15s
remote_read: []
remote_write: []
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- 0.0.0.0:9090
- job_name: nginx
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:8060
- job_name: redis
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9121
- job_name: postgres
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9187
- job_name: node
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9100
- job_name: gitlab-workhorse
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9229
- job_name: gitlab-unicorn
metrics_path: "/-/metrics"
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:8080
- job_name: gitlab-sidekiq
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:8082
- job_name: gitlab_monitor_database
metrics_path: "/database"
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9168
- job_name: gitlab_monitor_sidekiq
metrics_path: "/sidekiq"
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9168
- job_name: gitlab_monitor_process
metrics_path: "/process"
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9168
- job_name: gitaly
static_configs:
- targets:
- IP_OF_DOCKER_HOST_FOR_GITLAB:9236
Now in my Grafana, when I use the official GitLab Omnibus dashboard, all graph are not working :
Anyone can help me with that ? And how can target multiple GitLab instance in the prometheus.yml file ?
Thank you very much for your help.
Best regards