Skip to content

How to configure correctly external Prometheus to use GitLab Omnibus Grafana dashboard

Everyone can contribute. Help move this issue forward while earning points, leveling up and collecting rewards.

Hello,

I write this issue because I am trying to configure an exernal Prometheus server to monitor all my Docker GitLab instance (CE and EE).

I have disable the embeded GitLab Prometheus in my docker-compose file and setup exporter as mentionned in this documentation (https://docs.gitlab.com/ee/administration/monitoring/prometheus/#using-an-external-prometheus-server) like this :

version: '3'
services:
  web:
    image: 'gitlab/gitlab-ce:latest'
    restart: always
    hostname: 'mycustomurl'
    environment:
      TZ: Europe/Paris
      GITLAB_OMNIBUS_CONFIG: |
        external_url 'https://mycustomurl'
        gitlab_rails['monitoring_whitelist'] = ['127.0.0.1', 'PROMETHEUS_IP']
        prometheus['listen_address'] = '0.0.0.0:9090'
        prometheus['enable'] = false
        gitlab_monitor['listen_address'] = '0.0.0.0'
        gitlab_monitor['listen_port'] = '9168'
        gitaly['prometheus_listen_addr'] = "0.0.0.0:9236"
        node_exporter['listen_address'] = '0.0.0.0:9100'
        redis_exporter['listen_address'] = '0.0.0.0:9121'
        postgres_exporter['listen_address'] = '0.0.0.0:9187'
        #gitlab_rails['gitlab_default_projects_features_builds'] = false
        gitlab_rails['gitlab_default_projects_features_container_registry'] = false
        #gitlab_rails['lfs_enabled'] = false
        gitlab_rails['lfs_storage_path'] = "/var/opt/git-lfs"
        # limit backup lifetime to 2 days - 172800 seconds
        gitlab_rails['backup_keep_time'] = 172800
        gitlab_rails['gitlab_shell_ssh_port'] = 2224
        gitlab_rails['smtp_enable'] = true;
        gitlab_rails['smtp_address'] = '172.17.0.1';
        gitlab_rails['smtp_port'] = 25;
        gitlab_rails['smtp_domain'] = '172.17.0.1';
        gitlab_rails['smtp_tls'] = false;
        gitlab_rails['smtp_openssl_verify_mode'] = 'none'
        gitlab_rails['smtp_enable_starttls_auto'] = false
        gitlab_rails['smtp_ssl'] = false
        gitlab_rails['smtp_force_ssl'] = false
        gitlab_rails['omniauth_enabled'] = false
    ports:
      - '80:80'
      - '443:443'
      - '2224:22'
      - '9168:9168'
      - '9236:9236'
      - '9100:9100'
      - '9121:9121'
      - '9187:9187'
      - '8060:8060'
      - '9229:9229'
      - '8080:8080'
      - '8082:8082'
    volumes:
      - '/srv/git_nfs/gitlab/config:/etc/gitlab'
      - '/srv/git_nfs/gitlab/logs:/var/log/gitlab'
      - '/srv/git_nfs/gitlab/data:/var/opt/gitlab'
      - '/srv/git_nfs/gitlab/secret:/secret/gitlab/backups'
      - '/tmp/lfs-objects:/var/opt/git-lfs'

image image

Here is my prometheus.yml config file (it's a copy of the GitLab one) :

---
global:
  scrape_interval: 15s
  scrape_timeout: 15s
remote_read: []
remote_write: []
scrape_configs:
- job_name: prometheus
  static_configs:
  - targets:
    - 0.0.0.0:9090
- job_name: nginx
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:8060
- job_name: redis
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9121
- job_name: postgres
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9187
- job_name: node
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9100
- job_name: gitlab-workhorse
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9229
- job_name: gitlab-unicorn
  metrics_path: "/-/metrics"
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:8080
- job_name: gitlab-sidekiq
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:8082
- job_name: gitlab_monitor_database
  metrics_path: "/database"
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9168
- job_name: gitlab_monitor_sidekiq
  metrics_path: "/sidekiq"
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9168
- job_name: gitlab_monitor_process
  metrics_path: "/process"
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9168
- job_name: gitaly
  static_configs:
  - targets:
    - IP_OF_DOCKER_HOST_FOR_GITLAB:9236

Now in my Grafana, when I use the official GitLab Omnibus dashboard, all graph are not working : image image image

Anyone can help me with that ? And how can target multiple GitLab instance in the prometheus.yml file ?

Thank you very much for your help.

Best regards

Edited by 🤖 GitLab Bot 🤖