diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 346583a6f3027df740e5305660729a3c47afe1cf..543f5b3c11949fa6bcc62a84fce04daf1c0a940d 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -143,7 +143,13 @@ Gitaly and GitLab use two shared secrets for authentication: To configure the _Gitaly token_, edit `/etc/gitlab/gitlab.rb`: ```ruby - gitaly['auth_token'] = 'abc123secret' + gitaly['configuration'] = { + # ... + auth: { + # ... + token: 'abc123secret', + }, + } ``` Configure the _GitLab Shell token_ in one of two ways. @@ -230,14 +236,21 @@ Updates to example must be made at: # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from Gitaly client to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Authentication token to ensure only authorized servers can communicate with - # Gitaly server - gitaly['auth_token'] = 'AUTH_TOKEN' + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + auth: { + # ... + # + # Authentication token to ensure only authorized servers can communicate with + # Gitaly server + token: 'AUTH_TOKEN', + }, + } ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective Gitaly server: @@ -247,24 +260,33 @@ Updates to example must be made at: On `gitaly1.internal`: ```ruby - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'default', + path: '/var/opt/gitlab/git-data', + }, + { + name: 'storage1', + path: '/mnt/gitlab/git-data', + }, + ], + } ``` On `gitaly2.internal`: ```ruby - git_data_dirs({ - 'storage2' => { - 'path' => '/srv/gitlab/git-data' - }, - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'storage2', + path: '/srv/gitlab/git-data', + }, + ], + } ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -461,17 +483,28 @@ example: git_data_dirs({ 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, # Address of the GitLab server that also has Gitaly running on it - 'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075', 'path' => '/mnt/gitlab/git-data' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075' }, 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, }) -# Make Gitaly accept connections on all network interfaces -gitaly['listen_addr'] = "0.0.0.0:8075" - -# Or for TLS -gitaly['tls_listen_addr'] = "0.0.0.0:9999" -gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" -gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" +gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces + listen_addr: '0.0.0.0:8075', + # Or for TLS + tls_listen_addr: '0.0.0.0:9999', + tls: { + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + storage: [ + { + name: 'storage1', + path: '/mnt/gitlab/git-data', + }, + ], +} ``` `path` can be included only for storage shards on the local Gitaly server. @@ -599,16 +632,21 @@ To configure Gitaly with TLS: <!-- Updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + gitaly['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:9999', + tls: { + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Verify Gitaly traffic is being served over TLS by [observing the types of Gitaly connections](#observe-type-of-gitaly-connections). 1. Optional. Improve security by: - 1. Disabling non-TLS connections by commenting out or deleting `gitaly['listen_addr']` in + 1. Disabling non-TLS connections by commenting out or deleting `gitaly['configuration'][:listen_addr]` in `/etc/gitlab/gitlab.rb`. 1. Saving the file. 1. [Reconfiguring GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -731,9 +769,16 @@ settings: 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby - # Default is 2 workers. The minimum is 2; 1 worker is always reserved as - # a passive stand-by. - gitaly['ruby_num_workers'] = 4 + gitaly['configuration'] = { + # ... + 'gitaly-ruby': { + # ... + # + # Default is 2 workers. The minimum is 2; 1 worker is always reserved as + # a passive stand-by. + num_workers: 4 + }, + } ``` 1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -773,21 +818,23 @@ example: ```ruby # in /etc/gitlab/gitlab.rb - -gitaly['concurrency'] = [ - { - 'rpc' => "/gitaly.SmartHTTPService/PostUploadPackWithSidechannel", - 'max_per_repo' => 20, - 'max_queue_time' => "1s", - 'max_queue_size' => 10 - }, - { - 'rpc' => "/gitaly.SSHService/SSHUploadPackWithSidechannel", - 'max_per_repo' => 20 - 'max_queue_time' => "1s", - 'max_queue_size' => 10 - } -] +gitaly['configuration'] = { + # ... + concurrency: [ + { + rpc: '/gitaly.SmartHTTPService/PostUploadPackWithSidechannel', + max_per_repo: 20, + max_queue_time: '1s', + max_queue_size: 10, + }, + { + rpc: '/gitaly.SSHService/SSHUploadPackWithSidechannel', + max_per_repo: 20, + max_queue_time: '1s', + max_queue_size: 10, + }, + ], +} ``` - `rpc` is the name of the RPC to set a concurrency limit for per repository. @@ -863,34 +910,32 @@ When these limits are reached, performance may be reduced and users may be disco > - `cpu_quota_us`[introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5422) in GitLab 15.10. To configure repository cgroups in Gitaly using the new method, use the following settings for the new configuration method -to `gitaly['cgroups']` in `/etc/gitlab/gitlab.rb`: +to `gitaly['configuration'][:cgroups]` in `/etc/gitlab/gitlab.rb`: -- `cgroups_mountpoint` is where the parent cgroup directory is mounted. Defaults to `/sys/fs/cgroup`. -- `cgroups_hierarchy_root` is the parent cgroup under which Gitaly creates groups, and +- `mountpoint` is where the parent cgroup directory is mounted. Defaults to `/sys/fs/cgroup`. +- `hierarchy_root` is the parent cgroup under which Gitaly creates groups, and is expected to be owned by the user and group Gitaly runs as. Omnibus GitLab creates the set of directories `mountpoint/<cpu|memory>/hierarchy_root` when Gitaly starts. -- `cgroups_memory_bytes` is the total memory limit that is imposed collectively on all +- `memory_bytes` is the total memory limit that is imposed collectively on all Git processes that Gitaly spawns. 0 implies no limit. -- `cgroups_cpu_shares` is the CPU limit that is imposed collectively on all Git +- `cpu_shares` is the CPU limit that is imposed collectively on all Git processes that Gitaly spawns. 0 implies no limit. The maximum is 1024 shares, which represents 100% of CPU. -- `cgroups_cpu_quota_us` is the - [`cfs_quota_us`](https://docs.kernel.org/scheduler/sched-bwc.html#management) +- `cpu_quota_us` is the [`cfs_quota_us`](https://docs.kernel.org/scheduler/sched-bwc.html#management) to throttle the cgroups' processes if they exceed this quota value. We set `cfs_period_us` to `100ms` so 1 core is `100000`. 0 implies no limit. -- `cgroups_repositories_count` is the number of cgroups in the cgroups pool. Each time a new Git +- `repositories.count` is the number of cgroups in the cgroups pool. Each time a new Git command is spawned, Gitaly assigns it to one of these cgroups based on the repository the command is for. A circular hashing algorithm assigns Git commands to these cgroups, so a Git command for a repository is always assigned to the same cgroup. -- `cgroups_repositories_memory_bytes` is the total memory limit imposed on all Git processes contained in a repository cgroup. - 0 implies no limit. This value cannot exceed that of the top level `cgroups_memory_bytes`. -- `cgroups_repositories_cpu_shares` is the CPU limit that is imposed on all Git processes contained in a repository cgroup. +- `repositories.memory_bytes` is the total memory limit imposed on all Git processes contained in a repository cgroup. + 0 implies no limit. This value cannot exceed that of the top level `memory_bytes`. +- `repositories.cpu_shares` is the CPU limit that is imposed on all Git processes contained in a repository cgroup. 0 implies no limit. The maximum is 1024 shares, which represents 100% of CPU. - This value cannot exceed that of the top level`cgroups_cpu_shares`. -- `cgroups_repositories_cpu_quota_us` is the - [`cfs_quota_us`](https://docs.kernel.org/scheduler/sched-bwc.html#management) + This value cannot exceed that of the top level`cpu_shares`. +- `repositories.cpu_quota_us` is the [`cfs_quota_us`](https://docs.kernel.org/scheduler/sched-bwc.html#management) that is imposed on all Git processes contained in a repository cgroup. A Git process can't use more then the given quota. We set `cfs_period_us` to `100ms` so 1 core is `100000`. 0 implies no limit. @@ -900,14 +945,16 @@ For example: ```ruby # in /etc/gitlab/gitlab.rb gitaly['configuration'] = { + # ... cgroups: { mountpoint: '/sys/fs/cgroup', hierarchy_root: 'gitaly', - memory_bytes: 64424509440, # 60gb, + memory_bytes: 64424509440, # 60gb + cpu_shares: 1024, cpu_quota_us: 400000 # 4 cores repositories: { count: 1000, - memory_bytes: 32212254720 # 20gb, + memory_bytes: 32212254720, # 20gb cpu_shares: 512, cpu_quota_us: 200000 # 2 cores }, @@ -983,10 +1030,16 @@ Make sure to schedule this during off-peak hours and keep the duration short (fo Edit `/etc/gitlab/gitlab.rb` and add: ```ruby -gitaly['daily_maintenance_start_hour'] = 4 -gitaly['daily_maintenance_start_minute'] = 30 -gitaly['daily_maintenance_duration'] = '30m' -gitaly['daily_maintenance_storages'] = ["default"] +gitaly['configuration'] = { + # ... + daily_maintenance: { + # ... + start_hour: 4, + start_minute: 30, + duration: '30m', + storages: ['default'], + }, +} ``` **For installations from source** @@ -1033,7 +1086,13 @@ transitioning" mode as follows: ```ruby # in /etc/gitlab/gitlab.rb -gitaly['auth_transitioning'] = true +gitaly['configuration'] = { + # ... + auth: { + # ... + transitioning: true, + }, +} ``` After you have made this change, your [Prometheus query](#verify-authentication-monitoring) @@ -1053,8 +1112,13 @@ To update to a new Gitaly authentication token, on each Gitaly client **and** Gi ```ruby # in /etc/gitlab/gitlab.rb - - gitaly['auth_token'] = '<new secret token>' + gitaly['configuration'] = { + # ... + auth: { + # ... + token: '<new secret token>', + }, + } ``` 1. Restart Gitaly: @@ -1084,7 +1148,13 @@ your Gitaly servers as follows: ```ruby # in /etc/gitlab/gitlab.rb -gitaly['auth_transitioning'] = false +gitaly['configuration'] = { + # ... + auth: { + # ... + transitioning: false, + }, +} ``` WARNING: @@ -1156,9 +1226,15 @@ below. In `/etc/gitlab/gitlab.rb`, set: ```ruby -gitaly['pack_objects_cache_enabled'] = true -## gitaly['pack_objects_cache_dir'] = '/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache' -## gitaly['pack_objects_cache_max_age'] = '5m' +gitaly['configuration'] = { + # ... + pack_objects_cache: { + # ... + enabled: true, + # dir: '/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache', + # max_age: '5m', + }, +} ``` #### `enabled` defaults to `false` @@ -1322,7 +1398,32 @@ process repositories that do not pass consistency checks. For Omnibus GitLab installations, edit `/etc/gitlab/gitlab.rb` and set the following keys (in this example, to disable the `hasDotgit` consistency check): -- In [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) and later: +- In [GitLab 15.10](https://gitlab.com/gitlab-org/gitaly/-/issues/4754) and later: + + ```ruby + ignored_blobs = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" + + gitaly['configuration'] = { + # ... + git: { + # ... + config: [ + # Populate a file with one unabbreviated SHA-1 per line. + # See https://git-scm.com/docs/git-config#Documentation/git-config.txt-fsckskipList + { key: "fsck.skipList", value: ignored_blobs }, + { key: "fetch.fsck.skipList", value: ignored_blobs }, + { key: "receive.fsck.skipList", value: ignored_blobs }, + + { key: "fsck.hasDotgit", value: "ignore" }, + { key: "fetch.fsck.hasDotgit", value: "ignore" }, + { key: "receive.fsck.hasDotgit", value: "ignore" }, + { key: "fsck.missingSpaceBeforeEmail", value: "ignore" }, + ], + }, + } + ``` + +- In [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) to GitLab 15.9: ```ruby ignored_blobs = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" @@ -1430,7 +1531,13 @@ proposed in issue [19185](https://gitlab.com/gitlab-org/gitlab/-/issues/19185). 1. Edit `/etc/gitlab/gitlab.rb` and configure `gitaly['gpg_signing_key_path']`: ```ruby - gitaly['gpg_signing_key_path'] = "/etc/gitlab/gitaly/signing_key.gpg" + gitaly['configuration'] = { + # ... + git: { + # ... + signing_key: '/etc/gitlab/gitaly/signing_key.gpg', + }, + } ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index cc64fe327a0b22118c66f15177b60cbc49fd86f3..2d2d3fd7b44751a73b8f99492a6fcc65bccbae6c 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -638,7 +638,7 @@ Updates to example must be made at: ```ruby # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('default') and in git_data_dirs on Gitaly nodes ('gitaly-1') + # server ('default') and in gitaly['configuration'][:storage][INDEX][:name] on Gitaly nodes ('gitaly-1') praefect['configuration'] = { # ... virtual_storage: [ @@ -894,10 +894,10 @@ because we rely on Praefect to route operations correctly. Particular attention should be shown to: -- The `gitaly['auth_token']` configured in this section must match the `token` value under - `praefect['configuration'][:virtual_storage][<index>][:node][<index>][:token]` on the Praefect node. This value was +- The `gitaly['configuration'][:auth][:token]` configured in this section must match the `token` + value under `praefect['configuration'][:virtual_storage][<index>][:node][<index>][:token]` on the Praefect node. This value was set in the [previous section](#praefect). This document uses the placeholder `PRAEFECT_INTERNAL_TOKEN` throughout. -- The storage names in `git_data_dirs` configured in this section must match the +- The storage names in `gitaly['configuration'][:storage]` configured in this section must match the storage names under `praefect['configuration'][:virtual_storage]` on the Praefect node. This was set in the [previous section](#praefect). This document uses `gitaly-1`, `gitaly-2`, and `gitaly-3` as Gitaly storage names. @@ -939,13 +939,16 @@ For more information on Gitaly server configuration, see our `/etc/gitlab/gitlab.rb`: ```ruby - # Make Gitaly accept connections on all network interfaces. - # Use firewalls to restrict access to this address/port. - gitaly['listen_addr'] = '0.0.0.0:8075' - - # Enable Prometheus metrics access to Gitaly. You must use firewalls - # to restrict access to this address/port. - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces. + # Use firewalls to restrict access to this address/port. + listen_addr: '0.0.0.0:8075', + # Enable Prometheus metrics access to Gitaly. You must use firewalls + # to restrict access to this address/port. + prometheus_listen_addr: '0.0.0.0:9236', + } ``` 1. Configure a strong `auth_token` for **Gitaly** by editing @@ -954,7 +957,13 @@ For more information on Gitaly server configuration, see our nodes. ```ruby - gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN' + gitaly['configuration'] = { + # ... + auth: { + # ... + token: 'PRAEFECT_INTERNAL_TOKEN', + }, + } ``` 1. Configure the GitLab Shell secret token, which is needed for `git push` operations. Either: @@ -983,13 +992,13 @@ For more information on Gitaly server configuration, see our gitlab_rails['internal_api_url'] = 'http://GITLAB_HOST' ``` -1. Configure the storage location for Git data by setting `git_data_dirs` in +1. Configure the storage location for Git data by setting `gitaly['configuration'][:storage]` in `/etc/gitlab/gitlab.rb`. Each Gitaly node should have a unique storage name (such as `gitaly-1`). - Instead of configuring `git_data_dirs` uniquely for each Gitaly node, it is + Instead of configuring `gitaly['configuration'][:storage]` uniquely for each Gitaly node, it is often easier to have include the configuration for all Gitaly nodes on every - Gitaly node. You can do this because the Praefect `virtual_storages` + Gitaly node. You can do this because the Praefect `virtual_storage` configuration maps each storage name (such as `gitaly-1`) to a specific node, and requests are routed accordingly. This means every Gitaly node in your fleet can share the same configuration. @@ -998,17 +1007,23 @@ For more information on Gitaly server configuration, see our # You can include the data dirs for all nodes in the same config, because # Praefect will only route requests according to the addresses provided in the # prior step. - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - }, - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - }, - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Save the changes to `/etc/gitlab/gitlab.rb` and diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index 52d307790bee66439d187e95d89f7094d68a0b49..baec8420a82582ec2afafa7aa6bc360ebbc7797f 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -358,7 +358,7 @@ server to keep them synchronized. ### Gitaly not listening on new address after reconfiguring -When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` values, Gitaly may +When updating the `gitaly['configuration'][:listen_addr]` or `gitaly['configuration'][:prometheus_listen_addr]` values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. When this occurs, run `sudo gitlab-ctl restart` to resolve the issue. This should no longer be @@ -692,7 +692,7 @@ praefect sql-migrate: OK (applied 21 migrations) This indicates that the virtual storage name used in the [Praefect configuration](praefect.md#praefect) does not match the storage name used in -[`git_data_dirs` setting](praefect.md#gitaly) for GitLab. +[`gitaly['configuration'][:storage][<index>][:name]` setting](praefect.md#gitaly) for GitLab. Resolve this by matching the virtual storage names used in Praefect and GitLab configuration. diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 56583deca89975a0bb936f1ce34eaf3e9b0105ee..37a7445c29019438d1a501b45df7b9c804c20058 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -201,7 +201,10 @@ To use an external Prometheus server: postgres_exporter['listen_address'] = '0.0.0.0:9187' # Gitaly nodes - gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" + gitaly['configuration'] = { + # ... + prometheus_listen_addr: '0.0.0.0:9236', + } ``` 1. Install and set up a dedicated Prometheus instance, if necessary, using the [official installation instructions](https://prometheus.io/docs/prometheus/latest/installation/). diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 7ee3a0ce3273ccd6a2bede8c161fd3c2681979a7..5944b41dea486c64ba043377f5ac357b6669478b 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1430,7 +1430,7 @@ Updates to example must be made at: }, # Praefect Virtual Storage config # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') virtual_storage: [ { # ... @@ -1522,7 +1522,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1571,20 +1571,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '<praefect_internal_token>' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1599,9 +1585,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '<praefect_internal_token>', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1610,31 +1616,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 2319cfb0356547d1cf60f76b24adeb31291f865c..8f89a6c8238ac9add54b280576fa34ed6924980f 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -1447,7 +1447,7 @@ Updates to example must be made at: }, # Praefect Virtual Storage config # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') virtual_storage: [ { # ... @@ -1539,7 +1539,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1588,20 +1588,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '<praefect_internal_token>' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1616,9 +1602,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '<praefect_internal_token>', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1627,31 +1633,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 62589ac3375afabb89674faf2f82ff479b1517e7..d87e8270dcbc7b2d319ea14f877f89d4025c3499 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -459,7 +459,7 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: storage paths, enable the network listener, and to configure the token: NOTE: - You can't remove the `default` entry from `git_data_dirs` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage). + You can't remove the `default` entry from `gitaly['configuration'][:storage]` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage). <!-- Updates to example must be made at: @@ -493,30 +493,48 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" - - # Gitaly and GitLab use two shared secrets for authentication, one to authenticate gRPC requests - # to Gitaly, and a second for authentication callbacks from GitLab-Shell to the GitLab internal API. - # The following two values must be the same as their respective values - # of the GitLab Rails application setup - gitaly['auth_token'] = 'gitalysecret' + # The secret token is used for authentication callbacks from Gitaly to the GitLab internal API. + # This must match the respective value in GitLab Rails application setup. gitlab_shell['secret_token'] = 'shellsecret' # Set the network addresses that the exporters used for monitoring will listen on node_exporter['listen_address'] = '0.0.0.0:9100' - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + prometheus_listen_addr: '0.0.0.0:9236', + # Gitaly Auth Token + # Should be the same as praefect_internal_token + auth: { + # ... + # + # Gitaly's authentication token is used to authenticate gRPC requests to Gitaly. This must match + # the respective value in GitLab Rails application setup. + token: 'gitalysecret', + }, + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + pack_objects_cache: { + # ... + enabled: true, + }, + storage: [ + { + name: 'default', + path: '/var/opt/gitlab/git-data', + }, + { + name: 'storage1', + path: '/mnt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace @@ -574,9 +592,14 @@ To configure Gitaly with TLS: <!-- Updates to following example must also be made at https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab --> ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + gitaly['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:9999', + tls: { + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Delete `gitaly['listen_addr']` to allow only encrypted connections. diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index b9fcd60327ec17fdf95baa08bcef68c8dce81c69..c28799ed4595f23e6d53bd64e534a1e29dfae3b5 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -1382,7 +1382,7 @@ Updates to example must be made at: }, # Praefect Virtual Storage config # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') virtual_storage: [ { # ... @@ -1474,7 +1474,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1523,20 +1523,6 @@ Updates to example must be made at: # balancer. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '<praefect_internal_token>' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1551,9 +1537,33 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + # Gitaly Auth Token + # Should be the same as praefect_internal_token + auth: { + # ... + token: '<praefect_internal_token>', + }, + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + pack_objects_cache: { + # ... + enabled: true, + }, + } + # # END user configuration ``` @@ -1562,31 +1572,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index c6bad8d97b0dac71654cf73a2f875e63d071754f..abdc1d28312de3d98a0739a36cf706b67715d924 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -1443,7 +1443,7 @@ Updates to example must be made at: }, # Praefect Virtual Storage config # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') virtual_storage: [ { # ... @@ -1535,7 +1535,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1584,20 +1584,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '<praefect_internal_token>' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1612,9 +1598,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '<praefect_internal_token>', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1623,31 +1629,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index d41e8ff6f042a911c2e5cced97e7f6fa9d763d8e..43d63d4f08e1c2a9ecc96b9066b47543a32db693 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -1379,7 +1379,7 @@ Updates to example must be made at: }, # Praefect Virtual Storage config # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') virtual_storage: [ { # ... @@ -1471,7 +1471,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs are used as an example: @@ -1520,20 +1520,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '<praefect_internal_token>' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1548,9 +1534,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '<praefect_internal_token>', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1559,31 +1565,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 3d4f39b5ff0719914ef68289d99be5e96e9f68fb..87ae6fdb003f4577cc5ed8b47e7cf14d5a167b6b 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -84,7 +84,7 @@ To create a Git hook that applies to all repositories, set a global server hook. Before creating a global server hook, you must choose a directory for it. -For Omnibus GitLab installations, the directory is set in `gitlab.rb` under `gitaly['custom_hooks_dir']`. You can either: +For Omnibus GitLab installations, the directory is set in `gitlab.rb` under `gitaly['configuration'][:hooks][:custom_hooks_dir]`. You can either: - Use the default suggestion of the `/var/opt/gitlab/gitaly/custom_hooks` directory by uncommenting it. - Add your own setting. diff --git a/doc/administration/sidekiq/index.md b/doc/administration/sidekiq/index.md index b5e943ccd7df3321709ac279f3a2c54b9e756d7c..8645df55a62f5424deff0260a647f036da87ebaa 100644 --- a/doc/administration/sidekiq/index.md +++ b/doc/administration/sidekiq/index.md @@ -32,11 +32,19 @@ By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. T ## Gitaly - # Make Gitaly accept connections on all network interfaces - gitaly['listen_addr'] = "0.0.0.0:8075" - ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network - ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token - gitaly['auth_token'] = 'abc123secret' + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces + listen_addr: '0.0.0.0:8075', + auth: { + ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network + ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token + token: 'abc123secret', + }, + } + + gitaly['auth_token'] = '' praefect['configuration'][:auth][:token] = 'abc123secret' gitlab_rails['gitaly_token'] = 'abc123secret' diff --git a/doc/topics/git/troubleshooting_git.md b/doc/topics/git/troubleshooting_git.md index a116f6cc978c9c0b38365e8d5773f34281a97867..3745031935001ab127cc1a9d56cf3e59d5cfe8fe 100644 --- a/doc/topics/git/troubleshooting_git.md +++ b/doc/topics/git/troubleshooting_git.md @@ -221,11 +221,17 @@ apply more than one: 1. Modify the GitLab instance's [`gitlab.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/13.5.1+ee.0/files/gitlab-config-template/gitlab.rb.template#L1435-1455) file: - ```shell - gitaly['gitconfig'] = [ - # Set the http.postBuffer size, in bytes - {key: "http.postBuffer", value: "524288000"}, - ] + ```ruby + gitaly['configuration'] = { + # ... + git: { + # ... + config: [ + # Set the http.postBuffer size, in bytes + {key: "http.postBuffer", value: "524288000"}, + ], + }, + } ``` 1. After applying this change, apply the configuration change: diff --git a/doc/update/index.md b/doc/update/index.md index e2e46086e9da7359d72f4fffa72388ddc65edab1..8d89a330beba2504a5351a1be5d7656f0b5963a8 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -264,6 +264,11 @@ NOTE: Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/) and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches. +### 15.10.0 + +- Gitaly configuration changes significantly in Omnibus GitLab 16.0. You can begin migrating to the new structure in Omnibus GitLab 15.10 while backwards compatibility is + maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](#gitaly-omnibus-gitlab-configuration-structure-change). + ### 15.9.0 - **Upgrade to patch release 15.9.3 or later**. This provides fixes for two database migration bugs: @@ -1465,6 +1470,163 @@ If you have already upgraded to GitLab 15.9 following these instructions, your i See [issue 393216](https://gitlab.com/gitlab-org/gitlab/-/issues/393216) for more information. +### Gitaly: Omnibus GitLab configuration structure change + +Gitaly configuration structure in Omnibus GitLab [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 to be consistent with the Gitaly configuration +structure used in source installs. + +As a result of this change, a single hash under `gitaly['configuration']` holds most Gitaly +configuration. Some `gitaly['..']` configuration options will continue to be used by Omnibus GitLab 16.0 and later: + +- `enable` +- `dir` +- `log_directory` +- `bin_path` +- `env_directory` +- `env` +- `open_files_ulimit` +- `consul_service_name` +- `consul_service_meta` + +Migrate by moving your existing configuration under the new structure. The new structure is supported from Omnibus GitLab 15.10. + +The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration: + +1. Replace the `...` with the value from the old key. +1. Skip any keys you haven't configured a value for previously. +1. Remove the old keys from the configuration once migrated. +1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added. + + ```ruby +gitaly['configuration'] = { + # gitaly['socket_path'] + socket_path: ..., + # gitaly['runtime_dir'] + runtime_dir: ..., + # gitaly['listen_addr'] + listen_addr: ..., + # gitaly['prometheus_listen_addr'] + prometheus_listen_addr: ..., + # gitaly['tls_listen_addr'] + tls_listen_addr: ..., + tls: { + # gitaly['certificate_path'] + certificate_path: ..., + # gitaly['key_path'] + key_path: ..., + }, + # gitaly['graceful_restart_timeout'] + graceful_restart_timeout: ..., + logging: { + # gitaly['logging_level'] + level: ..., + # gitaly['logging_format'] + format: ..., + # gitaly['logging_sentry_dsn'] + sentry_dsn: ..., + # gitaly['logging_ruby_sentry_dsn'] + ruby_sentry_dsn: ..., + # gitaly['logging_sentry_environment'] + sentry_environment: ..., + # gitaly['log_directory'] + dir: ..., + }, + prometheus: { + # gitaly['prometheus_grpc_latency_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + grpc_latency_buckets: ..., + }, + auth: { + # gitaly['auth_token'] + token: ..., + # gitaly['auth_transitioning'] + transitioning: ..., + }, + git: { + # gitaly['git_catfile_cache_size'] + catfile_cache_size: ..., + # gitaly['git_bin_path'] + bin_path: ..., + # gitaly['use_bundled_git'] + use_bundled_binaries: ..., + # gitaly['gpg_signing_key_path'] + signing_key: ..., + # gitaly['gitconfig']. This is still an array but the type of the elements have changed. + config: [ + { + # Previously the elements contained 'section', and 'subsection' in addition to 'key'. Now + # these all should be concatenated into just 'key', separated by dots. For example, + # {section: 'first', subsection: 'middle', key: 'last', value: 'value'}, should become + # {key: 'first.middle.last', value: 'value'}. + key: ..., + value: ..., + }, + ], + }, + 'gitaly-ruby': { + # gitaly['ruby_max_rss'] + max_rss: ..., + # gitaly['ruby_graceful_restart_timeout'] + graceful_restart_timeout: ..., + # gitaly['ruby_restart_delay'] + restart_delay: ..., + # gitaly['ruby_num_workers'] + num_workers: ..., + }, + # gitaly['storage']. While the structure is the same, the string keys in the array elements + # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. + storage: ..., + hooks: { + # gitaly['custom_hooks_dir'] + custom_hooks_dir: ..., + }, + daily_maintenance: { + # gitaly['daily_maintenance_disabled'] + disabled: ..., + # gitaly['daily_maintenance_start_hour'] + start_hour: ..., + # gitaly['daily_maintenance_start_minute'] + start_minute: ..., + # gitaly['daily_maintenance_duration'] + duration: ..., + # gitaly['daily_maintenance_storages'] + storages: ..., + }, + cgroups: { + # gitaly['cgroups_mountpoint'] + mountpoint: ..., + # gitaly['cgroups_hierarchy_root'] + hierarchy_root: ..., + # gitaly['cgroups_memory_bytes'] + memory_bytes: ..., + # gitaly['cgroups_cpu_shares'] + cpu_shares: ..., + repositories: { + # gitaly['cgroups_repositories_count'] + count: ..., + # gitaly['cgroups_repositories_memory_bytes'] + memory_bytes: ..., + # gitaly['cgroups_repositories_cpu_shares'] + cpu_shares: ..., + } + }, + # gitaly['concurrency']. While the structure is the same, the string keys in the array elements + # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. + concurrency: ..., + # gitaly['rate_limiting']. While the structure is the same, the string keys in the array elements + # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. + rate_limiting: ..., + pack_objects_cache: { + # gitaly['pack_objects_cache_enabled'] + enabled: ..., + # gitaly['pack_objects_cache_dir'] + dir: ..., + # gitaly['pack_objects_cache_max_age'] + max_age: ..., + } +} +``` + ### Praefect: Omnibus GitLab configuration structure change Praefect configuration structure in Omnibus GitLab [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 to be consistent with the Praefect configuration