Commit cb3b4a15 authored by Paul Charlton's avatar Paul Charlton Committed by Robert Speicher

Support multiple Redis instances based on queue type

parent 4daa6da5
......@@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb
lib/gitlab/diff/position_tracer.rb
app/policies/project_policy.rb
app/models/concerns/relative_positioning.rb
lib/gitlab/redis/*.rb
......@@ -31,6 +31,9 @@ eslint-report.html
/config/initializers/smtp_settings.rb
/config/initializers/relative_url.rb
/config/resque.yml
/config/redis.cache.yml
/config/redis.queues.yml
/config/redis.shared_state.yml
/config/unicorn.rb
/config/secrets.yml
/config/sidekiq.yml
......
......@@ -4,7 +4,10 @@ class HealthController < ActionController::Base
CHECKS = [
Gitlab::HealthChecks::DbCheck,
Gitlab::HealthChecks::RedisCheck,
Gitlab::HealthChecks::Redis::RedisCheck,
Gitlab::HealthChecks::Redis::CacheCheck,
Gitlab::HealthChecks::Redis::QueuesCheck,
Gitlab::HealthChecks::Redis::SharedStateCheck,
Gitlab::HealthChecks::FsShardsCheck
].freeze
......
......@@ -149,7 +149,7 @@ module Ci
private
def cleanup_runner_queue
Gitlab::Redis.with do |redis|
Gitlab::Redis::Queues.with do |redis|
redis.del(runner_queue_key)
end
end
......
......@@ -1386,15 +1386,15 @@ class Project < ActiveRecord::Base
end
def pushes_since_gc
Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i }
Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
end
def increment_pushes_since_gc
Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) }
Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) }
end
def reset_pushes_since_gc
Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) }
Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) }
end
def route_map_for(commit_sha)
......@@ -1457,7 +1457,7 @@ class Project < ActiveRecord::Base
from && self != from
end
def pushes_since_gc_redis_key
def pushes_since_gc_redis_shared_state_key
"projects/#{id}/pushes_since_gc"
end
......
......@@ -3,7 +3,10 @@ require 'prometheus/client/formats/text'
class MetricsService
CHECKS = [
Gitlab::HealthChecks::DbCheck,
Gitlab::HealthChecks::RedisCheck,
Gitlab::HealthChecks::Redis::RedisCheck,
Gitlab::HealthChecks::Redis::CacheCheck,
Gitlab::HealthChecks::Redis::QueuesCheck,
Gitlab::HealthChecks::Redis::SharedStateCheck,
Gitlab::HealthChecks::FsShardsCheck
].freeze
......
module Milestones
class DestroyService < Milestones::BaseService
def execute(milestone)
return unless milestone.is_project_milestone?
Please register or sign in to reply
Milestone.transaction do
update_params = { milestone: nil }
milestone.issues.each do |issue|
Issues::UpdateService.new(parent, current_user, update_params).execute(issue)
Issues::UpdateService.new(project, current_user, update_params).execute(issue)
end
milestone.merge_requests.each do |merge_request|
MergeRequests::UpdateService.new(parent, current_user, update_params).execute(merge_request)
MergeRequests::UpdateService.new(project, current_user, update_params).execute(merge_request)
end
event_service.destroy_milestone(milestone, current_user)
......
......@@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration.
This file is called `resque.yml` for historical reasons. We are **NOT**
using Resque at the moment. It is used to specify Redis configuration
values instead.
values when a single database instance of Redis is desired.
# Advanced Redis configuration files
In more advanced configurations of Redis key-value storage, it is desirable
to separate the keys by lifecycle and intended use to ease provisioning and
management of scalable Redis clusters.
These settings provide routing and other configuration data (such as sentinel,
persistence policies, and other Redis customization) for connections
to Redis single instances, Redis sentinel, and Redis clusters.
If desired, the routing URL provided by these settings can be used with:
1. Unix Socket
1. named socket for each Redis instance desired.
2. `database number` for each Redis instance desired.
2. TCP Socket
1. `host name` or IP for each Redis instance desired
2. TCP port number for each Redis instance desired
3. `database number` for each Redis instance desired
## Example URL attribute formats for GitLab Redis `.yml` configuration files
* Unix Socket, default Redis database (0)
* `url: unix:/path/to/redis.sock`
* `url: unix:/path/to/redis.sock?db=`
* Unix Socket, Redis database 44
* `url: unix:/path/to/redis.sock?db=44`
* `url: unix:/path/to/redis.sock?extra=foo&db=44`
* TCP Socket for Redis on localhost, port 6379, database 33
* `url: redis://:mynewpassword@localhost:6379/33`
* TCP Socket for Redis on remote host `myserver`, port 6379, database 33
* `url: redis://:mynewpassword@myserver:6379/33`
## redis.cache.yml
If configured, `redis.cache.yml` overrides the
`resque.yml` settings to configure the Redis database instance
used for `Rails.cache` and other volatile non-persistent data which enhances
the performance of GitLab.
Settings here can be overridden by the environment variable
`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides
an alternate location for configuration settings.
The order of precedence for the URL used to connect to the Redis instance
used for `cache` is:
1. URL from a configuration file pointed to by the
`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
2. URL from `redis.cache.yml`
3. URL from a configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. URL from `resque.yml`
5. `redis://localhost:6380`
The order of precedence for all other configuration settings for `cache`
are selected from only the first of the following files found (if a setting
is not provided in an earlier file, the remainder of the files are not
searched):
1. the configuration file pointed to by the
`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
2. the configuration file `redis.cache.yml`
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`
## redis.queues.yml
If configured, `redis.queues.yml` overrides the
`resque.yml` settings to configure the Redis database instance
used for clients of `::Gitlab::Redis::Queues`.
These queues are intended to be the foundation
of reliable inter-process communication between modules, whether on the same
host node, or within a cluster. The primary clients of the queues are
SideKiq, Mailroom, CI Runner, Workhorse, and push services. Settings here can
be overridden by the environment variable
`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for
configuration settings.
The order of precedence for the URL used to connect to the Redis instance
used for `queues` is:
1. URL from a configuration file pointed to by the
`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
2. URL from `redis.queues.yml`
3. URL from a configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. URL from `resque.yml`
5. `redis://localhost:6381`
The order of precedence for all other configuration settings for `queues`
are selected from only the first of the following files found (if a setting
is not provided in an earlier file, the remainder of the files are not
searched):
1. the configuration file pointed to by the
`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
2. the configuration file `redis.queues.yml`
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`
## redis.shared_state.yml
If configured, `redis.shared_state.yml` overrides the
`resque.yml` settings to configure the Redis database instance
used for clients of `::Gitlab::Redis::SharedState` such as session state,
and rate limiting.
Settings here can be overridden by the environment variable
`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides
an alternate location for configuration settings.
The order of precedence for the URL used to connect to the Redis instance
used for `shared_state` is:
1. URL from a configuration file pointed to by the
`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
2. URL from `redis.shared_state.yml`
3. URL from a configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. URL from `resque.yml`
5. `redis://localhost:6382`
The order of precedence for all other configuration settings for `shared_state`
are selected from only the first of the following files found (if a setting
is not provided in an earlier file, the remainder of the files are not
searched):
1. the configuration file pointed to by the
`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
2. the configuration file `redis.shared_state.yml`
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`
......@@ -6,7 +6,9 @@ Bundler.require(:default, Rails.env)
module Gitlab
class Application < Rails::Application
require_dependency Rails.root.join('lib/gitlab/redis')
require_dependency Rails.root.join('lib/gitlab/redis/cache')
require_dependency Rails.root.join('lib/gitlab/redis/queues')
require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
require_dependency Rails.root.join('lib/gitlab/request_context')
# Settings in config/environments/* take precedence over those specified here.
......@@ -142,15 +144,15 @@ module Gitlab
end
end
# Use Redis caching across all environments
redis_config_hash = Gitlab::Redis.params
redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
# Use caching across all environments
caching_config_hash = Gitlab::Redis::Cache.params
caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
if Sidekiq.server? # threaded context
redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
redis_config_hash[:pool_timeout] = 1
caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
caching_config_hash[:pool_timeout] = 1
end
config.cache_store = :redis_store, redis_config_hash
config.cache_store = :redis_store, caching_config_hash
config.active_record.raise_in_transactional_callbacks = true
......
# Make sure we initialize a Redis connection pool before Sidekiq starts
# multi-threaded execution.
Gitlab::Redis.with { nil }
# Make sure we initialize a Redis connection pool before multi-threaded
# execution starts by
# 1. Sidekiq
# 2. Rails.cache
# 3. HTTP clients
Gitlab::Redis::Cache.with { nil }
Gitlab::Redis::Queues.with { nil }
Gitlab::Redis::SharedState.with { nil }
Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) }
Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) }
Peek.into Peek::Views::Host
Peek.into Peek::Views::PerformanceBar
......
......@@ -19,12 +19,12 @@ cookie_key = if Rails.env.development?
if Rails.env.test?
Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
else
redis_config = Gitlab::Redis.params
redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE
sessions_config = Gitlab::Redis::SharedState.params
sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE
Gitlab::Application.config.session_store(
:redis_store, # Using the cookie_store would enable session replay attacks.
servers: redis_config,
servers: sessions_config,
key: cookie_key,
secure: Gitlab.config.gitlab.https,
httponly: true,
......
# Custom Redis configuration
redis_config_hash = Gitlab::Redis.params
redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
# Custom Queues configuration
queues_config_hash = Gitlab::Redis::Queues.params
queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
# Default is to retry 25 times with exponential backoff. That's too much.
Sidekiq.default_worker_options = { retry: 3 }
Sidekiq.configure_server do |config|
config.redis = redis_config_hash
config.redis = queues_config_hash
config.server_middleware do |chain|
chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
......@@ -54,7 +54,7 @@ Sidekiq.configure_server do |config|
end
Sidekiq.configure_client do |config|
config.redis = redis_config_hash
config.redis = queues_config_hash
config.client_middleware do |chain|
chain.add Gitlab::SidekiqStatus::ClientMiddleware
......
......@@ -21,7 +21,7 @@
:delivery_method: sidekiq
:delivery_options:
:redis_url: <%= config[:redis_url].to_json %>
:namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %>
:namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %>
:queue: email_receiver
:worker: EmailReceiverWorker
<% if config[:sentinels] %>
......@@ -36,7 +36,7 @@
:arbitration_method: redis
:arbitration_options:
:redis_url: <%= config[:redis_url].to_json %>
:namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %>
:namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %>
<% if config[:sentinels] %>
:sentinels:
<% config[:sentinels].each do |sentinel| %>
......
# If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
development:
url: redis://localhost:6379/10
#
# url: redis://localhost:6380
# sentinels:
# -
# host: localhost
# port: 26380 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26380 # point to sentinel, not to redis port
test:
url: redis://localhost:6379/10
#
# url: redis://localhost:6380
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.cache.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
##
# url: redis://master:6380
# sentinels:
# -
# host: slave1
# port: 26380 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26380 # point to sentinel, not to redis port
# If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
development:
url: redis://localhost:6379/11
#
# url: redis://localhost:6381
# sentinels:
# -
# host: localhost
# port: 26381 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26381 # point to sentinel, not to redis port
test:
url: redis://localhost:6379/11
#
# url: redis://localhost:6381
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.queues.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
##
# url: redis://master:6381
# sentinels:
# -
# host: slave1
# port: 26381 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26381 # point to sentinel, not to redis port
# If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
development:
url: redis://localhost:6379/12
#
# url: redis://localhost:6382
# sentinels:
# -
# host: localhost
# port: 26382 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26382 # point to sentinel, not to redis port
test:
url: redis://localhost:6379/12
#
# url: redis://localhost:6382
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.shared_state.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
##
# url: redis://master:6382
# sentinels:
# -
# host: slave1
# port: 26382 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26382 # point to sentinel, not to redis port
......@@ -56,7 +56,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
end
def activities(from, to, page: 1)
Gitlab::Redis.with do |redis|
Gitlab::Redis::SharedState.with do |redis|
redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
with_scores: true,
limit: limit(page))
......@@ -64,7 +64,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
end
def activities_count(from, to)
Gitlab::Redis.with do |redis|
Gitlab::Redis::SharedState.with do |redis|
redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
end
end
......
......@@ -4,6 +4,11 @@ This is the documentation for configuring a Highly Available Redis setup when
you have installed Redis all by yourself and not using the bundled one that
comes with the Omnibus packages.
Note also that you may elect to override all references to
`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis
settings outlined in
[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
We cannot stress enough the importance of reading the
[Overview section](redis.md#overview) of the Omnibus Redis HA as it provides
some invaluable information to the configuration of Redis. Please proceed to
......
......@@ -15,6 +15,12 @@ prefixed with 'session:gitlab:', so they would look like
'session:gitlab:976aa289e2189b17d7ef525a6702ace9'. Below we describe how to
remove the keys in the old format.
**Note:** the instructions below must be modified in accordance with your
configuration settings if you have used the advanced Redis
settings outlined in
[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
First we define a shell function with the proper Redis connection details.
```
......
......@@ -420,12 +420,6 @@ GitLab Shell is an SSH access and repository management software developed speci
**Note:** Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in /etc/hosts ("127.0.0.1 hostname"). This might be necessary for example if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check will fail with "Check GitLab API access: FAILED. code: 401" and pushing commits will be rejected with "[remote rejected] master -> master (hook declined)".
**Note:** GitLab Shell application startup time can be greatly reduced by disabling RubyGems. This can be done in several manners:
Please register or sign in to reply
* Export `RUBYOPT=--disable-gems` environment variable for the processes
* Compile Ruby with `configure --disable-rubygems` to disable RubyGems by default. Not recommened for system-wide Ruby.
* Omnibus GitLab [replaces the *shebang* line of the `gitlab-shell/bin/*` scripts](https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/1707)
### Install gitlab-workhorse
GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The
......
......@@ -27,7 +27,7 @@ module Gitlab
time = Time.now.utc.to_i
key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}"
Gitlab::Redis.with do |redis|
Gitlab::Redis::SharedState.with do |redis|
unique_ips_count = nil
redis.multi do |r|
r.zadd(key, time, ip)
......
......@@ -23,7 +23,7 @@ module Gitlab
end
def self.cached_results_for_projects(projects)
result = Gitlab::Redis.with do |redis|
result = Gitlab::Redis::Cache.with do |redis|
redis.multi do
projects.each do |project|
cache_key = cache_key_for_project(project)
......@@ -100,19 +100,19 @@ module Gitlab
end
def load_from_cache
Gitlab::Redis.with do |redis|
Gitlab::Redis::Cache.with do |redis|
self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref)
end
end
def store_in_cache
Gitlab::Redis.with do |redis|
Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
end
end
def delete_from_cache
Gitlab::Redis.with do |redis|
Gitlab::Redis::Cache.with do |redis|
redis.del(cache_key)
end
end
......@@ -120,7 +120,7 @@ module Gitlab
def has_cache?
return self.loaded unless self.loaded.nil?
Gitlab::Redis.with do |redis|
Gitlab::Redis::Cache.with do |redis|
redis.exists(cache_key)
end
end
......
......@@ -12,23 +12,23 @@ module Gitlab
end
def get
Gitlab::Redis.with do |redis|
data = redis.get(redis_key)
Gitlab::Redis::SharedState.with do |redis|
data = redis.get(redis_shared_state_key)
JSON.parse(data, symbolize_names: true) if data
end
end
def store!(params)
Gitlab::Redis.with do |redis|
Gitlab::Redis::SharedState.with do |redis|
params = params.to_json
redis.set(redis_key, params, ex: EXPIRY_TIME)
redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME)
token
end
end
def delete
Gitlab::Redis.with do |redis|
redis.del(redis_key)
Gitlab::Redis::SharedState.with do |redis|
redis.del(redis_shared_state_key)
end
end
......@@ -38,7 +38,7 @@ module Gitlab
Devise.friendly_token(TOKEN_LENGTH)
end
def redis_key
def redis_shared_state_key
"gitlab:chat_names:#{token}"
end
end
......
......@@ -25,7 +25,7 @@ module Gitlab
def cached_application_settings
begin
::ApplicationSetting.cached
rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL
Please register or sign in to reply
rescue ::Redis::BaseError, ::Errno::ENOENT
# In case Redis isn't running or the Redis UNIX socket file is not available
end
end
......
......@@ -145,7 +145,7 @@ module Gitlab
def track_rename(type, old_path, new_path)
key = redis_key_for_type(type)
Gitlab::Redis.with do |redis|
Gitlab::Redis::SharedState.with do |redis|
redis.lpush(key, [old_path, new_path].to_json)
redis.expire(key, 2.weeks.to_i)
end
......@@ -155,7 +155,7 @@ module Gitlab
def reverts_for_type(type)
key = redis_key_for_type(type)
Gitlab::Redis.with do |redis|
Gitlab::Redis::SharedState.with do |redis|
failed_reverts = []
while rename_info = redis.lpop(key)
......
......@@ -2,17 +2,17 @@ module Gitlab
module EtagCaching
class Store
EXPIRY_TIME = 20.minutes
REDIS_NAMESPACE = 'etag:'.freeze
SHARED_STATE_NAMESPACE = 'etag:'.freeze
def get(key)
Gitlab::Redis.with { |redis| redis.get(redis_key(key)) }
Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) }
end
def touch(key, only_if_missing: false)
etag = generate_etag
Gitlab::Redis.with do |redis|
redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
Gitlab::Redis::SharedState.with do |redis|
redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
end
etag
......@@ -24,10 +24,10 @@ module Gitlab
SecureRandom.hex
end
def redis_key(key)
def redis_shared_state_key(key)