diff --git a/doc/administration/logs.md b/doc/administration/logs.md index 9030c941cadc38bed2edf0312e7bfbd3e66bab13..9b1efb610f8da2bb7bd24b7a76294154604276f6 100644 --- a/doc/administration/logs.md +++ b/doc/administration/logs.md @@ -338,3 +338,12 @@ installations from source. [repocheck]: repository_checks.md [Rack Attack]: ../security/rack_attack.md [Rate Limit]: ../user/admin_area/settings/rate_limits_on_raw_endpoints.md + +## `database_load_balancing.log` + +Introduced in GitLab 12.3 for observability of [Database Load +Balancing](https://docs.gitlab.com/ee/administration/database_load_balancing.html) +when enabled. This file lives in +`/var/log/gitlab/gitlab-rails/database_load_balancing.log` for Omnibus GitLab +packages or in `/home/git/gitlab/log/database_load_balancing.log` for +installations from source. diff --git a/ee/lib/gitlab/database/load_balancing.rb b/ee/lib/gitlab/database/load_balancing.rb index 0aefad8e2ba7d7e175c635adb4f62b5861ca8977..e4ad6711801e4329469bc01ecac207a14ec98c46 100644 --- a/ee/lib/gitlab/database/load_balancing.rb +++ b/ee/lib/gitlab/database/load_balancing.rb @@ -1,15 +1,11 @@ # frozen_string_literal: true -# rubocop:disable GitlabSecurity/PublicSend - module Gitlab module Database module LoadBalancing # The connection proxy to use for load balancing (if enabled). cattr_accessor :proxy - LOG_TAG = 'DB-LB'.freeze - # The exceptions raised for connection errors. CONNECTION_ERRORS = if defined?(PG) [ @@ -70,14 +66,6 @@ def self.service_discovery_configuration } end - # rubocop:disable Gitlab/RailsLogger - def self.log(level, message) - Rails.logger.tagged(LOG_TAG) do - Rails.logger.send(level, message) - end - end - # rubocop:enable Gitlab/RailsLogger - def self.pool_size ActiveRecord::Base.configurations[Rails.env]['pool'] end diff --git a/ee/lib/gitlab/database/load_balancing/host.rb b/ee/lib/gitlab/database/load_balancing/host.rb index 20011158683c8062cac676b22a0e76289a573436..be3adf185df64d3f80cd111b3c352e5dc2dbea9b 100644 --- a/ee/lib/gitlab/database/load_balancing/host.rb +++ b/ee/lib/gitlab/database/load_balancing/host.rb @@ -54,7 +54,12 @@ def disconnect!(timeout = 120) end def offline! - LoadBalancing.log(:warn, "Marking host #{@host} as offline") + LoadBalancing::Logger.warn( + event: :host_offline, + message: 'Marking host as offline', + db_host: @host, + db_port: @port + ) @online = false @pool.disconnect! @@ -66,7 +71,14 @@ def online? refresh_status - LoadBalancing.log(:info, "Host #{@host} came back online") if @online + if @online + LoadBalancing::Logger.info( + event: :host_online, + message: 'Host came back online', + db_host: @host, + db_port: @port + ) + end @online rescue *CONNECTION_ERRORS diff --git a/ee/lib/gitlab/database/load_balancing/load_balancer.rb b/ee/lib/gitlab/database/load_balancing/load_balancer.rb index eddfbb2beaa9683f1c5d773b5bc7c6de33d44683..9668039a4c267b71a2e1a11f22844732e5625917 100644 --- a/ee/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/ee/lib/gitlab/database/load_balancing/load_balancer.rb @@ -38,7 +38,19 @@ def read(&block) # # In this event we'll cycle through the secondaries at most 3 # times before using the primary instead. - if conflict_retried < @host_list.length * 3 + will_retry = conflict_retried < @host_list.length * 3 + + LoadBalancing::Logger.warn( + event: :host_query_conflict, + message: 'Query conflict on host', + conflict_retried: conflict_retried, + will_retry: will_retry, + db_host: host.host, + db_port: host.port, + host_list_length: @host_list.length + ) + + if will_retry conflict_retried += 1 release_host else @@ -53,8 +65,12 @@ def read(&block) end end - LoadBalancing - .log(:warn, 'No secondaries were available, using primary instead') + LoadBalancing::Logger.warn( + event: :no_secondaries_available, + message: 'No secondaries were available, using primary instead', + conflict_retried: conflict_retried, + host_list_length: @host_list.length + ) read_write(&block) end diff --git a/ee/lib/gitlab/database/load_balancing/logger.rb b/ee/lib/gitlab/database/load_balancing/logger.rb new file mode 100644 index 0000000000000000000000000000000000000000..ee67ffcc99ce8edd1997ad84ca8c31c4b1bfa824 --- /dev/null +++ b/ee/lib/gitlab/database/load_balancing/logger.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'database_load_balancing' + end + end + end + end +end diff --git a/ee/spec/lib/gitlab/database/load_balancing_spec.rb b/ee/spec/lib/gitlab/database/load_balancing_spec.rb index 976d938199419de3455561f7c426a8f6e8803414..b34f69645ca58ab19728707671a0dcaa4f532f2d 100644 --- a/ee/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/ee/spec/lib/gitlab/database/load_balancing_spec.rb @@ -1,14 +1,6 @@ require 'spec_helper' describe Gitlab::Database::LoadBalancing do - describe '.log' do - it 'logs a message' do - expect(Rails.logger).to receive(:info).with('boop') - - described_class.log(:info, 'boop') - end - end - describe '.configuration' do it 'returns a Hash' do config = { 'hosts' => %w(foo) }