gitaly_client.rb 14.6 KB
Newer Older
1 2
# frozen_string_literal: true

3 4
require 'base64'

5
require 'gitaly'
6 7
require 'grpc/health/v1/health_pb'
require 'grpc/health/v1/health_services_pb'
8 9 10

module Gitlab
  module GitalyClient
11
    include Gitlab::Metrics::Methods
12

13 14 15 16 17 18 19 20 21 22
    class TooManyInvocationsError < StandardError
      attr_reader :call_site, :invocation_count, :max_call_stack

      def initialize(call_site, invocation_count, max_call_stack, most_invoked_stack)
        @call_site = call_site
        @invocation_count = invocation_count
        @max_call_stack = max_call_stack
        stacks = most_invoked_stack.join('\n') if most_invoked_stack

        msg = "GitalyClient##{call_site} called #{invocation_count} times from single request. Potential n+1?"
23
        msg = "#{msg}\nThe following call site called into Gitaly #{max_call_stack} times:\n#{stacks}\n" if stacks
24 25 26 27 28

        super(msg)
      end
    end

Stan Hu's avatar
Stan Hu committed
29
    PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
30
    SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'
31
    MAXIMUM_GITALY_CALLS = 30
32
    CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
33
    GITALY_METADATA_FILENAME = '.gitaly-metadata'
34

35
    MUTEX = Mutex.new
36

37 38 39 40 41
    def self.stub(name, storage)
      MUTEX.synchronize do
        @stubs ||= {}
        @stubs[storage] ||= {}
        @stubs[storage][name] ||= begin
42 43
          klass = stub_class(name)
          addr = stub_address(storage)
44
          creds = stub_creds(storage)
45
          klass.new(addr, creds, interceptors: interceptors)
46 47
        end
      end
48 49
    end

50
    def self.interceptors
51
      return [] unless Labkit::Tracing.enabled?
52

53
      [Labkit::Tracing::GRPC::ClientInterceptor.instance]
54 55 56
    end
    private_class_method :interceptors

57
    def self.stub_cert_paths
58 59
      cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"]
      cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE
60 61 62 63 64
      cert_paths
    end

    def self.stub_certs
      return @certs if @certs
65

66
      @certs = stub_cert_paths.flat_map do |cert_file|
Ahmad Hassan's avatar
Ahmad Hassan committed
67
        File.read(cert_file).scan(PEM_REGEX).map do |cert|
Nick Thomas's avatar
Nick Thomas committed
68 69
          OpenSSL::X509::Certificate.new(cert).to_pem
        rescue OpenSSL::OpenSSLError => e
70
          Rails.logger.error "Could not load certificate #{cert_file} #{e}" # rubocop:disable Gitlab/RailsLogger
Nick Thomas's avatar
Nick Thomas committed
71 72
          Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file })
          nil
Ahmad Hassan's avatar
Ahmad Hassan committed
73 74
        end.compact
      end.uniq.join("\n")
75 76
    end

77 78
    def self.stub_creds(storage)
      if URI(address(storage)).scheme == 'tls'
Ahmad Hassan's avatar
Ahmad Hassan committed
79
        GRPC::Core::ChannelCredentials.new stub_certs
80 81 82 83 84
      else
        :this_channel_is_insecure
      end
    end

85 86 87 88 89 90 91 92 93
    def self.stub_class(name)
      if name == :health_check
        Grpc::Health::V1::Health::Stub
      else
        Gitaly.const_get(name.to_s.camelcase.to_sym).const_get(:Stub)
      end
    end

    def self.stub_address(storage)
Ahmad Hassan's avatar
Ahmad Hassan committed
94
      address(storage).sub(%r{^tcp://|^tls://}, '')
95 96
    end

97 98 99 100
    def self.clear_stubs!
      MUTEX.synchronize do
        @stubs = nil
      end
101 102
    end

103 104 105 106
    def self.random_storage
      Gitlab.config.repositories.storages.keys.sample
    end

107 108 109
    def self.address(storage)
      params = Gitlab.config.repositories.storages[storage]
      raise "storage not found: #{storage.inspect}" if params.nil?
110

111 112 113 114
      address = params['gitaly_address']
      unless address.present?
        raise "storage #{storage.inspect} is missing a gitaly_address"
      end
115

116 117
      unless URI(address).scheme.in?(%w(tcp unix tls))
        raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'"
118 119
      end

120
      address
121 122
    end

123
    def self.address_metadata(storage)
124 125 126 127 128
      Base64.strict_encode64(JSON.dump(storage => connection_data(storage)))
    end

    def self.connection_data(storage)
      { 'address' => address(storage), 'token' => token(storage) }
129 130
    end

131 132
    # All Gitaly RPC call sites should use GitalyClient.call. This method
    # makes sure that per-request authentication headers are set.
133 134 135 136 137 138 139 140 141 142 143 144
    #
    # This method optionally takes a block which receives the keyword
    # arguments hash 'kwargs' that will be passed to gRPC. This allows the
    # caller to modify or augment the keyword arguments. The block must
    # return a hash.
    #
    # For example:
    #
    # GitalyClient.call(storage, service, rpc, request) do |kwargs|
    #   kwargs.merge(deadline: Time.now + 10)
    # end
    #
145
    def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
146
      start = Gitlab::Metrics::System.monotonic_time
147
      request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {}
148

149 150
      enforce_gitaly_request_limits(:call)

151
      kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage)
152
      kwargs = yield(kwargs) if block_given?
153

154
      stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
155
    ensure
156 157
      duration = Gitlab::Metrics::System.monotonic_time - start

George Tsiolis's avatar
George Tsiolis committed
158
      # Keep track, separately, for the performance bar
159
      self.query_time += duration
160
      if Gitlab::PerformanceBar.enabled_for_request?
161 162 163
        add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc,
                         backtrace: Gitlab::Profiler.clean_backtrace(caller))
      end
164 165
    end

166 167 168 169 170 171 172 173 174 175 176 177
    def self.query_time
      SafeRequestStore[:gitaly_query_time] ||= 0
    end

    def self.query_time=(duration)
      SafeRequestStore[:gitaly_query_time] = duration
    end

    def self.query_time_ms
      (self.query_time * 1000).round(2)
    end

178 179
    def self.current_transaction_labels
      Gitlab::Metrics::Transaction.current&.labels || {}
180
    end
181
    private_class_method :current_transaction_labels
182

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
    # For some time related tasks we can't rely on `Time.now` since it will be
    # affected by Timecop in some tests, and the clock of some gitaly-related
    # components (grpc's c-core and gitaly server) use system time instead of
    # timecop's time, so tests will fail.
    # `Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))` will circumvent
    # timecop.
    def self.real_time
      Time.at(Process.clock_gettime(Process::CLOCK_REALTIME))
    end
    private_class_method :real_time

    def self.authorization_token(storage)
      token = token(storage).to_s
      issued_at = real_time.to_i.to_s
      hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at)

      "v2.#{hmac}.#{issued_at}"
    end
    private_class_method :authorization_token

203
    def self.request_kwargs(storage, timeout, remote_storage: nil)
204
      metadata = {
205
        'authorization' => "Bearer #{authorization_token(storage)}",
206 207 208 209 210 211
        'client_name' => CLIENT_NAME
      }

      feature_stack = Thread.current[:gitaly_feature_stack]
      feature = feature_stack && feature_stack[0]
      metadata['call_site'] = feature.to_s if feature
212
      metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
213
      metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
214
      metadata['gitaly-session-id'] = session_id
215
      metadata.merge!(Feature::Gitaly.server_feature_flags)
216

217 218 219 220 221 222 223
      result = { metadata: metadata }

      # nil timeout indicates that we should use the default
      timeout = default_timeout if timeout.nil?

      return result unless timeout > 0

224
      deadline = real_time + timeout
225 226 227
      result[:deadline] = deadline

      result
228 229
    end

230 231 232
    def self.session_id
      Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid
    end
233

234 235 236 237 238 239 240
    def self.token(storage)
      params = Gitlab.config.repositories.storages[storage]
      raise "storage not found: #{storage.inspect}" if params.nil?

      params['gitaly_token'].presence || Gitlab.config.gitaly['token']
    end

241 242
    # Ensures that Gitaly is not being abuse through n+1 misuse etc
    def self.enforce_gitaly_request_limits(call_site)
243
      # Only count limits in request-response environments
244
      return unless Gitlab::SafeRequestStore.active?
245 246 247 248

      # This is this actual number of times this call was made. Used for information purposes only
      actual_call_count = increment_call_count("gitaly_#{call_site}_actual")

249
      return unless enforce_gitaly_request_limits?
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265

      # Check if this call is nested within a allow_n_plus_1_calls
      # block and skip check if it is
      return if get_call_count(:gitaly_call_count_exception_block_depth) > 0

      # This is the count of calls outside of a `allow_n_plus_1_calls` block
      # It is used for enforcement but not statistics
      permitted_call_count = increment_call_count("gitaly_#{call_site}_permitted")

      count_stack

      return if permitted_call_count <= MAXIMUM_GITALY_CALLS

      raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks)
    end

266 267 268 269 270 271 272
    def self.enforce_gitaly_request_limits?
      # We typically don't want to enforce request limits in production
      # However, we have some production-like test environments, i.e., ones
      # where `Rails.env.production?` returns `true`. We do want to be able to
      # check if the limit is being exceeded while testing in those environments
      # In that case we can use a feature flag to indicate that we do want to
      # enforce request limits.
273
      return true if Feature::Gitaly.enabled?('enforce_requests_limits')
274 275 276 277 278

      !(Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"])
    end
    private_class_method :enforce_gitaly_request_limits?

279
    def self.allow_n_plus_1_calls
280
      return yield unless Gitlab::SafeRequestStore.active?
281 282 283 284 285 286 287 288 289

      begin
        increment_call_count(:gitaly_call_count_exception_block_depth)
        yield
      ensure
        decrement_call_count(:gitaly_call_count_exception_block_depth)
      end
    end

290 291 292 293 294 295
    # Normally a FindCommit RPC will cache the commit with its SHA
    # instead of a ref name, since it's possible the branch is mutated
    # afterwards. However, for read-only requests that never mutate the
    # branch, this method allows caching of the ref name directly.
    def self.allow_ref_name_caching
      return yield unless Gitlab::SafeRequestStore.active?
296
      return yield if ref_name_caching_allowed?
297 298 299 300 301 302 303 304 305 306 307 308 309

      begin
        Gitlab::SafeRequestStore[:allow_ref_name_caching] = true
        yield
      ensure
        Gitlab::SafeRequestStore[:allow_ref_name_caching] = false
      end
    end

    def self.ref_name_caching_allowed?
      Gitlab::SafeRequestStore[:allow_ref_name_caching]
    end

310
    def self.get_call_count(key)
311
      Gitlab::SafeRequestStore[key] || 0
312 313 314 315
    end
    private_class_method :get_call_count

    def self.increment_call_count(key)
316 317
      Gitlab::SafeRequestStore[key] ||= 0
      Gitlab::SafeRequestStore[key] += 1
318 319 320 321
    end
    private_class_method :increment_call_count

    def self.decrement_call_count(key)
322
      Gitlab::SafeRequestStore[key] -= 1
323 324 325
    end
    private_class_method :decrement_call_count

326
    # Returns the of the number of Gitaly calls made for this request
327
    def self.get_request_count
328
      get_call_count("gitaly_call_actual")
329 330 331
    end

    def self.reset_counts
332
      return unless Gitlab::SafeRequestStore.active?
333

334 335
      Gitlab::SafeRequestStore["gitaly_call_actual"] = 0
      Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0
336
    end
337

338
    def self.add_call_details(details)
339 340
      Gitlab::SafeRequestStore['gitaly_call_details'] ||= []
      Gitlab::SafeRequestStore['gitaly_call_details'] << details
341 342
    end

343
    def self.list_call_details
344
      return [] unless Gitlab::PerformanceBar.enabled_for_request?
345

346
      Gitlab::SafeRequestStore['gitaly_call_details'] || []
347 348
    end

349 350 351 352
    def self.expected_server_version
      path = Rails.root.join(SERVER_VERSION_FILE)
      path.read.chomp
    end
353

354 355
    def self.timestamp(time)
      Google::Protobuf::Timestamp.new(seconds: time.to_i)
356 357
    end

358 359
    # The default timeout on all Gitaly calls
    def self.default_timeout
360
      return no_timeout if Sidekiq.server?
361 362 363 364 365 366 367 368 369 370 371 372

      timeout(:gitaly_timeout_default)
    end

    def self.fast_timeout
      timeout(:gitaly_timeout_fast)
    end

    def self.medium_timeout
      timeout(:gitaly_timeout_medium)
    end

373 374 375 376
    def self.no_timeout
      0
    end

377 378 379 380 381 382 383 384 385
    def self.storage_metadata_file_path(storage)
      Gitlab::GitalyClient::StorageSettings.allow_disk_access do
        File.join(
          Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME
        )
      end
    end

    def self.can_use_disk?(storage)
386 387 388 389
      cached_value = MUTEX.synchronize do
        @can_use_disk ||= {}
        @can_use_disk[storage]
      end
390

391
      return cached_value unless cached_value.nil?
392

393 394
      gitaly_filesystem_id = filesystem_id(storage)
      direct_filesystem_id = filesystem_id_from_disk(storage)
395

396 397 398 399
      MUTEX.synchronize do
        @can_use_disk[storage] = gitaly_filesystem_id.present? &&
          gitaly_filesystem_id == direct_filesystem_id
      end
400 401 402 403 404
    end

    def self.filesystem_id(storage)
      response = Gitlab::GitalyClient::ServerService.new(storage).info
      storage_status = response.storage_statuses.find { |status| status.storage_name == storage }
405 406

      storage_status&.filesystem_id
407 408 409 410 411 412
    end

    def self.filesystem_id_from_disk(storage)
      metadata_file = File.read(storage_metadata_file_path(storage))
      metadata_hash = JSON.parse(metadata_file)
      metadata_hash['gitaly_filesystem_id']
413
    rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
414 415 416
      nil
    end

417 418 419 420 421
    def self.timeout(timeout_name)
      Gitlab::CurrentSettings.current_application_settings[timeout_name]
    end
    private_class_method :timeout

422 423
    # Count a stack. Used for n+1 detection
    def self.count_stack
424
      return unless Gitlab::SafeRequestStore.active?
425

426
      stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n")
427

428
      Gitlab::SafeRequestStore[:stack_counter] ||= Hash.new
429

430 431
      count = Gitlab::SafeRequestStore[:stack_counter][stack_string] || 0
      Gitlab::SafeRequestStore[:stack_counter][stack_string] = count + 1
432 433 434 435 436
    end
    private_class_method :count_stack

    # Returns a count for the stack which called Gitaly the most times. Used for n+1 detection
    def self.max_call_count
437
      return 0 unless Gitlab::SafeRequestStore.active?
438

439
      stack_counter = Gitlab::SafeRequestStore[:stack_counter]
440 441 442 443 444 445 446 447
      return 0 unless stack_counter

      stack_counter.values.max
    end
    private_class_method :max_call_count

    # Returns the stacks that calls Gitaly the most times. Used for n+1 detection
    def self.max_stacks
448
      return unless Gitlab::SafeRequestStore.active?
449

450
      stack_counter = Gitlab::SafeRequestStore[:stack_counter]
451
      return unless stack_counter
452 453

      max = max_call_count
454
      return if max.zero?
455 456 457 458

      stack_counter.select { |_, v| v == max }.keys
    end
    private_class_method :max_stacks
459 460
  end
end