Merge branch 'runner-token-authentication-prometheus-metric' into 'master'

Add metrics for runner authentication success/failure

See merge request !91420

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -42,6 +42,8 @@ The following metrics are available:
 | `gitlab_ci_pipeline_builder_scoped_variables_duration`           | Histogram   | 14.5   | Time in seconds it takes to create the scoped variables for a CI/CD job
 | `gitlab_ci_pipeline_creation_duration_seconds`                   | Histogram   | 13.0    | Time in seconds it takes to create a CI/CD pipeline                                                                   |                                                           |
 | `gitlab_ci_pipeline_size_builds`                                 | Histogram   | 13.1    | Total number of builds within a pipeline grouped by a pipeline source                                                 | `source`                                                  |
+| `gitlab_ci_runner_authentication_success_total`                  | Counter     | 15.2    | Total number of times that runner authentication has succeeded                                                        | `type`                                                    |
+| `gitlab_ci_runner_authentication_failure_total`                  | Counter     | 15.2    | Total number of times that runner authentication has failed
 | `job_waiter_started_total`                                       | Counter     | 12.9    | Number of batches of jobs started where a web request is waiting for the jobs to complete                             | `worker`                                                  |
 | `job_waiter_timeouts_total`                                      | Counter     | 12.9    | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete                      | `worker`                                                  |
 | `gitlab_ci_active_jobs`                                          | Histogram   | 14.2    | Count of active jobs when pipeline is created                                                                         |                                                           |

lib/api/ci/helpers/runner.rb

@@ -12,6 +12,7 @@ module Runner
         JOB_TOKEN_PARAM = :token
 
         def authenticate_runner!
+          track_runner_authentication
           forbidden! unless current_runner
 
           current_runner
@@ -42,6 +43,14 @@ def current_runner
           end
         end
 
+        def track_runner_authentication
+          if current_runner
+            metrics.increment_runner_authentication_success_counter(runner_type: current_runner.runner_type)
+          else
+            metrics.increment_runner_authentication_failure_counter
+          end
+        end
+
         # HTTP status codes to terminate the job on GitLab Runner:
         # - 403
         def authenticate_job!(require_running: true, heartbeat_runner: false)
@@ -149,6 +158,10 @@ def get_runner_config_from_request
         def request_using_running_job_token?
           current_job.present? && current_authenticated_job.present? && current_job != current_authenticated_job
         end
+
+        def metrics
+          strong_memoize(:metrics) { ::Gitlab::Ci::Runner::Metrics.new }
+        end
       end
     end
   end

lib/gitlab/ci/runner/metrics.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Runner
+      class Metrics
+        extend Gitlab::Utils::StrongMemoize
+
+        def increment_runner_authentication_success_counter(runner_type: 'unknown_type')
+          raise ArgumentError, "unknown runner type: #{runner_type}" unless
+            ::Ci::Runner.runner_types.include? runner_type
+
+          self.class.runner_authentication_success_counter.increment(runner_type: runner_type)
+        end
+
+        def increment_runner_authentication_failure_counter
+          self.class.runner_authentication_failure_counter.increment
+        end
+
+        def self.runner_authentication_success_counter
+          strong_memoize(:runner_authentication_success) do
+            name = :gitlab_ci_runner_authentication_success_total
+            comment = 'Runner authentication success'
+            labels = { runner_type: nil }
+
+            ::Gitlab::Metrics.counter(name, comment, labels)
+          end
+        end
+
+        def self.runner_authentication_failure_counter
+          strong_memoize(:runner_authentication_failure) do
+            name = :gitlab_ci_runner_authentication_failure_total
+            comment = 'Runner authentication failure'
+
+            ::Gitlab::Metrics.counter(name, comment)
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/api/ci/helpers/runner_spec.rb

@@ -66,4 +66,30 @@
       expect(helper.current_runner).to eq(runner)
     end
   end
+
+  describe '#track_runner_authentication', :prometheus do
+    subject { helper.track_runner_authentication }
+
+    let(:runner) { create(:ci_runner, token: 'foo') }
+
+    it 'increments gitlab_ci_runner_authentication_success_total' do
+      allow(helper).to receive(:params).and_return(token: runner.token)
+
+      success_counter = ::Gitlab::Ci::Runner::Metrics.runner_authentication_success_counter
+      failure_counter = ::Gitlab::Ci::Runner::Metrics.runner_authentication_failure_counter
+      expect { subject }.to change { success_counter.get(runner_type: 'instance_type') }.by(1)
+        .and not_change { success_counter.get(runner_type: 'project_type') }
+        .and not_change { failure_counter.get }
+    end
+
+    it 'increments gitlab_ci_runner_authentication_failure_total' do
+      allow(helper).to receive(:params).and_return(token: 'invalid')
+
+      success_counter = ::Gitlab::Ci::Runner::Metrics.runner_authentication_success_counter
+      failure_counter = ::Gitlab::Ci::Runner::Metrics.runner_authentication_failure_counter
+      expect { subject }.to change { failure_counter.get }.by(1)
+        .and not_change { success_counter.get(runner_type: 'instance_type') }
+        .and not_change { success_counter.get(runner_type: 'project_type') }
+    end
+  end
 end

spec/lib/gitlab/ci/runner/metrics_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Runner::Metrics, :prometheus do
+  subject { described_class.new }
+
+  describe '#increment_runner_authentication_success_counter' do
+    it 'increments count for same type' do
+      expect { subject.increment_runner_authentication_success_counter(runner_type: 'instance_type') }
+        .to change { described_class.runner_authentication_success_counter.get(runner_type: 'instance_type') }.by(1)
+    end
+
+    it 'does not increment count for different type' do
+      expect { subject.increment_runner_authentication_success_counter(runner_type: 'group_type') }
+        .to not_change { described_class.runner_authentication_success_counter.get(runner_type: 'project_type') }
+    end
+
+    it 'does not increment failure count' do
+      expect { subject.increment_runner_authentication_success_counter(runner_type: 'project_type') }
+        .to not_change { described_class.runner_authentication_failure_counter.get }
+    end
+
+    it 'throws ArgumentError for invalid runner type' do
+      expect { subject.increment_runner_authentication_success_counter(runner_type: 'unknown_type') }
+        .to raise_error(ArgumentError, 'unknown runner type: unknown_type')
+    end
+  end
+
+  describe '#increment_runner_authentication_failure_counter' do
+    it 'increments count' do
+      expect { subject.increment_runner_authentication_failure_counter }
+        .to change { described_class.runner_authentication_failure_counter.get }.by(1)
+    end
+
+    it 'does not increment success count' do
+      expect { subject.increment_runner_authentication_failure_counter }
+        .to not_change { described_class.runner_authentication_success_counter.get(runner_type: 'instance_type') }
+    end
+  end
+end