Merge branch 'mc/feature/custom-metrics-ce' into 'master'

Backport metrics report type

See merge request gitlab-org/gitlab-ce!26798

(cherry picked from commit c16619c5)

0cfa25ff Backport changes from EE

app/controllers/projects/merge_requests_controller.rb

@@ -98,20 +98,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
   end
 
   def test_reports
-    result = @merge_request.compare_test_reports
-
-    case result[:status]
-    when :parsing
-      Gitlab::PollingInterval.set_header(response, interval: 3000)
-
-      render json: '', status: :no_content
-    when :parsed
-      render json: result[:data].to_json, status: :ok
-    when :error
-      render json: { status_reason: result[:status_reason] }, status: :bad_request
-    else
-      render json: { status_reason: 'Unknown error' }, status: :internal_server_error
-    end
+    reports_response(@merge_request.compare_test_reports)
   end
 
   def edit
@@ -353,4 +340,19 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
     # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42441
     Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42438')
   end
+
+  def reports_response(report_comparison)
+    case report_comparison[:status]
+    when :parsing
+      ::Gitlab::PollingInterval.set_header(response, interval: 3000)
+
+      render json: '', status: :no_content
+    when :parsed
+      render json: report_comparison[:data].to_json, status: :ok
+    when :error
+      render json: { status_reason: report_comparison[:status_reason] }, status: :bad_request
+    else
+      render json: { status_reason: 'Unknown error' }, status: :internal_server_error
+    end
+  end
 end

app/models/ci/build.rb

@@ -104,8 +104,8 @@ module Ci
       where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace)
     end
 
-    scope :with_test_reports, ->() do
-      with_existing_job_artifacts(Ci::JobArtifact.test_reports)
+    scope :with_reports, ->(reports_scope) do
+      with_existing_job_artifacts(reports_scope)
         .eager_load_job_artifacts
     end
 

app/models/ci/job_artifact.rb

@@ -21,7 +21,8 @@ module Ci
       container_scanning: 'gl-container-scanning-report.json',
       dast: 'gl-dast-report.json',
       license_management: 'gl-license-management-report.json',
-      performance: 'performance.json'
+      performance: 'performance.json',
+      metrics: 'metrics.txt'
     }.freeze
 
     TYPE_AND_FORMAT_PAIRS = {
@@ -29,6 +30,7 @@ module Ci
       metadata: :gzip,
       trace: :raw,
       junit: :gzip,
+      metrics: :gzip,
 
       # All these file formats use `raw` as we need to store them uncompressed
       # for Frontend to fetch the files and do analysis
@@ -88,7 +90,8 @@ module Ci
       dast: 8, ## EE-specific
       codequality: 9, ## EE-specific
       license_management: 10, ## EE-specific
-      performance: 11 ## EE-specific
+      performance: 11, ## EE-specific
+      metrics: 12 ## EE-specific
     }
 
     enum file_format: {

app/models/ci/pipeline.rb

@@ -210,6 +210,10 @@ module Ci
       where(source: branch_pipeline_sources).where(ref: ref, tag: false)
     end
 
+    scope :with_reports, -> (reports_scope) do
+      where('EXISTS (?)', ::Ci::Build.latest.with_reports(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1))
+    end
+
     # Returns the pipelines in descending order (= newest first), optionally
     # limited to a number of references.
     #
@@ -689,13 +693,13 @@ module Ci
       @latest_builds_with_artifacts ||= builds.latest.with_artifacts_archive.to_a
     end
 
-    def has_test_reports?
-      complete? && builds.latest.with_test_reports.any?
+    def has_reports?(reports_scope)
+      complete? && builds.latest.with_reports(reports_scope).exists?
     end
 
     def test_reports
       Gitlab::Ci::Reports::TestReports.new.tap do |test_reports|
-        builds.latest.with_test_reports.each do |build|
+        builds.latest.with_reports(Ci::JobArtifact.test_reports).each do |build|
           build.collect_test_reports!(test_reports)
         end
       end

app/models/merge_request.rb

@@ -1195,7 +1195,7 @@ class MergeRequest < ApplicationRecord
   end
 
   def has_test_reports?
-    actual_head_pipeline&.has_test_reports?
+    actual_head_pipeline&.has_reports?(Ci::JobArtifact.test_reports)
   end
 
   def predefined_variables

lib/gitlab/ci/config/entry/reports.rb

@@ -11,7 +11,7 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management].freeze
+          ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management metrics].freeze
 
           attributes ALLOWED_KEYS
 
@@ -28,6 +28,7 @@ module Gitlab
               validates :dast, array_of_strings_or_string: true
               validates :performance, array_of_strings_or_string: true
               validates :license_management, array_of_strings_or_string: true
+              validates :metrics, array_of_strings_or_string: true
             end
           end
 

spec/models/ci/build_spec.rb

@@ -166,8 +166,8 @@ describe Ci::Build do
     end
   end
 
-  describe '.with_test_reports' do
-    subject { described_class.with_test_reports }
+  describe '.with_reports' do
+    subject { described_class.with_reports(Ci::JobArtifact.test_reports) }
 
     context 'when build has a test report' do
       let!(:build) { create(:ci_build, :success, :test_reports) }

spec/models/ci/pipeline_spec.rb

@@ -426,6 +426,26 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '.with_reports' do
+    subject { described_class.with_reports(Ci::JobArtifact.test_reports) }
+
+    context 'when pipeline has a test report' do
+      let!(:pipeline_with_report) { create(:ci_pipeline, :with_test_reports) }
+
+      it 'selects the pipeline' do
+        is_expected.to eq([pipeline_with_report])
+      end
+    end
+
+    context 'when pipeline does not have metrics reports' do
+      let!(:pipeline_without_report) { create(:ci_empty_pipeline) }
+
+      it 'does not select the pipeline' do
+        is_expected.to be_empty
+      end
+    end
+  end
+
   describe '.merge_request_event' do
     subject { described_class.merge_request_event }
 
@@ -2728,8 +2748,8 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
-  describe '#has_test_reports?' do
-    subject { pipeline.has_test_reports? }
+  describe '#has_reports?' do
+    subject { pipeline.has_reports?(Ci::JobArtifact.test_reports) }
 
     context 'when pipeline has builds with test reports' do
       before do

spec/services/ci/retry_build_service_spec.rb

@@ -28,7 +28,7 @@ describe Ci::RetryBuildService do
        job_artifacts_sast job_artifacts_dependency_scanning
        job_artifacts_container_scanning job_artifacts_dast
        job_artifacts_license_management job_artifacts_performance
-       job_artifacts_codequality scheduled_at].freeze
+       job_artifacts_codequality job_artifacts_metrics scheduled_at].freeze
 
   IGNORE_ACCESSORS =
     %i[type lock_version target_url base_tags trace_sections