Commit b75e92a1 authored by Sean McGivern's avatar Sean McGivern

Merge branch '59974-multiple-dashboards-be' into 'master'

Resolve BE for "Load dashboards from project's git repository"

Closes #59974

See merge request gitlab-org/gitlab-ce!27608
parents d7b75b66 552a3d2f
Pipeline #59262176 passed with stages
in 57 minutes and 7 seconds
......@@ -13,6 +13,7 @@ class Projects::EnvironmentsController < Projects::ApplicationController
before_action only: [:metrics, :additional_metrics, :metrics_dashboard] do
push_frontend_feature_flag(:metrics_time_window)
push_frontend_feature_flag(:environment_metrics_use_prometheus_endpoint)
push_frontend_feature_flag(:environment_metrics_show_multiple_dashboards)
end
def index
......@@ -158,15 +159,28 @@ class Projects::EnvironmentsController < Projects::ApplicationController
end
def metrics_dashboard
return render_403 unless Feature.enabled?(:environment_metrics_use_prometheus_endpoint, @project)
return render_403 unless Feature.enabled?(:environment_metrics_use_prometheus_endpoint, project)
result = Gitlab::Metrics::Dashboard::Service.new(@project, @current_user, environment: environment).get_dashboard
if Feature.enabled?(:environment_metrics_show_multiple_dashboards, project)
result = dashboard_finder.find(project, current_user, environment, params[:dashboard])
result[:all_dashboards] = project.repository.metrics_dashboard_paths
else
result = dashboard_finder.find(project, current_user, environment)
end
respond_to do |format|
if result[:status] == :success
format.json { render status: :ok, json: result }
format.json do
render status: :ok, json: result.slice(:all_dashboards, :dashboard, :status)
end
else
format.json { render status: result[:http_status], json: result }
format.json do
render(
status: result[:http_status],
json: result.slice(:all_dashboards, :message, :status)
)
end
end
end
end
......@@ -211,6 +225,10 @@ class Projects::EnvironmentsController < Projects::ApplicationController
params.require([:start, :end])
end
def dashboard_finder
Gitlab::Metrics::Dashboard::Finder
end
def search_environment_names
return [] unless params[:query]
......
......@@ -39,7 +39,8 @@ class Repository
changelog license_blob license_key gitignore
gitlab_ci_yml branch_names tag_names branch_count
tag_count avatar exists? root_ref has_visible_content?
issue_template_names merge_request_template_names xcode_project?).freeze
issue_template_names merge_request_template_names
metrics_dashboard_paths xcode_project?).freeze
# Methods that use cache_method but only memoize the value
MEMOIZED_CACHED_METHODS = %i(license).freeze
......@@ -57,6 +58,7 @@ class Repository
avatar: :avatar,
issue_template: :issue_template_names,
merge_request_template: :merge_request_template_names,
metrics_dashboard: :metrics_dashboard_paths,
xcode_config: :xcode_project?
}.freeze
......@@ -602,6 +604,11 @@ class Repository
end
cache_method :merge_request_template_names, fallback: []
def metrics_dashboard_paths
Gitlab::Metrics::Dashboard::Finder.find_all_paths_from_source(project)
end
cache_method :metrics_dashboard_paths
def readme
head_tree&.readme
end
......
......@@ -16,6 +16,7 @@ module Gitlab
avatar: /\Alogo\.(png|jpg|gif)\z/,
issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z},
merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z},
metrics_dashboard: %r{\A\.gitlab/dashboards/[^/]+\.yml\z},
xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)(/.+)?\z},
# Configuration files
......
# frozen_string_literal: true
# Searches a projects repository for a metrics dashboard and formats the output.
# Expects any custom dashboards will be located in `.gitlab/dashboards`
module Gitlab
module Metrics
module Dashboard
class BaseService < ::BaseService
DASHBOARD_LAYOUT_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError
def get_dashboard
return error("#{dashboard_path} could not be found.", :not_found) unless path_available?
success(dashboard: process_dashboard)
rescue DASHBOARD_LAYOUT_ERROR => e
error(e.message, :unprocessable_entity)
end
# Summary of all known dashboards for the service.
# @return [Array<Hash>] ex) [{ path: String, default: Boolean }]
def all_dashboard_paths(_project)
raise NotImplementedError
end
private
# Returns a new dashboard Hash, supplemented with DB info
def process_dashboard
Gitlab::Metrics::Dashboard::Processor
.new(project, params[:environment], raw_dashboard)
.process(insert_project_metrics: insert_project_metrics?)
end
# @return [String] Relative filepath of the dashboard yml
def dashboard_path
params[:dashboard_path]
end
# Returns an un-processed dashboard from the cache.
def raw_dashboard
Rails.cache.fetch(cache_key) { get_raw_dashboard }
end
# @return [Hash] an unmodified dashboard
def get_raw_dashboard
raise NotImplementedError
end
# @return [String]
def cache_key
raise NotImplementedError
end
# Determines whether custom metrics should be included
# in the processed output.
def insert_project_metrics?
false
end
# Checks if dashboard path exists or should be rejected
# as a result of file-changes to the project repository.
# @return [Boolean]
def path_available?
available_paths = Gitlab::Metrics::Dashboard::Finder.find_all_paths(project)
available_paths.any? do |path_params|
path_params[:path] == dashboard_path
end
end
end
end
end
end
# frozen_string_literal: true
# Returns DB-supplmented dashboard info for determining
# the layout of UI. Intended entry-point for the Metrics::Dashboard
# module.
module Gitlab
module Metrics
module Dashboard
class Finder
class << self
# Returns a formatted dashboard packed with DB info.
# @return [Hash]
def find(project, user, environment, dashboard_path = nil)
service = system_dashboard?(dashboard_path) ? system_service : project_service
service
.new(project, user, environment: environment, dashboard_path: dashboard_path)
.get_dashboard
end
# Summary of all known dashboards.
# @return [Array<Hash>] ex) [{ path: String, default: Boolean }]
def find_all_paths(project)
project.repository.metrics_dashboard_paths
end
# Summary of all known dashboards. Used to populate repo cache.
# Prefer #find_all_paths.
def find_all_paths_from_source(project)
system_service.all_dashboard_paths(project)
.+ project_service.all_dashboard_paths(project)
end
private
def system_service
Gitlab::Metrics::Dashboard::SystemDashboardService
end
def project_service
Gitlab::Metrics::Dashboard::ProjectDashboardService
end
def system_dashboard?(filepath)
!filepath || system_service.system_dashboard?(filepath)
end
end
end
end
end
end
......@@ -8,12 +8,17 @@ module Gitlab
# the UI. These includes shared metric info, custom metrics
# info, and alerts (only in EE).
class Processor
SEQUENCE = [
SYSTEM_SEQUENCE = [
Stages::CommonMetricsInserter,
Stages::ProjectMetricsInserter,
Stages::Sorter
].freeze
PROJECT_SEQUENCE = [
Stages::CommonMetricsInserter,
Stages::Sorter
].freeze
def initialize(project, environment, dashboard)
@project = project
@environment = environment
......@@ -22,9 +27,9 @@ module Gitlab
# Returns a new dashboard hash with the results of
# running transforms on the dashboard.
def process
def process(insert_project_metrics:)
@dashboard.deep_symbolize_keys.tap do |dashboard|
sequence.each do |stage|
sequence(insert_project_metrics).each do |stage|
stage.new(@project, @environment, dashboard).transform!
end
end
......@@ -32,8 +37,8 @@ module Gitlab
private
def sequence
SEQUENCE
def sequence(insert_project_metrics)
insert_project_metrics ? SYSTEM_SEQUENCE : PROJECT_SEQUENCE
end
end
end
......
# frozen_string_literal: true
# Searches a projects repository for a metrics dashboard and formats the output.
# Expects any custom dashboards will be located in `.gitlab/dashboards`
# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
module Gitlab
module Metrics
module Dashboard
class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService
DASHBOARD_ROOT = ".gitlab/dashboards"
class << self
def all_dashboard_paths(project)
file_finder(project)
.list_files_for(DASHBOARD_ROOT)
.map do |filepath|
Rails.cache.delete(cache_key(project.id, filepath))
{ path: filepath, default: false }
end
end
def file_finder(project)
Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml')
end
def cache_key(id, dashboard_path)
"project_#{id}_metrics_dashboard_#{dashboard_path}"
end
end
private
# Searches the project repo for a custom-defined dashboard.
def get_raw_dashboard
yml = self.class.file_finder(project).read(dashboard_path)
YAML.safe_load(yml)
end
def cache_key
self.class.cache_key(project.id, dashboard_path)
end
end
end
end
end
# frozen_string_literal: true
# Fetches the metrics dashboard layout and supplemented the output with DB info.
module Gitlab
module Metrics
module Dashboard
class Service < ::BaseService
SYSTEM_DASHBOARD_NAME = 'common_metrics'
SYSTEM_DASHBOARD_PATH = Rails.root.join('config', 'prometheus', "#{SYSTEM_DASHBOARD_NAME}.yml")
# Returns a DB-supplemented json representation of a dashboard config file.
def get_dashboard
dashboard_string = Rails.cache.fetch(cache_key) { system_dashboard }
dashboard = process_dashboard(dashboard_string)
success(dashboard: dashboard)
rescue Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError => e
error(e.message, :unprocessable_entity)
end
private
# Returns the base metrics shipped with every GitLab service.
def system_dashboard
YAML.safe_load(File.read(SYSTEM_DASHBOARD_PATH))
end
def cache_key
"metrics_dashboard_#{SYSTEM_DASHBOARD_NAME}"
end
# Returns a new dashboard Hash, supplemented with DB info
def process_dashboard(dashboard)
Gitlab::Metrics::Dashboard::Processor.new(project, params[:environment], dashboard).process
end
end
end
end
end
......@@ -36,7 +36,7 @@ module Gitlab
raise DashboardLayoutError.new('Each "panel" must define an array :metrics')
end
def for_metrics(dashboard)
def for_metrics
missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array)
dashboard[:panel_groups].each do |panel_group|
......
......@@ -11,7 +11,7 @@ module Gitlab
def transform!
common_metrics = ::PrometheusMetric.common
for_metrics(dashboard) do |metric|
for_metrics do |metric|
metric_record = common_metrics.find { |m| m.identifier == metric[:id] }
metric[:metric_id] = metric_record.id if metric_record
end
......
# frozen_string_literal: true
# Fetches the system metrics dashboard and formats the output.
# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
module Gitlab
module Metrics
module Dashboard
class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService
SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml'
class << self
def all_dashboard_paths(_project)
[{
path: SYSTEM_DASHBOARD_PATH,
default: true
}]
end
def system_dashboard?(filepath)
filepath == SYSTEM_DASHBOARD_PATH
end
end
private
def dashboard_path
SYSTEM_DASHBOARD_PATH
end
# Returns the base metrics shipped with every GitLab service.
def get_raw_dashboard
yml = File.read(Rails.root.join(dashboard_path))
YAML.safe_load(yml)
end
def cache_key
"metrics_dashboard_#{dashboard_path}"
end
def insert_project_metrics?
true
end
end
end
end
end
......@@ -474,25 +474,102 @@ describe Projects::EnvironmentsController do
end
end
context 'when prometheus endpoint is enabled' do
shared_examples_for '200 response' do |contains_all_dashboards: false|
let(:expected_keys) { %w(dashboard status) }
before do
expected_keys << 'all_dashboards' if contains_all_dashboards
end
it 'returns a json representation of the environment dashboard' do
get :metrics_dashboard, params: environment_params(format: :json)
get :metrics_dashboard, params: environment_params(dashboard_params)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.keys).to contain_exactly('dashboard', 'status')
expect(json_response.keys).to contain_exactly(*expected_keys)
expect(json_response['dashboard']).to be_an_instance_of(Hash)
end
end
shared_examples_for 'error response' do |status_code, contains_all_dashboards: false|
let(:expected_keys) { %w(message status) }
before do
expected_keys << 'all_dashboards' if contains_all_dashboards
end
it 'returns an error response' do
get :metrics_dashboard, params: environment_params(dashboard_params)
expect(response).to have_gitlab_http_status(status_code)
expect(json_response.keys).to contain_exactly(*expected_keys)
end
end
shared_examples_for 'has all dashboards' do
it 'includes an index of all available dashboards' do
get :metrics_dashboard, params: environment_params(dashboard_params)
expect(json_response.keys).to include('all_dashboards')
expect(json_response['all_dashboards']).to be_an_instance_of(Array)
expect(json_response['all_dashboards']).to all( include('path', 'default') )
end
end
context 'when multiple dashboards is disabled' do
before do
stub_feature_flags(environment_metrics_show_multiple_dashboards: false)
end
let(:dashboard_params) { { format: :json } }
it_behaves_like '200 response'
context 'when the dashboard could not be provided' do
before do
allow(YAML).to receive(:safe_load).and_return({})
end
it 'returns an error response' do
get :metrics_dashboard, params: environment_params(format: :json)
it_behaves_like 'error response', :unprocessable_entity
end
context 'when a dashboard param is specified' do
let(:dashboard_params) { { format: :json, dashboard: '.gitlab/dashboards/not_there_dashboard.yml' } }
it_behaves_like '200 response'
end
end
context 'when multiple dashboards is enabled' do
let(:dashboard_params) { { format: :json } }
it_behaves_like '200 response', contains_all_dashboards: true
it_behaves_like 'has all dashboards'
context 'when a dashboard could not be provided' do
before do
allow(YAML).to receive(:safe_load).and_return({})
end
it_behaves_like 'error response', :unprocessable_entity, contains_all_dashboards: true
it_behaves_like 'has all dashboards'
end
context 'when a dashboard param is specified' do
let(:dashboard_params) { { format: :json, dashboard: '.gitlab/dashboards/test.yml' } }
context 'when the dashboard is available' do
let(:dashboard_yml) { fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
let(:dashboard_file) { { '.gitlab/dashboards/test.yml' => dashboard_yml } }
let(:project) { create(:project, :custom_repo, files: dashboard_file) }
let(:environment) { create(:environment, name: 'production', project: project) }
it_behaves_like '200 response', contains_all_dashboards: true
it_behaves_like 'has all dashboards'
end
expect(response).to have_gitlab_http_status(:unprocessable_entity)
expect(json_response.keys).to contain_exactly('message', 'status', 'http_status')
context 'when the dashboard does not exist' do
it_behaves_like 'error response', :not_found, contains_all_dashboards: true
it_behaves_like 'has all dashboards'
end
end
end
......
......@@ -2,7 +2,7 @@ dashboard: 'Test Dashboard'
priority: 1
panel_groups:
- group: Group A
priority: 10
priority: 1
panels:
- title: "Super Chart A1"
type: "area-chart"
......@@ -23,7 +23,7 @@ panel_groups:
label: Legend Label
unit: unit
- group: Group B
priority: 1
priority: 10
panels:
- title: "Super Chart B"
type: "area-chart"
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Metrics::Dashboard::Finder, :use_clean_rails_memory_store_caching do
include MetricsDashboardHelpers
set(:project) { build(:project) }
set(:environment) { build(:environment, project: project) }
let(:system_dashboard_path) { Gitlab::Metrics::Dashboard::SystemDashboardService::SYSTEM_DASHBOARD_PATH}
describe '.find' do
let(:dashboard_path) { '.gitlab/dashboards/test.yml' }
let(:service_call) { described_class.find(project, nil, environment, dashboard_path) }
it_behaves_like 'misconfigured dashboard service response', :not_found
context 'when the dashboard exists' do
let(:project) { project_with_dashboard(dashboard_path) }
it_behaves_like 'valid dashboard service response'
end
context 'when the dashboard is configured incorrectly' do
let(:project) { project_with_dashboard(dashboard_path, {}) }
it_behaves_like 'misconfigured dashboard service response', :unprocessable_entity
end
context 'when the system dashboard is specified' do
let(:dashboard_path) { system_dashboard_path }
it_behaves_like 'valid dashboard service response'
end
context 'when no dashboard is specified' do
let(:service_call) { described_class.find(project, nil, environment) }
it_behaves_like 'valid dashboard service response'
end
end
describe '.find_all_paths' do
let(:all_dashboard_paths) { described_class.find_all_paths(project) }
let(:system_dashboard) { { path: system_dashboard_path, default: true } }
it 'includes only the system dashboard by default' do
expect(all_dashboard_paths).to eq([system_dashboard])
end
context 'when the project contains dashboards' do
let(:dashboard_path) { '.gitlab/dashboards/test.yml' }
let(:project) { project_with_dashboard(dashboard_path) }
it 'includes system and project dashboards' do
project_dashboard = { path: dashboard_path, default: false }
expect(all_dashboard_paths).to contain_exactly(system_dashboard, project_dashboard)
end
end
end
end
......@@ -4,12 +4,12 @@ require 'spec_helper'
describe Gitlab::Metrics::Dashboard::Processor do
let(:project) { build(:project) }
let(:environment) { build(:environment) }
let(:environment) { build(:environment, project: project) }
let(:dashboard_yml) { YAML.load_file('spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
describe 'process' do
let(:process_params) { [project, environment, dashboard_yml] }
let(:dashboard) { described_class.new(*process_params).process }
let(:dashboard) { described_class.new(*process_params).process(insert_project_metrics: true) }
context 'when dashboard config corresponds to common metrics' do
let!(:common_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') }
......@@ -35,9 +35,9 @@ describe Gitlab::Metrics::Dashboard::Processor do
it 'orders groups by priority and panels by weight' do
expected_metrics_order = [
'metric_a2', # group priority 10, panel weight 2
'metric_a1', # group priority 10, panel weight 1
'metric_b', # group priority 1, panel weight 1
'metric_b', # group priority 10, panel weight 1
'metric_a2', # group priority 1, panel weight 2
'metric_a1', # group priority 1, panel weight 1
project_business_metric.id, # group priority 0, panel weight nil (0)
project_response_metric.id, # group priority -5, panel weight nil (0)
project_system_metric.id, # group priority -10, panel weight nil (0)
......@@ -46,6 +46,17 @@ describe Gitlab::Metrics::Dashboard::Processor do
expect(actual_metrics_order).to eq expected_metrics_order
end
context 'when the dashboard should not include project metrics' do
let(:dashboard) { described_class.new(*process_params).process(insert_project_metrics: false) }
it 'includes only dashboard metrics' do
metrics = all_metrics.map { |m| m[:id] }
expect(metrics.length).to be(3)
expect(metrics).to eq %w(metric_b metric_a2 metric_a1)
end
end
end
shared_examples_for 'errors with message' do |expected_message|
......
# frozen_string_literal: true
require 'rails_helper'
describe Gitlab::Metrics::Dashboard::ProjectDashboardService, :use_clean_rails_memory_store_caching do
include MetricsDashboardHelpers
set(:user) { build(:user) }
set(:project) { build(:project) }
set(:environment) { build(:environment, project: project) }
before do
project.add_maintainer(user)
end
describe 'get_dashboard' do
let(:dashboard_path) { '.gitlab/dashboards/test.yml' }
let(:service_params) { [project, user, { environment: environment, dashboard_path: dashboard_path }] }
let(:service_call) { described_class.new(*service_params).get_dashboard }
context 'when the dashboard does not exist' do
it_behaves_like 'misconfigured dashboard service response', :not_found
end
context 'when the dashboard exists' do
let(:project) { project_with_dashboard(dashboard_path) }
it_behaves_like 'valid dashboard service response'
it 'caches the unprocessed dashboard for subsequent calls' do
expect_any_instance_of(described_class)
.to receive(:get_raw_dashboard)
.once
.and_call_original
described_class.new(*service_params).get_dashboard
described_class.new(*service_params).get_dashboard
end
context 'and the dashboard is then deleted' do
it 'does not return the previously cached dashboard' do
described_class.new(*service_params).get_dashboard
delete_project_dashboard(project, user, dashboard_path)