Commit 5c17bc70 authored by Stan Hu's avatar Stan Hu

Merge branch '32358-add-modsec-blocking-usage-stats-per-project' into 'master'

Implement usage metrics for ModSecurity Web Application Firewall

See merge request !20196
parents bef91563 72cc34c9
Pipeline #100717417 passed with stages
in 159 minutes and 58 seconds
# frozen_string_literal: true
# rubocop: disable CodeReuse/ActiveRecord
module Clusters
module Applications
##
# This service measures usage of the Modsecurity Web Application Firewall across the entire
# instance's deployed environments.
#
# The default configuration is`AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE=DetectionOnly` so we
# measure non-default values via definition of either ci_variables or ci_pipeline_variables.
# Since both these values are encrypted, we must decrypt and count them in memory.
#
# NOTE: this service is an approximation as it does not yet take into account `environment_scope` or `ci_group_variables`.
##
class IngressModsecurityUsageService
ADO_MODSEC_KEY = "AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE"
def initialize(blocking_count: 0, disabled_count: 0)
@blocking_count = blocking_count
@disabled_count = disabled_count
end
def execute
conditions = -> { merge(::Environment.available).merge(::Deployment.success).where(key: ADO_MODSEC_KEY) }
ci_pipeline_var_enabled =
::Ci::PipelineVariable
.joins(pipeline: { environments: :last_visible_deployment })
.merge(conditions)
.order('deployments.environment_id, deployments.id DESC')
ci_var_enabled =
::Ci::Variable
.joins(project: { environments: :last_visible_deployment })
.merge(conditions)
.merge(
# Give priority to pipeline variables by excluding from dataset
::Ci::Variable.joins(project: :environments).where.not(
environments: { id: ci_pipeline_var_enabled.select('DISTINCT ON (deployments.environment_id) deployments.environment_id') }
)
).select('DISTINCT ON (deployments.environment_id) ci_variables.*')
sum_modsec_config_counts(
ci_pipeline_var_enabled.select('DISTINCT ON (deployments.environment_id) ci_pipeline_variables.*')
)
sum_modsec_config_counts(ci_var_enabled)
{
ingress_modsecurity_blocking: @blocking_count,
ingress_modsecurity_disabled: @disabled_count
}
end
private
# These are encrypted so we must decrypt and count in memory
def sum_modsec_config_counts(dataset)
dataset.each do |var|
case var.value
when "On" then @blocking_count += 1
when "Off" then @disabled_count += 1
# `else` could be default or any unsupported user input
end
end
end
end
end
end
---
title: Add modsecurity deployment counts to usage ping
merge_request: 20196
author:
type: added
# frozen_string_literal: true
class AddIndexToModSecCiVariables < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :ci_variables, :project_id, where: "key = 'AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE'"
end
def down
remove_concurrent_index :ci_variables, :project_id
end
end
# frozen_string_literal: true
class AddIndexToModSecCiPipelineVariables < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :ci_pipeline_variables, :pipeline_id, where: "key = 'AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE'"
end
def down
remove_concurrent_index :ci_pipeline_variables, :pipeline_id
end
end
......@@ -821,6 +821,7 @@ ActiveRecord::Schema.define(version: 2019_11_25_140458) do
t.integer "pipeline_id", null: false
t.integer "variable_type", limit: 2, default: 1, null: false
t.index ["pipeline_id", "key"], name: "index_ci_pipeline_variables_on_pipeline_id_and_key", unique: true
t.index ["pipeline_id"], name: "index_ci_pipeline_variables_on_pipeline_id", where: "((key)::text = 'AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE'::text)"
end
create_table "ci_pipelines", id: :serial, force: :cascade do |t|
......@@ -979,6 +980,7 @@ ActiveRecord::Schema.define(version: 2019_11_25_140458) do
t.boolean "masked", default: false, null: false
t.integer "variable_type", limit: 2, default: 1, null: false
t.index ["project_id", "key", "environment_scope"], name: "index_ci_variables_on_project_id_and_key_and_environment_scope", unique: true
t.index ["project_id"], name: "index_ci_variables_on_project_id", where: "((key)::text = 'AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE'::text)"
end
create_table "cluster_groups", id: :serial, force: :cascade do |t|
......
......@@ -108,7 +108,8 @@ module Gitlab
services_usage,
approximate_counts,
usage_counters,
user_preferences_usage
user_preferences_usage,
ingress_modsecurity_usage
)
}
end
......@@ -170,6 +171,10 @@ module Gitlab
}
end
def ingress_modsecurity_usage
::Clusters::Applications::IngressModsecurityUsageService.new.execute
end
# rubocop: disable CodeReuse/ActiveRecord
def services_usage
types = {
......
......@@ -297,6 +297,24 @@ describe Gitlab::UsageData do
end
end
describe '#ingress_modsecurity_usage' do
subject { described_class.ingress_modsecurity_usage }
it 'gathers variable data' do
allow_any_instance_of(
::Clusters::Applications::IngressModsecurityUsageService
).to receive(:execute).and_return(
{
ingress_modsecurity_blocking: 1,
ingress_modsecurity_disabled: 2
}
)
expect(subject[:ingress_modsecurity_blocking]).to eq(1)
expect(subject[:ingress_modsecurity_disabled]).to eq(2)
end
end
describe '#license_usage_data' do
subject { described_class.license_usage_data }
......
# frozen_string_literal: true
require 'spec_helper'
describe Clusters::Applications::IngressModsecurityUsageService do
describe '#execute' do
ADO_MODSEC_KEY = Clusters::Applications::IngressModsecurityUsageService::ADO_MODSEC_KEY
let(:project_with_ci_var) { create(:environment).project }
let(:project_with_pipeline_var) { create(:environment).project }
subject { described_class.new.execute }
context 'with multiple projects' do
let(:pipeline1) { create(:ci_pipeline, :with_job, project: project_with_pipeline_var) }
let(:pipeline2) { create(:ci_pipeline, :with_job, project: project_with_ci_var) }
let!(:deployment_with_pipeline_var) do
create(
:deployment,
:success,
environment: project_with_pipeline_var.environments.first,
project: project_with_pipeline_var,
deployable: pipeline1.builds.last
)
end
let!(:deployment_with_project_var) do
create(
:deployment,
:success,
environment: project_with_ci_var.environments.first,
project: project_with_ci_var,
deployable: pipeline2.builds.last
)
end
context 'mixed data' do
let!(:ci_variable) { create(:ci_variable, project: project_with_ci_var, key: ADO_MODSEC_KEY, value: "On") }
let!(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline1, key: ADO_MODSEC_KEY, value: "Off") }
it 'gathers variable data' do
expect(subject[:ingress_modsecurity_blocking]).to eq(1)
expect(subject[:ingress_modsecurity_disabled]).to eq(1)
end
end
context 'blocking' do
let(:modsec_values) { { key: ADO_MODSEC_KEY, value: "On" } }
let!(:ci_variable) { create(:ci_variable, project: project_with_ci_var, **modsec_values) }
let!(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline1, **modsec_values) }
it 'gathers variable data' do
expect(subject[:ingress_modsecurity_blocking]).to eq(2)
expect(subject[:ingress_modsecurity_disabled]).to eq(0)
end
end
context 'disabled' do
let(:modsec_values) { { key: ADO_MODSEC_KEY, value: "Off" } }
let!(:ci_variable) { create(:ci_variable, project: project_with_ci_var, **modsec_values) }
let!(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline1, **modsec_values) }
it 'gathers variable data' do
expect(subject[:ingress_modsecurity_blocking]).to eq(0)
expect(subject[:ingress_modsecurity_disabled]).to eq(2)
end
end
end
context 'when set as both ci and pipeline variables' do
let(:modsec_values) { { key: ADO_MODSEC_KEY, value: "Off" } }
let(:pipeline) { create(:ci_pipeline, :with_job, project: project_with_ci_var) }
let!(:deployment) do
create(
:deployment,
:success,
environment: project_with_ci_var.environments.first,
project: project_with_ci_var,
deployable: pipeline.builds.last
)
end
let!(:ci_variable) { create(:ci_variable, project: project_with_ci_var, **modsec_values) }
let!(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline, **modsec_values) }
it 'wont double-count projects' do
expect(subject[:ingress_modsecurity_blocking]).to eq(0)
expect(subject[:ingress_modsecurity_disabled]).to eq(1)
end
it 'gives precedence to pipeline variable' do
pipeline_variable.update(value: "On")
expect(subject[:ingress_modsecurity_blocking]).to eq(1)
expect(subject[:ingress_modsecurity_disabled]).to eq(0)
end
end
context 'when a project has multiple environments' do
let(:modsec_values) { { key: ADO_MODSEC_KEY, value: "On" } }
let!(:env1) { project_with_pipeline_var.environments.first }
let!(:env2) { create(:environment, project: project_with_pipeline_var) }
let!(:pipeline_with_2_deployments) do
create(:ci_pipeline, :with_job, project: project_with_ci_var).tap do |pip|
pip.builds << build(:ci_build, pipeline: pip, project: project_with_pipeline_var)
end
end
let!(:deployment1) do
create(
:deployment,
:success,
environment: env1,
project: project_with_pipeline_var,
deployable: pipeline_with_2_deployments.builds.last
)
end
let!(:deployment2) do
create(
:deployment,
:success,
environment: env2,
project: project_with_pipeline_var,
deployable: pipeline_with_2_deployments.builds.last
)
end
context 'when set as ci variable' do
let!(:ci_variable) { create(:ci_variable, project: project_with_pipeline_var, **modsec_values) }
it 'gathers variable data' do
expect(subject[:ingress_modsecurity_blocking]).to eq(2)
expect(subject[:ingress_modsecurity_disabled]).to eq(0)
end
end
context 'when set as pipeline variable' do
let!(:pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline_with_2_deployments, **modsec_values) }
it 'gathers variable data' do
expect(subject[:ingress_modsecurity_blocking]).to eq(2)
expect(subject[:ingress_modsecurity_disabled]).to eq(0)
end
end
end
context 'when an environment has multiple deployments' do
let!(:env) { project_with_pipeline_var.environments.first }
let!(:pipeline_first) do
create(:ci_pipeline, :with_job, project: project_with_pipeline_var).tap do |pip|
pip.builds << build(:ci_build, pipeline: pip, project: project_with_pipeline_var)
end
end
let!(:pipeline_last) do
create(:ci_pipeline, :with_job, project: project_with_pipeline_var).tap do |pip|
pip.builds << build(:ci_build, pipeline: pip, project: project_with_pipeline_var)
end
end
let!(:deployment_first) do
create(
:deployment,
:success,
environment: env,
project: project_with_pipeline_var,
deployable: pipeline_first.builds.last
)
end
let!(:deployment_last) do
create(
:deployment,
:success,
environment: env,
project: project_with_pipeline_var,
deployable: pipeline_last.builds.last
)
end
context 'when set as pipeline variable' do
let!(:first_pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline_first, key: ADO_MODSEC_KEY, value: "On") }
let!(:last_pipeline_variable) { create(:ci_pipeline_variable, pipeline: pipeline_last, key: ADO_MODSEC_KEY, value: "Off") }
it 'gives precedence to latest deployment' do
expect(subject[:ingress_modsecurity_blocking]).to eq(0)
expect(subject[:ingress_modsecurity_disabled]).to eq(1)
end
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment