From 2a4da833925458f1a43af248ab0b44246a71a407 Mon Sep 17 00:00:00 2001 From: Max Woolf <mwoolf@gitlab.com> Date: Tue, 30 Nov 2021 09:21:32 +0000 Subject: [PATCH] Add event streaming metrics Adds two new metrics: - Count of all event streaming destinations - Distinct count of groups with any streaming destinations Changelog: added EE: true --- ee/app/models/ee/group.rb | 4 ++++ ...ge_compliance_audit_event_destinations.yml | 19 +++++++++++++++++++ ...oups_with_event_streaming_destinations.yml | 19 +++++++++++++++++++ ...ge_compliance_audit_event_destinations.yml | 19 +++++++++++++++++++ ...oups_with_event_streaming_destinations.yml | 19 +++++++++++++++++++ ee/lib/ee/gitlab/usage_data.rb | 5 ++++- ...unt_event_streaming_destinations_metric.rb | 17 +++++++++++++++++ ...ith_event_streaming_destinations_metric.rb | 17 +++++++++++++++++ ...vent_streaming_destinations_metric_spec.rb | 12 ++++++++++++ ...vent_streaming_destinations_metric_spec.rb | 17 +++++++++++++++++ 10 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml create mode 100644 ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml create mode 100644 ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml create mode 100644 ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml create mode 100644 ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb create mode 100644 ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb create mode 100644 ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb create mode 100644 ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb diff --git a/ee/app/models/ee/group.rb b/ee/app/models/ee/group.rb index b0315078faa80cc5..269a71e6d6da2ec0 100644 --- a/ee/app/models/ee/group.rb +++ b/ee/app/models/ee/group.rb @@ -91,6 +91,10 @@ module Group joins(:ldap_group_links).where(ldap_group_links: { provider: provider }) end + scope :with_external_audit_event_destinations, -> do + joins(:external_audit_event_destinations) + end + scope :with_managed_accounts_enabled, -> do joins(:saml_provider).where(saml_providers: { diff --git a/ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml b/ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml new file mode 100644 index 0000000000000000..84422517f6cdd98c --- /dev/null +++ b/ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml @@ -0,0 +1,19 @@ +--- +key_path: usage_activity_by_stage_monthly.manage.audit_event_destinations +description: "Count of audit event streaming destinations" +product_section: "dev" +product_stage: "manage" +product_group: "compliance" +product_category: "Audit Events" +value_type: number +status: active +milestone: "14.6" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670 +time_frame: "28d" +data_source: database +data_category: optional +instrumentation_class: CountEventStreamingDestinationsMetric +distribution: +- ee +tier: +- ultimate diff --git a/ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml b/ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml new file mode 100644 index 0000000000000000..df5a75d25f91d884 --- /dev/null +++ b/ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml @@ -0,0 +1,19 @@ +--- +key_path: usage_activity_by_stage_monthly.manage.groups_with_event_streaming_destinations +description: "Distinct count of groups with any event streaming destinations" +product_section: "dev" +product_stage: "manage" +product_group: "compliance" +product_category: "Audit Events" +value_type: number +status: active +milestone: "14.6" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670 +time_frame: "28d" +data_source: "database" +data_category: optional +instrumentation_class: CountGroupsWithEventStreamingDestinationsMetric +distribution: +- ee +tier: +- ultimate diff --git a/ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml b/ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml new file mode 100644 index 0000000000000000..9743a17dec051143 --- /dev/null +++ b/ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml @@ -0,0 +1,19 @@ +--- +key_path: usage_activity_by_stage.manage.audit_event_destinations +description: "Count of audit event streaming destinations" +product_section: "dev" +product_stage: "manage" +product_group: "compliance" +product_category: "Audit Events" +value_type: number +status: active +milestone: "14.6" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670 +time_frame: "all" +data_source: database +data_category: optional +instrumentation_class: CountEventStreamingDestinationsMetric +distribution: +- ee +tier: +- ultimate diff --git a/ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml b/ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml new file mode 100644 index 0000000000000000..00ac22d475549a5c --- /dev/null +++ b/ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml @@ -0,0 +1,19 @@ +--- +key_path: usage_activity_by_stage.manage.groups_with_event_streaming_destinations +description: "Distinct count of groups with any event streaming destinations" +product_section: "dev" +product_stage: "manage" +product_group: "compliance" +product_category: "Audit Events" +value_type: number +status: active +milestone: "14.6" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670 +time_frame: all +data_source: "database" +data_category: optional +instrumentation_class: CountGroupsWithEventStreamingDestinationsMetric +distribution: +- ee +tier: +- ultimate diff --git a/ee/lib/ee/gitlab/usage_data.rb b/ee/lib/ee/gitlab/usage_data.rb index b54be52422b4d05b..eca9e28d6ee566b7 100644 --- a/ee/lib/ee/gitlab/usage_data.rb +++ b/ee/lib/ee/gitlab/usage_data.rb @@ -306,6 +306,7 @@ def usage_activity_by_stage_enablement(time_period) # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links` override :usage_activity_by_stage_manage def usage_activity_by_stage_manage(time_period) + time_frame = metric_time_period(time_period) super.merge({ ldap_keys: distinct_count(::LDAPKey.where(time_period), :user_id), ldap_users: distinct_count(::GroupMember.of_ldap_type.where(time_period), :user_id), @@ -316,7 +317,9 @@ def usage_activity_by_stage_manage(time_period) ldap_servers: ldap_available_servers.size, ldap_group_sync_enabled: ldap_config_present_for_any_provider?(:group_base), ldap_admin_sync_enabled: ldap_config_present_for_any_provider?(:admin_group), - group_saml_enabled: omniauth_provider_names.include?('group_saml') + group_saml_enabled: omniauth_provider_names.include?('group_saml'), + audit_event_destinations: add_metric('CountEventStreamingDestinationsMetric', time_frame: time_frame), + groups_with_event_streaming_destinations: add_metric('CountGroupsWithEventStreamingDestinationsMetric', time_frame: time_frame) }) end diff --git a/ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb b/ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb new file mode 100644 index 0000000000000000..949400a70b69dfe7 --- /dev/null +++ b/ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountEventStreamingDestinationsMetric < DatabaseMetric + operation :count + + relation do + AuditEvents::ExternalAuditEventDestination + end + end + end + end + end +end diff --git a/ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb b/ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb new file mode 100644 index 0000000000000000..6ba8caaec08c979c --- /dev/null +++ b/ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountGroupsWithEventStreamingDestinationsMetric < DatabaseMetric + operation :distinct_count + + relation do + Group.with_external_audit_event_destinations + end + end + end + end + end +end diff --git a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb new file mode 100644 index 0000000000000000..c58f31e6e0c44716 --- /dev/null +++ b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountEventStreamingDestinationsMetric do + let_it_be(:destination) { create(:external_audit_event_destination) } + + it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do + let(:expected_value) { 1 } + let(:expected_query) { 'SELECT COUNT("audit_events_external_audit_event_destinations"."id") FROM "audit_events_external_audit_event_destinations"' } + end +end diff --git a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb new file mode 100644 index 0000000000000000..13bbc62ed672cb7a --- /dev/null +++ b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountGroupsWithEventStreamingDestinationsMetric do + let_it_be(:group_with_destination) { create(:group) } + let_it_be(:group_without_destination) { create(:group) } + + before do + create(:external_audit_event_destination, group: group_with_destination) + end + + it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do + let(:expected_value) { 1 } + let(:expected_query) { 'SELECT COUNT(DISTINCT "namespaces"."id") FROM "namespaces" INNER JOIN "audit_events_external_audit_event_destinations" ON "audit_events_external_audit_event_destinations"."namespace_id" = "namespaces"."id" WHERE "namespaces"."type" = \'Group\'' } + end +end -- GitLab