From 2a4da833925458f1a43af248ab0b44246a71a407 Mon Sep 17 00:00:00 2001
From: Max Woolf <mwoolf@gitlab.com>
Date: Tue, 30 Nov 2021 09:21:32 +0000
Subject: [PATCH] Add event streaming metrics

Adds two new metrics:
- Count of all event streaming destinations
- Distinct count of groups with any streaming destinations

Changelog: added
EE: true
---
 ee/app/models/ee/group.rb                     |  4 ++++
 ...ge_compliance_audit_event_destinations.yml | 19 +++++++++++++++++++
 ...oups_with_event_streaming_destinations.yml | 19 +++++++++++++++++++
 ...ge_compliance_audit_event_destinations.yml | 19 +++++++++++++++++++
 ...oups_with_event_streaming_destinations.yml | 19 +++++++++++++++++++
 ee/lib/ee/gitlab/usage_data.rb                |  5 ++++-
 ...unt_event_streaming_destinations_metric.rb | 17 +++++++++++++++++
 ...ith_event_streaming_destinations_metric.rb | 17 +++++++++++++++++
 ...vent_streaming_destinations_metric_spec.rb | 12 ++++++++++++
 ...vent_streaming_destinations_metric_spec.rb | 17 +++++++++++++++++
 10 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100644 ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml
 create mode 100644 ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml
 create mode 100644 ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml
 create mode 100644 ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml
 create mode 100644 ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb
 create mode 100644 ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb
 create mode 100644 ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb
 create mode 100644 ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb

diff --git a/ee/app/models/ee/group.rb b/ee/app/models/ee/group.rb
index b0315078faa80cc5..269a71e6d6da2ec0 100644
--- a/ee/app/models/ee/group.rb
+++ b/ee/app/models/ee/group.rb
@@ -91,6 +91,10 @@ module Group
         joins(:ldap_group_links).where(ldap_group_links: { provider: provider })
       end
 
+      scope :with_external_audit_event_destinations, -> do
+        joins(:external_audit_event_destinations)
+      end
+
       scope :with_managed_accounts_enabled, -> do
         joins(:saml_provider).where(saml_providers:
           {
diff --git a/ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml b/ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml
new file mode 100644
index 0000000000000000..84422517f6cdd98c
--- /dev/null
+++ b/ee/config/metrics/counts_28d/20211130085433_g_manage_compliance_audit_event_destinations.yml
@@ -0,0 +1,19 @@
+---
+key_path: usage_activity_by_stage_monthly.manage.audit_event_destinations
+description: "Count of audit event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: "28d"
+data_source: database
+data_category: optional
+instrumentation_class: CountEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate
diff --git a/ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml b/ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml
new file mode 100644
index 0000000000000000..df5a75d25f91d884
--- /dev/null
+++ b/ee/config/metrics/counts_28d/20211130091657_groups_with_event_streaming_destinations.yml
@@ -0,0 +1,19 @@
+---
+key_path: usage_activity_by_stage_monthly.manage.groups_with_event_streaming_destinations
+description: "Distinct count of groups with any event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: "28d"
+data_source: "database"
+data_category: optional
+instrumentation_class: CountGroupsWithEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate
diff --git a/ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml b/ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml
new file mode 100644
index 0000000000000000..9743a17dec051143
--- /dev/null
+++ b/ee/config/metrics/counts_all/20211130085433_g_manage_compliance_audit_event_destinations.yml
@@ -0,0 +1,19 @@
+---
+key_path: usage_activity_by_stage.manage.audit_event_destinations
+description: "Count of audit event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: "all"
+data_source: database
+data_category: optional
+instrumentation_class: CountEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate
diff --git a/ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml b/ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml
new file mode 100644
index 0000000000000000..00ac22d475549a5c
--- /dev/null
+++ b/ee/config/metrics/counts_all/20211130091657_groups_with_event_streaming_destinations.yml
@@ -0,0 +1,19 @@
+---
+key_path: usage_activity_by_stage.manage.groups_with_event_streaming_destinations
+description: "Distinct count of groups with any event streaming destinations"
+product_section: "dev"
+product_stage: "manage"
+product_group: "compliance"
+product_category: "Audit Events"
+value_type: number
+status: active
+milestone: "14.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/344670
+time_frame: all
+data_source: "database"
+data_category: optional
+instrumentation_class: CountGroupsWithEventStreamingDestinationsMetric
+distribution:
+- ee
+tier:
+- ultimate
diff --git a/ee/lib/ee/gitlab/usage_data.rb b/ee/lib/ee/gitlab/usage_data.rb
index b54be52422b4d05b..eca9e28d6ee566b7 100644
--- a/ee/lib/ee/gitlab/usage_data.rb
+++ b/ee/lib/ee/gitlab/usage_data.rb
@@ -306,6 +306,7 @@ def usage_activity_by_stage_enablement(time_period)
         # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links`
         override :usage_activity_by_stage_manage
         def usage_activity_by_stage_manage(time_period)
+          time_frame = metric_time_period(time_period)
           super.merge({
             ldap_keys: distinct_count(::LDAPKey.where(time_period), :user_id),
             ldap_users: distinct_count(::GroupMember.of_ldap_type.where(time_period), :user_id),
@@ -316,7 +317,9 @@ def usage_activity_by_stage_manage(time_period)
             ldap_servers: ldap_available_servers.size,
             ldap_group_sync_enabled: ldap_config_present_for_any_provider?(:group_base),
             ldap_admin_sync_enabled: ldap_config_present_for_any_provider?(:admin_group),
-            group_saml_enabled: omniauth_provider_names.include?('group_saml')
+            group_saml_enabled: omniauth_provider_names.include?('group_saml'),
+            audit_event_destinations: add_metric('CountEventStreamingDestinationsMetric', time_frame: time_frame),
+            groups_with_event_streaming_destinations: add_metric('CountGroupsWithEventStreamingDestinationsMetric', time_frame: time_frame)
           })
         end
 
diff --git a/ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb b/ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb
new file mode 100644
index 0000000000000000..949400a70b69dfe7
--- /dev/null
+++ b/ee/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class CountEventStreamingDestinationsMetric < DatabaseMetric
+          operation :count
+
+          relation do
+            AuditEvents::ExternalAuditEventDestination
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb b/ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb
new file mode 100644
index 0000000000000000..6ba8caaec08c979c
--- /dev/null
+++ b/ee/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class CountGroupsWithEventStreamingDestinationsMetric < DatabaseMetric
+          operation :distinct_count
+
+          relation do
+            Group.with_external_audit_event_destinations
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb
new file mode 100644
index 0000000000000000..c58f31e6e0c44716
--- /dev/null
+++ b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_event_streaming_destinations_metric_spec.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountEventStreamingDestinationsMetric do
+  let_it_be(:destination) { create(:external_audit_event_destination) }
+
+  it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do
+    let(:expected_value) { 1 }
+    let(:expected_query) { 'SELECT COUNT("audit_events_external_audit_event_destinations"."id") FROM "audit_events_external_audit_event_destinations"' }
+  end
+end
diff --git a/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb
new file mode 100644
index 0000000000000000..13bbc62ed672cb7a
--- /dev/null
+++ b/ee/spec/lib/gitlab/usage/metrics/instrumentations/count_groups_with_event_streaming_destinations_metric_spec.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CountGroupsWithEventStreamingDestinationsMetric do
+  let_it_be(:group_with_destination) { create(:group) }
+  let_it_be(:group_without_destination) { create(:group) }
+
+  before do
+    create(:external_audit_event_destination, group: group_with_destination)
+  end
+
+  it_behaves_like 'a correct instrumented metric value and query', { time_frame: 'all', data_source: 'database' } do
+    let(:expected_value) { 1 }
+    let(:expected_query) { 'SELECT COUNT(DISTINCT "namespaces"."id") FROM "namespaces" INNER JOIN "audit_events_external_audit_event_destinations" ON "audit_events_external_audit_event_destinations"."namespace_id" = "namespaces"."id" WHERE "namespaces"."type" = \'Group\'' }
+  end
+end
-- 
GitLab