POC for snowplow tracking verification

- verify growth funnels and help ensure there are no regressions.

.rubocop_todo/rspec/feature_category.yml

@@ -4072,7 +4072,6 @@ RSpec/FeatureCategory:
     - 'spec/lib/gitlab/themes_spec.rb'
     - 'spec/lib/gitlab/throttle_spec.rb'
     - 'spec/lib/gitlab/time_tracking_formatter_spec.rb'
-    - 'spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb'
     - 'spec/lib/gitlab/tracking/destinations/snowplow_spec.rb'
     - 'spec/lib/gitlab/tracking/event_definition_spec.rb'
     - 'spec/lib/gitlab/tracking/helpers/weak_password_error_event_spec.rb'

ee/spec/features/registrations/saas/trial_flow_company_creating_project_spec.rb

@@ -3,15 +3,15 @@
 require 'spec_helper'
 
 RSpec.describe 'Trial flow for user picking company and creating a project', :js, :saas_registration, feature_category: :onboarding do
-  where(:case_name, :sign_up_method) do
+  where(:case_name, :sign_up_method, :tracking_events_key) do
     [
-      ['with regular trial sign up', ->(params) { trial_registration_sign_up(params) }],
-      ['with sso trial sign up', ->(params) { sso_trial_registration_sign_up(params) }]
+      ['with regular trial sign up', ->(params) { trial_registration_sign_up(params) }, :trial_regular_signup],
+      ['with sso trial sign up', ->(params) { sso_trial_registration_sign_up(params) }, :trial_sso_signup]
     ]
   end
 
   with_them do
-    it 'registers the user and creates a group and project reaching onboarding', :sidekiq_inline do
+    it 'registers the user and creates a group and project reaching onboarding', :snowplow_micro, :sidekiq_inline do
       sign_up_method.call(glm_params)
 
       ensure_onboarding { expect_to_see_welcome_form }
@@ -41,6 +41,7 @@
       click_on 'Ok, let\'s go'
 
       expect_to_be_in_learn_gitlab
+      expect(tracking_events_key).to have_all_expected_events
     end
 
     context 'with free_trial_registration_redesign experiment candidate' do

ee/spec/fixtures/snowplow/tracking_verification/trial_regular_signup.yml

+---
+page_view_events:
+  - /-/trial_registrations/new
+  - /users/identity_verification
+  - /users/identity_verification/success
+  - /users/sign_up/welcome
+  - /users/sign_up/company
+  - /users/sign_up/company/new
+  - /users/sign_up/groups/new
+  - /-/learn_gitlab/onboarding
+  - /-/learn_gitlab
+structured_events:
+  - category: 'registrations:welcome:show'
+    action: 'render'
+    label: 'trial_registration'
+  - category: 'Registrations::CompanyController'
+    action: 'render'
+    label: 'trial_registration'
+  - category: 'Registrations::GroupsController'
+    action: 'view_new_group_action'
+    label: 'trial_registration'

ee/spec/fixtures/snowplow/tracking_verification/trial_sso_signup.yml

+---
+page_view_events:
+  - /-/trial_registrations/new
+  - /users/identity_verification
+  - /users/identity_verification/success
+  - /users/sign_up/welcome
+  - /users/sign_up/company
+  - /users/sign_up/company/new
+  - /users/sign_up/groups/new
+  - /-/learn_gitlab/onboarding
+  - /-/learn_gitlab
+structured_events:
+  - category: 'OmniauthCallbacksController'
+    action: 'google_oauth2_sso'
+  - category: 'registrations:welcome:show'
+    action: 'render'
+    label: 'trial_registration'
+  - category: 'Registrations::CompanyController'
+    action: 'render'
+    label: 'trial_registration'
+  - category: 'Registrations::GroupsController'
+    action: 'view_new_group_action'
+    label: 'trial_registration'

ee/spec/spec_helper.rb

@@ -47,4 +47,19 @@
   config.around(:each, :geo_tracking_db) do |example|
     example.run if Gitlab::Geo.geo_database_configured?
   end
+
+  config.define_derived_metadata do |metadata|
+    metadata[:do_not_stub_snowplow_by_default] = true if metadata.has_key?(:snowplow_micro)
+  end
+
+  config.before(:example, :snowplow_micro) do
+    config.include(Matchers::Snowplow)
+
+    next unless Gitlab::Tracking.micro_verification_enabled?
+
+    Matchers::Snowplow.clean_snowplow_queue
+
+    stub_application_setting(snowplow_enabled: true)
+    stub_application_setting(snowplow_app_id: 'gitlab-test')
+  end
 end

ee/spec/support/helpers/saas_registration_helpers.rb

@@ -359,7 +359,7 @@ def fills_in_group_and_project_creation_form
     # The groups_and_projects_controller (on `click_on 'Create project'`) is over
     # the query limit threshold, so we have to adjust it.
     # https://gitlab.com/gitlab-org/gitlab/-/issues/404805
-    allow(Gitlab::QueryLimiting::Transaction).to receive(:threshold).and_return(167)
+    allow(Gitlab::QueryLimiting::Transaction).to receive(:threshold).and_return(170)
 
     fill_in 'group_name', with: 'Test Group'
     fill_in 'blank_project_name', with: 'Test Project'

ee/spec/support/matchers/snowplow/tracking_event_verifier.rb

+# frozen_string_literal: true
+
+module Matchers
+  module Snowplow
+    def self.clean_snowplow_queue
+      return unless Gitlab::Tracking.micro_verification_enabled?
+
+      # http://localhost:9090/micro/reset
+      Gitlab::HTTP
+        .try_get("#{Gitlab::Tracking::Destinations::SnowplowMicro.new.uri}/micro/reset", allow_local_requests: true)
+    end
+
+    def have_all_expected_events
+      TrackingEventVerifier.new
+    end
+
+    class TrackingEventVerifier
+      def matches?(events_key)
+        return true unless Gitlab::Tracking.micro_verification_enabled?
+
+        load_file_contents(events_key)
+
+        flush_remaining_events
+        fetch_results
+
+        process_page_view_events
+        process_structured_events
+
+        not_found_page_view_events.empty? && not_found_structured_events.empty?
+      end
+
+      def failure_message
+        suffix_msg = "to be found in tracking results and were not"
+
+        final_message = ''
+
+        if not_found_page_view_events.present?
+          final_message = "expected page view events #{not_found_page_view_events.inspect} #{suffix_msg}.\n"
+        end
+
+        if not_found_structured_events.present?
+          final_message += "expected structured events #{not_found_structured_events.inspect} #{suffix_msg}.\n"
+        end
+
+        final_message
+      end
+
+      private
+
+      attr_reader :page_view_events, :structured_events, :results, :not_found_page_view_events,
+        :not_found_structured_events
+
+      def load_file_contents(events_key)
+        file_name = "#{events_key}.yml"
+        file_contents = YAML.safe_load(
+          File.read(Rails.root.join('ee/spec/fixtures/snowplow/tracking_verification', file_name))
+        )
+        @structured_events = file_contents.fetch('structured_events', [])
+        @page_view_events = file_contents.fetch('page_view_events', [])
+        @not_found_page_view_events = page_view_events.dup
+        @not_found_structured_events = structured_events.dup
+      end
+
+      def flush_remaining_events
+        # one last non-async flush just in case anything is left in the buffer(though we are set to 1 for buffer_size)
+        ::Gitlab::Tracking.flush
+      end
+
+      def fetch_results
+        # http://localhost:9090/micro/good
+        # take the above output and verify it vs some predefined json/yml
+        # maybe some validation against all too for brokenness
+        # RestClient.get('http://localhost:9090/micro/good') # parse it against what we expect to see
+        @results = Gitlab::HTTP.try_get(
+          "#{Gitlab::Tracking::Destinations::SnowplowMicro.new.uri}/micro/good", allow_local_requests: true
+        )
+      end
+
+      def process_page_view_events
+        page_view_events.each do |page_view_event|
+          verify_page_view_event(page_view_event)
+        end
+      end
+
+      def verify_page_view_event(page_view_event)
+        results.select { |r, _v| r['eventType'] == 'page_view' }.each do |e|
+          # fuzzy this a bit initially since we have project/group in path /namespace97/project31/-/learn_gitlab
+          if e.dig('event', 'page_urlpath').match?(/#{page_view_event}\z/)
+            @not_found_page_view_events.delete(page_view_event)
+          end
+        end
+      end
+
+      def process_structured_events
+        structured_events.each do |structured_event|
+          verify_structured_event(structured_event)
+        end
+      end
+
+      def verify_structured_event(structured_event)
+        results.select { |r, _v| r['eventType'] == 'struct' }.each do |result_event|
+          next unless structured_event_found?(result_event, structured_event)
+
+          @not_found_structured_events.delete(structured_event)
+        end
+      end
+
+      def structured_event_found?(result_event, structured_event)
+        found = false
+
+        %w[category action label property value].each do |field|
+          next unless structured_event[field].present?
+
+          found = result_event.dig('event', "se_#{field}") == structured_event[field]
+
+          break unless found
+        end
+
+        found
+      end
+    end
+  end
+end

lib/gitlab/tracking.rb

@@ -6,10 +6,16 @@
 module Gitlab
   module Tracking
     class << self
+      delegate :flush, to: :tracker
+
       def enabled?
         tracker.enabled?
       end
 
+      def micro_verification_enabled?
+        Gitlab::Utils.to_boolean(ENV['VERIFY_TRACKING'], default: false)
+      end
+
       def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
         action = action.to_s
 
@@ -66,7 +72,7 @@ def collector_hostname
       end
 
       def snowplow_micro_enabled?
-        Rails.env.development? && Gitlab.config.snowplow_micro.enabled
+        (Rails.env.development? || micro_verification_enabled?) && Gitlab.config.snowplow_micro.enabled
       rescue GitlabSettings::MissingSetting
         false
       end

lib/gitlab/tracking/destinations/snowplow_micro.rb

@@ -7,6 +7,8 @@ class SnowplowMicro < Snowplow
         include ::Gitlab::Utils::StrongMemoize
         extend ::Gitlab::Utils::Override
 
+        delegate :flush, to: :tracker
+
         DEFAULT_URI = 'http://localhost:9090'
 
         override :options

spec/lib/gitlab/tracking/destinations/snowplow_micro_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::Tracking::Destinations::SnowplowMicro do
+RSpec.describe Gitlab::Tracking::Destinations::SnowplowMicro, feature_category: :application_instrumentation do
   include StubENV
 
   let(:snowplow_micro_settings) do
@@ -18,6 +18,8 @@
     allow(Rails.env).to receive(:development?).and_return(true)
   end
 
+  it { is_expected.to delegate_method(:flush).to(:tracker) }
+
   describe '#hostname' do
     context 'when snowplow_micro config is set' do
       let(:address) { '127.0.0.1:9091' }

spec/lib/gitlab/tracking_spec.rb

@@ -3,6 +3,7 @@
 
 RSpec.describe Gitlab::Tracking, feature_category: :application_instrumentation do
   include StubENV
+  using RSpec::Parameterized::TableSyntax
 
   before do
     stub_application_setting(snowplow_enabled: true)
@@ -17,6 +18,8 @@
     described_class.instance_variable_set(:@tracker, nil)
   end
 
+  it { is_expected.to delegate_method(:flush).to(:tracker) }
+
   describe '.options' do
     shared_examples 'delegates to destination' do |klass|
       before do
@@ -295,29 +298,57 @@
   end
 
   describe 'snowplow_micro_enabled?' do
-    before do
-      allow(Rails.env).to receive(:development?).and_return(true)
+    where(:development?, :micro_verification_enabled?, :snowplow_micro_enabled, :result) do
+      true  | true  | true  | true
+      true  | true  | false | false
+      false | true  | true  | true
+      false | true  | false | false
+      false | false | true  | false
+      false | false | false | false
+      true  | false | true  | true
+      true  | false | false | false
     end
 
-    it 'returns true when snowplow_micro is enabled' do
-      stub_config(snowplow_micro: { enabled: true })
-
-      expect(described_class).to be_snowplow_micro_enabled
-    end
+    with_them do
+      before do
+        allow(Rails.env).to receive(:development?).and_return(development?)
+        allow(described_class).to receive(:micro_verification_enabled?).and_return(micro_verification_enabled?)
+        stub_config(snowplow_micro: { enabled: snowplow_micro_enabled })
+      end
 
-    it 'returns false when snowplow_micro is disabled' do
-      stub_config(snowplow_micro: { enabled: false })
+      subject { described_class.snowplow_micro_enabled? }
 
-      expect(described_class).not_to be_snowplow_micro_enabled
+      it { is_expected.to be(result) }
     end
 
     it 'returns false when snowplow_micro is not configured' do
+      allow(Rails.env).to receive(:development?).and_return(true)
       allow(Gitlab.config).to receive(:snowplow_micro).and_raise(GitlabSettings::MissingSetting)
 
       expect(described_class).not_to be_snowplow_micro_enabled
     end
   end
 
+  describe '.micro_verification_enabled?' do
+    where(:verify_tracking, :result) do
+      nil     | false
+      'true'  | true
+      'false' | false
+      '0'     | false
+      '1'     | true
+    end
+
+    with_them do
+      before do
+        stub_env('VERIFY_TRACKING', verify_tracking)
+      end
+
+      subject { described_class.micro_verification_enabled? }
+
+      it { is_expected.to be(result) }
+    end
+  end
+
   describe 'tracker' do
     it 'returns a SnowPlowMicro instance in development' do
       allow(Rails.env).to receive(:development?).and_return(true)