Merge branch '250699-epic-index' into 'master'

Create Epic Elasticsearch index

See merge request !121635



Merged-by: John Mason <9717668-johnmason@users.noreply.gitlab.com>
Approved-by: Ravi Kumar <rkumar@gitlab.com>
Approved-by: John Mason <9717668-johnmason@users.noreply.gitlab.com>
Co-authored-by: maddievn <mvanniekerk@gitlab.com>

.rubocop_todo/search/namespaced_class.yml

@@ -93,6 +93,9 @@ Search/NamespacedClass:
     - 'ee/lib/elastic/latest/config.rb'
     - 'ee/lib/elastic/latest/custom_language_analyzers.rb'
     - 'ee/lib/elastic/latest/document_should_be_deleted_from_index_error.rb'
+    - 'ee/lib/elastic/latest/epic_class_proxy.rb'
+    - 'ee/lib/elastic/latest/epic_config.rb'
+    - 'ee/lib/elastic/latest/epic_instance_proxy.rb'
     - 'ee/lib/elastic/latest/git_class_proxy.rb'
     - 'ee/lib/elastic/latest/git_instance_proxy.rb'
     - 'ee/lib/elastic/latest/issue_class_proxy.rb'
@@ -131,6 +134,8 @@ Search/NamespacedClass:
     - 'ee/lib/elastic/v12p1/application_class_proxy.rb'
     - 'ee/lib/elastic/v12p1/application_instance_proxy.rb'
     - 'ee/lib/elastic/v12p1/config.rb'
+    - 'ee/lib/elastic/v12p1/epic_class_proxy.rb'
+    - 'ee/lib/elastic/v12p1/epic_instance_proxy.rb'
     - 'ee/lib/elastic/v12p1/issue_class_proxy.rb'
     - 'ee/lib/elastic/v12p1/issue_instance_proxy.rb'
     - 'ee/lib/elastic/v12p1/merge_request_class_proxy.rb'

config/feature_flags/ops/search_index_curation_epics.yml

+---
+name: search_index_curation_epics
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121635
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/413605
+milestone: '16.2'
+type: ops
+group: group::global search
+default_enabled: false

ee/app/models/ee/epic.rb

@@ -23,6 +23,7 @@ module Epic
       include EachBatch
       include ::Exportable
       include Epics::MetadataCacheUpdate
+      include Elastic::ApplicationVersionedSearch
 
       DEFAULT_COLOR = ::Gitlab::Color.of('#1068bf')
       MAX_HIERARCHY_DEPTH = 7
@@ -325,6 +326,17 @@ def search(query)
         fuzzy_search(query, [:title, :description])
       end
 
+      def elasticsearch_available?
+        return false unless ::Feature.enabled?(:elastic_index_epics)
+
+        ::Elastic::DataMigrationService.migration_has_finished?(:create_epic_index)
+      end
+
+      override :use_separate_indices?
+      def use_separate_indices?
+        true
+      end
+
       def ids_for_base_and_decendants(epic_ids)
         ::Gitlab::ObjectHierarchy.new(self.id_in(epic_ids)).base_and_descendants.pluck(:id)
       end
@@ -519,6 +531,12 @@ def validate_parent
       validate_parent_epic
     end
 
+    def use_elasticsearch?
+      return false unless self.class.elasticsearch_available?
+
+      group.use_elasticsearch?
+    end
+
     def issues_readable_by(current_user, preload: nil)
       related_issues = self.class.related_issues(ids: id, preload: preload)
 

ee/config/feature_flags/development/elastic_index_epics.yml

+---
+name: elastic_index_epics
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121635
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/412108
+milestone: '16.2'
+type: development
+group: group::global search
+default_enabled: false

ee/elastic/migrate/20230615101400_create_epic_index.rb

+# frozen_string_literal: true
+
+class CreateEpicIndex < Elastic::Migration
+  include Elastic::MigrationCreateIndex
+
+  retry_on_failure
+
+  def document_type
+    :epic
+  end
+
+  def target_class
+    Epic
+  end
+end

ee/lib/elastic/latest/epic_class_proxy.rb

+# frozen_string_literal: true
+
+module Elastic
+  module Latest
+    class EpicClassProxy < ApplicationClassProxy
+      def elastic_search(query, options: {})
+        raise NotImplementedError
+      end
+
+      def preload_indexing_data(relation)
+        # rubocop: disable CodeReuse/ActiveRecord
+        relation.includes(
+          :author,
+          :labels,
+          :group,
+          :start_date_sourcing_epic,
+          :due_date_sourcing_epic,
+          :start_date_sourcing_milestone,
+          :due_date_sourcing_milestone
+        )
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end

ee/lib/elastic/latest/epic_config.rb

+# frozen_string_literal: true
+
+module Elastic
+  module Latest
+    module EpicConfig
+      extend Elasticsearch::Model::Indexing::ClassMethods
+      extend Elasticsearch::Model::Naming::ClassMethods
+
+      self.index_name = [Rails.application.class.module_parent_name.downcase, Rails.env, 'epics'].join('-')
+
+      settings Elastic::Latest::Config.settings.to_hash.deep_merge(
+        index: Elastic::Latest::Config.separate_index_specific_settings(index_name)
+      )
+
+      mappings dynamic: 'strict' do
+        indexes :id, type: :integer
+        indexes :iid, type: :integer
+        indexes :group_id, type: :integer
+
+        indexes :created_at, type: :date
+        indexes :updated_at, type: :date
+
+        indexes :title, type: :text, index_options: 'positions'
+        indexes :description, type: :text, index_options: 'positions'
+        indexes :state, type: :keyword
+        indexes :confidential, type: :boolean
+        indexes :author_id, type: :integer
+        indexes :label_ids, type: :keyword
+        indexes :start_date, type: :date
+        indexes :due_date, type: :date
+
+        indexes :traversal_ids, type: :keyword
+        indexes :hashed_root_namespace_id, type: :integer
+        indexes :visibility_level, type: :integer
+
+        indexes :schema_version, type: :short
+        indexes :type, type: :keyword
+      end
+    end
+  end
+end

ee/lib/elastic/latest/epic_instance_proxy.rb

+# frozen_string_literal: true
+
+module Elastic
+  module Latest
+    class EpicInstanceProxy < ApplicationInstanceProxy
+      def as_indexed_json(_options = {})
+        data = {}
+
+        [
+          :id,
+          :iid,
+          :group_id,
+          :created_at,
+          :updated_at,
+          :title,
+          :description,
+          :state,
+          :confidential,
+          :author_id
+        ].each do |attr|
+          data[attr.to_s] = safely_read_attribute_for_elasticsearch(attr)
+        end
+
+        data['label_ids'] = target.label_ids.map(&:to_s)
+        data['start_date'] = target.start_date || target.start_date_from_inherited_source
+        data['due_date'] = target.end_date || target.due_date_from_inherited_source
+
+        data['traversal_ids'] = target.group.elastic_namespace_ancestry
+        data['hashed_root_namespace_id'] = target.group.hashed_root_namespace_id
+        data['visibility_level'] = target.group.visibility_level
+
+        # Schema version. The format is Date.today.strftime('%y_%m')
+        # Please update if you're changing the schema of the document
+        data['schema_version'] = 23_06
+
+        data.merge(generic_attributes)
+      end
+
+      def generic_attributes
+        super.except('join_field')
+      end
+
+      def es_parent
+        "group_#{group.root_ancestor.id}"
+      end
+    end
+  end
+end

ee/lib/elastic/v12p1/epic_class_proxy.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Elastic
+  module V12p1
+    EpicClassProxy = Elastic::Latest::EpicClassProxy
+  end
+end
+
+# rubocop:enable Naming/FileName

ee/lib/elastic/v12p1/epic_instance_proxy.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Elastic
+  module V12p1
+    EpicInstanceProxy = Elastic::Latest::EpicInstanceProxy
+  end
+end
+
+# rubocop:enable Naming/FileName

ee/lib/gitlab/elastic/helper.rb

@@ -19,6 +19,7 @@ class Helper
         Note,
         MergeRequest,
         Commit,
+        Epic,
         User,
         Wiki,
         Project

ee/spec/elastic/migrate/20230615101400_create_epic_index_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative 'migration_shared_examples'
+require File.expand_path('ee/elastic/migrate/20230615101400_create_epic_index.rb')
+
+RSpec.describe CreateEpicIndex, feature_category: :global_search do
+  it_behaves_like 'migration creates a new index', 20230615101400, Epic
+end

ee/spec/elastic_integration/epic_index_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Epic index', feature_category: :global_search do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:parent_group) { create(:group) }
+  let_it_be_with_refind(:group) { create(:group, parent: parent_group) }
+  let_it_be_with_refind(:epic) { create(:epic, group: group) }
+  let_it_be(:member) { create(:group_member, :owner, group: group, user: user) }
+  let(:epic_index) { Epic.__elasticsearch__.index_name }
+  let(:helper) { Gitlab::Elastic::Helper.default }
+  let(:client) { helper.client }
+
+  before do
+    stub_feature_flags(elastic_index_epics: true)
+    allow(::Elastic::DataMigrationService).to receive(:migration_has_finished?)
+      .with(:create_epic_index).and_return(true)
+    stub_ee_application_setting(elasticsearch_indexing: true)
+    allow(::Elastic::ProcessBookkeepingService).to receive(:track!)
+  end
+
+  shared_examples 'epics get tracked in Elasticsearch' do
+    it 'use_elasticsearch? is true' do
+      expect(epic).to be_use_elasticsearch
+    end
+
+    context 'when an epic is created' do
+      let(:epic) { build(:epic, group: group) }
+
+      it 'tracks the epic' do
+        expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(epic).once
+        epic.save!
+      end
+    end
+
+    context 'when an epic is updated' do
+      it 'tracks the epic' do
+        expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(epic).once
+        epic.update!(title: 'A new title')
+      end
+    end
+
+    context 'when an epic is deleted' do
+      it 'tracks the epic' do
+        expect(::Elastic::ProcessBookkeepingService).to receive(:track!).with(epic).once
+        epic.destroy!
+      end
+
+      it 'deletes the epic from elasticsearch', :elastic_clean do
+        allow(::Elastic::ProcessBookkeepingService).to receive(:track!).and_call_original
+
+        epic = create(:epic, group: group)
+        ensure_elasticsearch_index!
+        expect(epics_in_index).to eq([epic.id])
+
+        epic.destroy!
+
+        ensure_elasticsearch_index!
+        expect(epics_in_index).to be_empty
+      end
+    end
+  end
+
+  shared_examples 'epics do not get tracked in Elasticsearch' do
+    it 'use_elasticsearch? is false' do
+      expect(epic).not_to be_use_elasticsearch
+    end
+
+    context 'when an epic is created' do
+      let(:epic) { build(:epic, group: group) }
+
+      it 'does not track the epic' do
+        expect(::Elastic::ProcessBookkeepingService).not_to receive(:track!).with(epic)
+        epic.save!
+      end
+    end
+
+    context 'when an epic is updated' do
+      it 'does not track the epic' do
+        expect(::Elastic::ProcessBookkeepingService).not_to receive(:track!).with(epic)
+        epic.update!(title: 'A new title')
+      end
+    end
+
+    context 'when an epic is deleted' do
+      it 'does not track the epic' do
+        expect(::Elastic::ProcessBookkeepingService).not_to receive(:track!).with(epic)
+        epic.destroy!
+      end
+    end
+  end
+
+  it_behaves_like 'epics get tracked in Elasticsearch'
+
+  context 'when elasticsearch_limit_indexing? is true' do
+    before do
+      stub_ee_application_setting(elasticsearch_limit_indexing?: true)
+    end
+
+    context 'if the parent group is not in the limited indexes list' do
+      it_behaves_like 'epics do not get tracked in Elasticsearch'
+    end
+
+    context 'if the parent group is in the limited indexes list' do
+      before do
+        create(:elasticsearch_indexed_namespace, namespace: parent_group)
+      end
+
+      it_behaves_like 'epics get tracked in Elasticsearch'
+    end
+  end
+
+  def epics_in_index
+    client.search(index: epic_index).dig('hits', 'hits').map { |hit| hit['_source']['id'] }
+  end
+end

ee/spec/lib/elastic/latest/epic_class_proxy_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Elastic::Latest::EpicClassProxy, feature_category: :global_search do
+  it 'raises an error when calling elastic_search' do
+    expect { described_class.new(Epic, use_separate_indices: true).elastic_search('*') }
+      .to raise_error(NotImplementedError)
+  end
+end

ee/spec/lib/elastic/latest/epic_config_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative './config_shared_examples'
+
+RSpec.describe Elastic::Latest::EpicConfig, feature_category: :global_search do
+  describe '.settings' do
+    it_behaves_like 'config settings return correct values'
+  end
+
+  describe '.mappings' do
+    it 'returns config' do
+      expect(described_class.mapping).to be_a(Elasticsearch::Model::Indexing::Mappings)
+    end
+  end
+
+  describe '.index_name' do
+    it 'includes' do
+      expect(described_class.index_name).to include('-epics')
+    end
+  end
+end

ee/spec/lib/elastic/latest/epic_instance_proxy_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Elastic::Latest::EpicInstanceProxy, feature_category: :global_search do
+  let_it_be(:parent_group) { create(:group) }
+  let_it_be(:group) { create(:group, parent: parent_group) }
+  let_it_be(:label) { create(:group_label, group: group) }
+  let_it_be(:epic) { create(:labeled_epic, :use_fixed_dates, :opened, group: group, labels: [label]) }
+
+  subject { described_class.new(epic) }
+
+  describe '#as_indexed_json' do
+    let(:result) { subject.as_indexed_json.with_indifferent_access }
+
+    it 'serializes the object as a hash' do
+      expect(result).to include(
+        id: epic.id,
+        iid: epic.iid,
+        group_id: group.id,
+        created_at: epic.created_at,
+        updated_at: epic.updated_at,
+        title: epic.title,
+        description: epic.description,
+        state: 'opened',
+        confidential: epic.confidential,
+        author_id: epic.author_id,
+        label_ids: [label.id.to_s],
+        start_date: epic.start_date,
+        due_date: epic.due_date,
+        traversal_ids: "#{parent_group.id}-#{group.id}-",
+        hashed_root_namespace_id: ::Search.hash_namespace_id(parent_group.id),
+        visibility_level: group.visibility_level,
+        schema_version: 2306,
+        type: 'epic'
+      )
+    end
+
+    context 'with start date inherited date from child epic and due date inherited from milestone' do
+      let_it_be(:epic) { create(:epic) }
+      let_it_be(:child_epic) { create(:epic, :use_fixed_dates) }
+      let_it_be(:milestone) { create(:milestone, :with_dates) }
+
+      before do
+        epic.start_date_sourcing_epic = child_epic
+        epic.due_date_sourcing_milestone = milestone
+        epic.save!
+      end
+
+      it 'sets start and due dates to inherited dates' do
+        expect(result[:start_date]).to eq(child_epic.start_date)
+        expect(result[:due_date]).to eq(milestone.due_date)
+      end
+    end
+  end
+
+  describe '#es_parent' do
+    it 'contains group id' do
+      expect(subject.es_parent).to eq("group_#{parent_group.id}")
+    end
+  end
+end

ee/spec/models/epic_spec.rb

@@ -1184,6 +1184,84 @@ def as_item(item)
     end
   end
 
+  describe 'ES related specs' do
+    let_it_be(:epic) { create(:epic, group: group) }
+
+    context 'when create epic index migration is not finished' do
+      before do
+        allow(::Elastic::DataMigrationService).to receive(:migration_has_finished?)
+          .with(:create_epic_index).and_return(false)
+      end
+
+      it 'use_elasticsearch? is false' do
+        expect(epic).not_to be_use_elasticsearch
+      end
+    end
+
+    context 'when create epic index migration is finished' do
+      before do
+        allow(::Elastic::DataMigrationService).to receive(:migration_has_finished?)
+          .with(:create_epic_index).and_return(true)
+      end
+
+      context 'when the group has use_elasticsearch? as true' do
+        before do
+          allow(group).to receive(:use_elasticsearch?).and_return(true)
+        end
+
+        it 'use_elasticsearch? is true' do
+          expect(epic).to be_use_elasticsearch
+        end
+
+        context 'when feature flag is disabled' do
+          before do
+            stub_feature_flags(elastic_index_epics: false)
+          end
+
+          it 'use_elasticsearch? is false' do
+            expect(epic).not_to be_use_elasticsearch
+          end
+        end
+
+        context 'with elasticsearch enabled' do
+          before do
+            allow(Gitlab::CurrentSettings.current_application_settings)
+              .to receive(:elasticsearch_indexing?).and_return(true)
+          end
+
+          it 'calls ::Elastic::ProcessBookkeepingService.track! when the epic is updated' do
+            expect(Elastic::ProcessBookkeepingService).to receive(:track!).with(*epic).once
+
+            epic.update!(title: 'A new title')
+          end
+        end
+      end
+
+      context 'when the group has use_elasticsearch? as false' do
+        before do
+          allow(group).to receive(:use_elasticsearch?).and_return(false)
+        end
+
+        it 'use_elasticsearch? is false' do
+          expect(epic).not_to be_use_elasticsearch
+        end
+
+        context 'with elasticsearch enabled' do
+          before do
+            allow(Gitlab::CurrentSettings.current_application_settings)
+              .to receive(:elasticsearch_indexing?).and_return(true)
+          end
+
+          it 'does not call ::Elastic::ProcessBookkeepingService.track! when the epic is updated' do
+            expect(Elastic::ProcessBookkeepingService).not_to receive(:track!).with(*epic)
+
+            epic.update!(title: 'A new title')
+          end
+        end
+      end
+    end
+  end
+
   describe '#epic_link_type' do
     let_it_be(:source_epic) { create(:epic, group: group) }
     let_it_be(:target_epic) { create(:epic, group: group) }

ee/spec/services/elastic/cluster_reindexing_service_spec.rb

@@ -78,7 +78,7 @@
         expect { cluster_reindexing_service.execute }.to change { task.reload.state }.from('indexing_paused').to('reindexing')
 
         subtasks = task.subtasks
-        expect(subtasks.count).to eq(8)
+        expect(subtasks.count).to eq(helper.standalone_indices_proxies.count + 1) # +1 for main index
 
         subtask_1 = subtasks.find { |subtask| subtask.alias_name == main_alias }
         slice_1 = subtask_1.slices.first

ee/spec/services/elastic/process_bookkeeping_service_spec.rb

@@ -463,6 +463,64 @@
           expect { described_class.new.execute }.not_to exceed_all_query_limit(control)
         end
       end
+
+      it 'does not have N+1 queries for epics' do
+        epics = create_list(:epic, 2, :use_fixed_dates)
+
+        described_class.track!(*epics)
+
+        control = ActiveRecord::QueryRecorder.new(skip_cached: false) { described_class.new.execute }
+
+        epics += create_list(:epic, 3, :use_fixed_dates)
+
+        described_class.track!(*epics)
+
+        expect { described_class.new.execute }.not_to exceed_all_query_limit(control)
+      end
+
+      it 'does not have N+1 queries for epics with inherited dates' do
+        child_epic = create(:epic, :use_fixed_dates)
+        milestone = create(:milestone, :with_dates)
+
+        epics = create_list(:epic, 2)
+        epics.each do |epic|
+          epic.start_date_sourcing_epic = child_epic
+          epic.due_date_sourcing_milestone = milestone
+          epic.save!
+        end
+
+        described_class.track!(*epics)
+
+        control = ActiveRecord::QueryRecorder.new(skip_cached: false) { described_class.new.execute }
+
+        epics += create_list(:epic, 3)
+        epics.each do |epic|
+          epic.start_date_sourcing_epic = child_epic
+          epic.due_date_sourcing_milestone = milestone
+          epic.save!
+        end
+
+        described_class.track!(*epics)
+
+        expect { described_class.new.execute }.not_to exceed_all_query_limit(control)
+      end
+
+      it 'does not have N+1 queries for epics in a group with multiple parents' do
+        parent_group = create(:group)
+        group = create(:group, parent: parent_group)
+
+        epics = create_list(:epic, 2, group: group)
+
+        described_class.track!(*epics)
+
+        control = ActiveRecord::QueryRecorder.new(skip_cached: false) { described_class.new.execute }
+
+        epics += create_list(:epic, 3, group: group)
+
+        described_class.track!(*epics)
+
+        expect { described_class.new.execute }.not_to exceed_all_query_limit(control)
+      end
     end
 
     def expect_processing(*refs, failures: [])

ee/spec/workers/search/index_curation_worker_spec.rb

@@ -11,7 +11,7 @@
   let(:logger) { ::Gitlab::Elasticsearch::Logger.build }
 
   describe '#curator_settings' do
-    let(:standalone_index_types) { %w[commits issues merge_requests notes users wikis projects] }
+    let(:standalone_index_types) { %w[commits issues merge_requests notes users wikis projects epics] }
     let(:curation_include_patterns) { [main_index_pattern] + standalone_index_types.map { |x| /#{x}/ } }
     let(:main_index_target_name) { "gitlab-test" }
     let(:main_index_name) { "gitlab-test-20220923-1517" }